Skip to content

Commit 26ebb17

Browse files
jaseemjaskpjaseemjaskp
committed
Initial commit on Unstract
1 parent 4d0f0d2 commit 26ebb17

File tree

1,048 files changed

+117863
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,048 files changed

+117863
-0
lines changed

‎.github/pull_request_template.md

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
## What
2+
3+
...
4+
5+
## Why
6+
7+
...
8+
9+
## How
10+
11+
...
12+
13+
## Relevant Docs
14+
15+
-
16+
17+
## Related Issues or PRs
18+
19+
-
20+
21+
## Dependencies Versions / Env Variables
22+
23+
-
24+
25+
## Notes on Testing
26+
27+
...
28+
29+
## Screenshots
30+
31+
...
32+
33+
## Checklist
34+
35+
I have read and understood the [Contribution Guidelines]().

‎.github/workflows/ci-container-build.yaml

+86
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
name: Container Image Build Test for PRs
2+
3+
env:
4+
VERSION: ci-test # Used for docker tag
5+
6+
on:
7+
push:
8+
branches:
9+
- main
10+
- development
11+
paths:
12+
- 'backend/**'
13+
- 'frontend/**'
14+
- 'unstract/**'
15+
- 'document-service/**'
16+
- 'platform-service/**'
17+
- 'x2text-service/**'
18+
- 'worker/**'
19+
- 'docker/dockerfiles/**'
20+
pull_request:
21+
types: [opened, synchronize, reopened, ready_for_review]
22+
branches:
23+
- main
24+
- development
25+
paths:
26+
- 'backend/**'
27+
- 'frontend/**'
28+
- 'unstract/**'
29+
- 'document-service/**'
30+
- 'platform-service/**'
31+
- 'x2text-service/**'
32+
- 'worker/**'
33+
- 'docker/dockerfiles/**'
34+
35+
jobs:
36+
build:
37+
if: github.event.pull_request.draft == false
38+
runs-on: custom-k8s-runner
39+
steps:
40+
- name: Checkout code
41+
uses: actions/checkout@v3
42+
- name: Login to Docker Hub
43+
uses: docker/login-action@v2
44+
with:
45+
username: ${{ secrets.DOCKERHUB_USERNAME }}
46+
password: ${{ secrets.DOCKERHUB_TOKEN }}
47+
- name: Container Build
48+
working-directory: ./docker
49+
run: |
50+
docker compose -f docker-compose.build.yaml build
51+
- name: Container Run
52+
working-directory: ./docker
53+
run: |
54+
cp ../backend/sample.env ../backend/.env
55+
cp ../document-service/sample.env ../document-service/.env
56+
cp ../platform-service/sample.env ../platform-service/.env
57+
cp ../prompt-service/sample.env ../prompt-service/.env
58+
cp ../worker/sample.env ../worker/.env
59+
cp ../x2text-service/sample.env ../x2text-service/.env
60+
cp sample.essentials.env essentials.env
61+
62+
docker compose -f docker-compose.yaml up -d
63+
sleep 10
64+
docker compose -f docker-compose.yaml ps -a
65+
# Get the names of exited containers
66+
custom_format="{{.Name}}\t{{.Image}}\t{{.Service}}"
67+
EXITED_CONTAINERS=$(docker compose -f docker-compose.yaml ps -a --filter status=exited --format "$custom_format")
68+
69+
line_count=$(echo "$EXITED_CONTAINERS" | wc -l)
70+
71+
if [ "$line_count" -gt 1 ]; then
72+
echo "Exited Containers: $EXITED_CONTAINERS"
73+
74+
SERVICE=$(echo "$EXITED_CONTAINERS" | awk 'NR>0 {print $3}')
75+
echo "Exited Services:"
76+
echo "$SERVICE"
77+
echo "There are exited containers."
78+
# Print logs of exited containers
79+
IFS=$'\n'
80+
for SERVICE in $SERVICE; do
81+
docker compose -f docker-compose.yaml logs "$SERVICE"
82+
done
83+
docker compose -f docker-compose.yaml down -v
84+
exit 1
85+
fi
86+
docker compose -f docker-compose.yaml down -v

‎.github/workflows/docker-build-push-dev.yaml

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
name: Unstract Docker Image Build and Push (Development)
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
tag:
7+
description: "Docker image tag"
8+
required: true
9+
default: "latest"
10+
service_name:
11+
description: "Service to build"
12+
required: true
13+
default: "backend" # Provide a default value
14+
type: choice
15+
options: # Define available options
16+
- all-services
17+
- frontend
18+
- backend
19+
- document-service
20+
- platform-service
21+
- worker
22+
- prompt-service
23+
- x2text-service
24+
25+
run-name: "[${{ inputs.service_name }}] Docker Image Build and Push (Development)"
26+
27+
jobs:
28+
build-and-push:
29+
runs-on: custom-k8s-runner
30+
steps:
31+
- name: Output Inputs
32+
run: echo "${{ toJSON(github.event.inputs) }}"
33+
34+
- name: Checkout code
35+
uses: actions/checkout@v2
36+
37+
- name: Login to Docker Hub
38+
uses: docker/login-action@v1
39+
with:
40+
username: ${{ secrets.DOCKERHUB_USERNAME }}
41+
password: ${{ secrets.DOCKERHUB_TOKEN }}
42+
43+
# Build and push Docker image for the specified service
44+
- name: Build and push image for ${{ github.event.inputs.service_name }}
45+
working-directory: ./docker
46+
if: github.event.inputs.service_name != 'all-services'
47+
run: |
48+
VERSION=${{ github.event.inputs.tag }} docker compose -f docker-compose.build.yaml build --no-cache ${{ github.event.inputs.service_name }}
49+
docker push unstract/${{ github.event.inputs.service_name }}:${{ github.event.inputs.tag }}
50+
51+
# Build and push all service images
52+
- name: Build and push all images
53+
working-directory: ./docker
54+
if: github.event.inputs.service_name == 'all-services'
55+
run: |
56+
VERSION=${{ github.event.inputs.tag }} docker-compose -f docker-compose.build.yaml build --no-cache
57+
# Push all built images
58+
docker push unstract/backend:${{ github.event.inputs.tag }}
59+
docker push unstract/frontend:${{ github.event.inputs.tag }}
60+
docker push unstract/document-service:${{ github.event.inputs.tag }}
61+
docker push unstract/platform-service:${{ github.event.inputs.tag }}
62+
docker push unstract/worker:${{ github.event.inputs.tag }}
63+
docker push unstract/prompt-service:${{ github.event.inputs.tag }}
64+
docker push unstract/x2text-service:${{ github.event.inputs.tag }}

‎.github/workflows/docker-tools-build-push.yaml

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
name: Unstract Tools Docker Image Build and Push (Development)
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
tag:
7+
description: "Docker image tag"
8+
required: true
9+
default: "latest"
10+
service_name:
11+
description: "Tool to build"
12+
required: true
13+
default: "tool-classifier" # Provide a default value
14+
type: choice
15+
options: # Define available options
16+
- tool-classifier
17+
- tool-doc-pii-redactor
18+
- tool-indexer
19+
- tool-ocr
20+
- tool-translate
21+
- tool-structure
22+
- tool-text-extractor
23+
24+
run-name: "[${{ inputs.service_name }}] Docker Image Build and Push (Development)"
25+
26+
jobs:
27+
build-and-push:
28+
runs-on: custom-k8s-runner
29+
steps:
30+
- name: Output Inputs
31+
run: echo "${{ toJSON(github.event.inputs) }}"
32+
33+
- name: Checkout code
34+
uses: actions/checkout@v2
35+
36+
- name: Login to Docker Hub
37+
uses: docker/login-action@v1
38+
with:
39+
username: ${{ secrets.DOCKERHUB_USERNAME }}
40+
password: ${{ secrets.DOCKERHUB_TOKEN }}
41+
42+
- name: Build tool-classifier
43+
if: github.event.inputs.service_name=='tool-classifier'
44+
run: docker build -t unstract/${{github.event.inputs.service_name}}:${{ github.event.inputs.tag }} ./tools/classifier
45+
- name: Build tool-doc-pii-redactor
46+
if: github.event.inputs.service_name=='tool-doc-pii-redactor'
47+
run: docker build -t unstract/${{github.event.inputs.service_name}}:${{ github.event.inputs.tag }} ./tools/doc_pii_redactor
48+
- name: Build tool-indexer
49+
if: github.event.inputs.service_name=='tool-indexer'
50+
run: docker build -t unstract/${{github.event.inputs.service_name}}:${{ github.event.inputs.tag }} ./tools/indexer
51+
- name: Build tool-ocr
52+
if: github.event.inputs.service_name=='tool-ocr'
53+
run: docker build -t unstract/${{github.event.inputs.service_name}}:${{ github.event.inputs.tag }} ./tools/ocr
54+
- name: Build tool-translate
55+
if: github.event.inputs.service_name=='tool-translate'
56+
run: docker build -t unstract/${{github.event.inputs.service_name}}:${{ github.event.inputs.tag }} ./tools/translate
57+
- name: Build tool-structure
58+
if: github.event.inputs.service_name=='tool-structure'
59+
run: docker build -t unstract/${{github.event.inputs.service_name}}:${{ github.event.inputs.tag }} ./tools/structure
60+
- name: Build tool-text-extractor
61+
if: github.event.inputs.service_name=='tool-text-extractor'
62+
run: docker build -t unstract/${{github.event.inputs.service_name}}:${{ github.event.inputs.tag }} ./tools/text_extractor
63+
64+
- name: Push Docker image to Docker Hub
65+
run: docker push unstract/${{ github.event.inputs.service_name }}:${{ github.event.inputs.tag }}

‎.github/workflows/production-build.yaml

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
name: Unstract Docker Image Build and Push (Production)
2+
3+
on:
4+
release:
5+
types:
6+
- created
7+
8+
run-name: "[${{ github.event.release.tag_name }}] Docker Image Build and Push (Development)"
9+
10+
jobs:
11+
build-and-push:
12+
runs-on: custom-k8s-runner
13+
strategy:
14+
matrix:
15+
service_name: [backend, frontend, document-service, platform-service, prompt-service, worker, x2text-service]
16+
17+
steps:
18+
- name: Checkout code
19+
uses: actions/checkout@v2
20+
with:
21+
ref: ${{ github.event.release.tag_name }}
22+
23+
- name: Login to Docker Hub
24+
uses: docker/login-action@v1
25+
with:
26+
username: ${{ secrets.DOCKERHUB_USERNAME }}
27+
password: ${{ secrets.DOCKERHUB_TOKEN }}
28+
29+
- name: Build and push image
30+
working-directory: ./docker
31+
run: |
32+
VERSION=${{ github.event.release.tag_name }} docker-compose -f docker-compose.build.yaml build --no-cache ${{ matrix.service_name }}
33+
docker push unstract/${{ matrix.service_name }}:${{ github.event.release.tag_name }}

0 commit comments

Comments
 (0)