Skip to content

Commit 97deca0

Browse files
author
skeptrune
committed
feature: add CI workflow to push images for pdf2md service
1 parent 6f568a8 commit 97deca0

File tree

3 files changed

+152
-40
lines changed

3 files changed

+152
-40
lines changed

‎.github/ISSUE_TEMPLATE/issue-template.md‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,7 @@ assignees: ''
1313

1414
### Target(s)
1515

16-
<replace w/ one or more of the following options: `server`, `search`, `chat`>
17-
18-
### Requirement to close
19-
20-
<please describe what is required to close this issue here>
16+
<replace w/ name of the service(s) which are associated with this issue>
2117

2218
### Community channels
2319

‎.github/workflows/push-pdf2md-server.yml‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
name: Create Docker Images
2+
3+
concurrency:
4+
group: ${{ github.workflow }}-${{ github.head_ref }}
5+
cancel-in-progress: true
6+
7+
on:
8+
workflow_dispatch:
9+
push:
10+
branches:
11+
- 'main'
12+
paths:
13+
- 'pdf2md/server/**'
14+
15+
jobs:
16+
pdf2md-server:
17+
name: Push PDF2MD Server image
18+
runs-on: ${{ matrix.runner }}
19+
strategy:
20+
matrix:
21+
runner: [blacksmith-8vcpu-ubuntu-2204]
22+
platform: [linux/amd64]
23+
exclude:
24+
- runner: blacksmith-8vcpu-ubuntu-2204
25+
platform: linux/arm64
26+
- runner: blacksmith-8vcpu-ubuntu-2204-arm
27+
platform: linux/amd64
28+
steps:
29+
- name: Checkout the repo
30+
uses: actions/checkout@v4
31+
32+
- name: Setup buildx
33+
uses: docker/setup-buildx-action@v3
34+
35+
- name: Login to Docker Hub
36+
uses: docker/login-action@v3
37+
with:
38+
username: ${{ secrets.DOCKER_USERNAME }}
39+
password: ${{ secrets.DOCKER_PASSWORD }}
40+
41+
- name: Docker meta
42+
id: meta
43+
uses: docker/metadata-action@v5
44+
with:
45+
images: |
46+
trieve/pdf2md-server
47+
tags: |
48+
type=raw,latest
49+
type=sha
50+
51+
- name: Build and push Docker image
52+
uses: useblacksmith/build-push-action@v1.0.0-beta
53+
with:
54+
platforms: ${{ matrix.platform }}
55+
context: pdf2md/
56+
file: ./pdf2md/server/Dockerfile.pdf2md-server
57+
push: true
58+
tags: ${{ steps.meta.outputs.tags }}
59+
labels: ${{ steps.meta.outputs.labels }}
60+
61+
chunk-worker:
62+
name: Push PDF2MD Chunk Worker image
63+
runs-on: ${{ matrix.runner }}
64+
strategy:
65+
matrix:
66+
runner: [blacksmith-8vcpu-ubuntu-2204]
67+
platform: [linux/amd64]
68+
exclude:
69+
- runner: blacksmith-8vcpu-ubuntu-2204
70+
platform: linux/arm64
71+
- runner: blacksmith-8vcpu-ubuntu-2204-arm
72+
platform: linux/amd64
73+
steps:
74+
- name: Checkout the repo
75+
uses: actions/checkout@v4
76+
77+
- name: Setup buildx
78+
uses: docker/setup-buildx-action@v3
79+
80+
- name: Login to Docker Hub
81+
uses: docker/login-action@v3
82+
with:
83+
username: ${{ secrets.DOCKER_USERNAME }}
84+
password: ${{ secrets.DOCKER_PASSWORD }}
85+
86+
- name: Docker meta
87+
id: meta
88+
uses: docker/metadata-action@v5
89+
with:
90+
images: |
91+
trieve/chunk-worker
92+
tags: |
93+
type=raw,latest
94+
type=sha
95+
96+
- name: Build and push Docker image
97+
uses: useblacksmith/build-push-action@v1.0.0-beta
98+
with:
99+
platforms: ${{ matrix.platform }}
100+
context: pdf2md/
101+
file: ./pdf2md/server/Dockerfile.chunk-worker
102+
push: true
103+
tags: ${{ steps.meta.outputs.tags }}
104+
labels: ${{ steps.meta.outputs.labels }}
105+
106+
supervisor-worker:
107+
name: Push PDF2MD Supervisor Worker image
108+
runs-on: ${{ matrix.runner }}
109+
strategy:
110+
matrix:
111+
runner: [blacksmith-8vcpu-ubuntu-2204]
112+
platform: [linux/amd64]
113+
exclude:
114+
- runner: blacksmith-8vcpu-ubuntu-2204
115+
platform: linux/arm64
116+
- runner: blacksmith-8vcpu-ubuntu-2204-arm
117+
platform: linux/amd64
118+
steps:
119+
- name: Checkout the repo
120+
uses: actions/checkout@v4
121+
122+
- name: Setup buildx
123+
uses: docker/setup-buildx-action@v3
124+
125+
- name: Login to Docker Hub
126+
uses: docker/login-action@v3
127+
with:
128+
username: ${{ secrets.DOCKER_USERNAME }}
129+
password: ${{ secrets.DOCKER_PASSWORD }}
130+
131+
- name: Docker meta
132+
id: meta
133+
uses: docker/metadata-action@v5
134+
with:
135+
images: |
136+
trieve/supervisor-worker
137+
tags: |
138+
type=raw,latest
139+
type=sha
140+
141+
- name: Build and push Docker image
142+
uses: useblacksmith/build-push-action@v1.0.0-beta
143+
with:
144+
platforms: ${{ matrix.platform }}
145+
context: pdf2md/
146+
file: ./pdf2md/server/Dockerfile.supervisor-worker
147+
push: true
148+
tags: ${{ steps.meta.outputs.tags }}
149+
labels: ${{ steps.meta.outputs.labels }}

‎pdf2md/server/src/workers/chunk-worker.rs‎

Lines changed: 2 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
use chm::tools::migrations::{run_pending_migrations, SetupArgs};
2-
use pdf2md_server::{
2+
use file_chunker::{
33
errors::ServiceError,
44
get_env,
55
models::ChunkingTask,
@@ -100,7 +100,7 @@ async fn main() {
100100
pub async fn chunk_sub_pdf(
101101
task: ChunkingTask,
102102
clickhouse_client: clickhouse::Client,
103-
) -> Result<(), pdf2md_server::errors::ServiceError> {
103+
) -> Result<(), file_chunker::errors::ServiceError> {
104104
let bucket = get_aws_bucket()?;
105105
let file_data = bucket
106106
.get_object(task.file_name.clone())
@@ -115,38 +115,5 @@ pub async fn chunk_sub_pdf(
115115
let result = chunk_pdf(file_data, task.clone(), task.page_range, &clickhouse_client).await?;
116116
log::info!("Got {} pages for {:?}", result.len(), task.task_id);
117117

118-
let mut page_inserter = clickhouse_client.insert("file_chunks").map_err(|e| {
119-
log::error!("Error inserting recommendations: {:?}", e);
120-
ServiceError::InternalServerError(format!("Error inserting task: {:?}", e))
121-
})?;
122-
123-
for page in &result {
124-
page_inserter.write(page).await.map_err(|e| {
125-
log::error!("Error inserting task: {:?}", e);
126-
ServiceError::InternalServerError(format!("Error inserting task: {:?}", e))
127-
})?;
128-
}
129-
130-
page_inserter.end().await.map_err(|e| {
131-
log::error!("Error inserting task: {:?}", e);
132-
ServiceError::InternalServerError(format!("Error inserting task: {:?}", e))
133-
})?;
134-
135-
let prev_task =
136-
pdf2md_server::operators::clickhouse::get_task(task.task_id, &clickhouse_client).await?;
137-
138-
let pages_processed = prev_task.pages_processed + 1;
139-
140-
if pages_processed == prev_task.pages {
141-
update_task_status(task.task_id, FileTaskStatus::Completed, &clickhouse_client).await?;
142-
} else {
143-
update_task_status(
144-
task.task_id,
145-
FileTaskStatus::ChunkingFile(result.len() as u32 + prev_task.pages_processed),
146-
&clickhouse_client,
147-
)
148-
.await?;
149-
}
150-
151118
Ok(())
152119
}

0 commit comments

Comments
 (0)