ops: cleanup collapse query logs, add sha tag to clustering script #1245
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Create Docker Images | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref }} | |
| cancel-in-progress: true | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - 'main' | |
| paths: | |
| - 'server/**' | |
| - '.github/workflows/**' | |
| jobs: | |
| build_server: | |
| name: Push No OCR image | |
| runs-on: ${{ matrix.runner }} | |
| strategy: | |
| matrix: | |
| runner: [blacksmith-8vcpu-ubuntu-2204] | |
| platform: [linux/amd64] | |
| exclude: | |
| - runner: blacksmith-8vcpu-ubuntu-2204 | |
| platform: linux/arm64 | |
| - runner: blacksmith-8vcpu-ubuntu-2204-arm | |
| platform: linux/amd64 | |
| steps: | |
| - name: Checkout the repo | |
| uses: actions/checkout@v4 | |
| # - name: Set up QEMU | |
| # uses: docker/setup-qemu-action@v3 | |
| - name: Setup buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| # list of Docker images to use as base name for tags | |
| images: | | |
| trieve/server | |
| tags: | | |
| type=raw,latest | |
| type=sha | |
| - name: Build and push Docker image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| platforms: ${{ matrix.platform }} | |
| cache-from: type=registry,ref=trieve/buildcache:server-${{matrix.runner}} | |
| cache-to: type=registry,ref=trieve/buildcache:server-${{matrix.runner}},mode=max | |
| context: server/ | |
| file: ./server/Dockerfile.server | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| ingest_worker: | |
| name: Push Ingest image | |
| runs-on: ${{ matrix.runner }} | |
| strategy: | |
| matrix: | |
| runner: [blacksmith-8vcpu-ubuntu-2204] | |
| platform: [linux/amd64] | |
| exclude: | |
| - runner: blacksmith-8vcpu-ubuntu-2204 | |
| platform: linux/arm64 | |
| - runner: blacksmith-8vcpu-ubuntu-2204-arm | |
| platform: linux/amd64 | |
| steps: | |
| - name: Checkout the repo | |
| uses: actions/checkout@v4 | |
| - name: Setup buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| # list of Docker images to use as base name for tags | |
| images: | | |
| trieve/ingest | |
| tags: | | |
| type=raw,latest | |
| type=sha | |
| - name: Build and push Docker image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| platforms: ${{ matrix.platform }} | |
| cache-from: type=registry,ref=trieve/buildcache:ingest-${{matrix.runner}} | |
| cache-to: type=registry,ref=trieve/buildcache:ingest-${{matrix.runner}},mode=max | |
| context: server/ | |
| file: ./server/Dockerfile.ingestion-worker | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| delete_worker: | |
| name: Push Delete worker | |
| runs-on: ${{ matrix.runner }} | |
| strategy: | |
| matrix: | |
| runner: [blacksmith-8vcpu-ubuntu-2204] | |
| platform: [linux/amd64] | |
| exclude: | |
| - runner: blacksmith-8vcpu-ubuntu-2204 | |
| platform: linux/arm64 | |
| - runner: blacksmith-8vcpu-ubuntu-2204-arm | |
| platform: linux/amd64 | |
| steps: | |
| - name: Checkout the repo | |
| uses: actions/checkout@v4 | |
| - name: Setup buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| # list of Docker images to use as base name for tags | |
| images: | | |
| trieve/delete_worker | |
| tags: | | |
| type=raw,latest | |
| type=sha | |
| - name: Build and push Docker image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| platforms: ${{ matrix.platform }} | |
| cache-from: type=registry,ref=trieve/buildcache:delete_worker-${{matrix.runner}} | |
| cache-to: type=registry,ref=trieve/buildcache:delete_worker-${{matrix.runner}},mode=max | |
| context: server/ | |
| file: ./server/Dockerfile.delete-worker | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| file_worker: | |
| name: Push File Worker image | |
| runs-on: ${{ matrix.runner }} | |
| strategy: | |
| matrix: | |
| runner: [blacksmith-8vcpu-ubuntu-2204] | |
| platform: [linux/amd64] | |
| exclude: | |
| - runner: blacksmith-8vcpu-ubuntu-2204 | |
| platform: linux/arm64 | |
| - runner: blacksmith-8vcpu-ubuntu-2204-arm | |
| platform: linux/amd64 | |
| steps: | |
| - name: Checkout the repo | |
| uses: actions/checkout@v4 | |
| - name: Setup buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| # list of Docker images to use as base name for tags | |
| images: | | |
| trieve/file_worker | |
| tags: | | |
| type=raw,latest | |
| type=sha | |
| - name: Build and push Docker image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| platforms: ${{ matrix.platform }} | |
| cache-from: type=registry,ref=trieve/buildcache:file_worker-${{matrix.runner}} | |
| cache-to: type=registry,ref=trieve/buildcache:file_worker-${{matrix.runner}},mode=max | |
| context: server/ | |
| file: ./server/Dockerfile.file-worker | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| group_worker: | |
| name: Push Group worker | |
| runs-on: ${{ matrix.runner }} | |
| strategy: | |
| matrix: | |
| runner: [blacksmith-8vcpu-ubuntu-2204] | |
| platform: [linux/amd64] | |
| exclude: | |
| - runner: blacksmith-8vcpu-ubuntu-2204 | |
| platform: linux/arm64 | |
| - runner: blacksmith-8vcpu-ubuntu-2204-arm | |
| platform: linux/amd64 | |
| steps: | |
| - name: Checkout the repo | |
| uses: actions/checkout@v4 | |
| - name: Setup buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| # list of Docker images to use as base name for tags | |
| images: | | |
| trieve/group_worker | |
| tags: | | |
| type=raw,latest | |
| type=sha | |
| - name: Build and push Docker image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| platforms: ${{ matrix.platform }} | |
| cache-from: type=registry,ref=trieve/buildcache:group_worker-${{matrix.runner}} | |
| cache-to: type=registry,ref=trieve/buildcache:group_worker-${{matrix.runner}},mode=max | |
| context: server/ | |
| file: ./server/Dockerfile.group-worker | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| sync_qdrant: | |
| name: Push Sync Qdrant script | |
| runs-on: ${{ matrix.runner }} | |
| strategy: | |
| matrix: | |
| runner: [blacksmith-8vcpu-ubuntu-2204] | |
| platform: [linux/amd64] | |
| exclude: | |
| - runner: blacksmith-8vcpu-ubuntu-2204 | |
| platform: linux/arm64 | |
| - runner: blacksmith-8vcpu-ubuntu-2204-arm | |
| platform: linux/amd64 | |
| steps: | |
| - name: Checkout the repo | |
| uses: actions/checkout@v4 | |
| - name: Setup buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| # list of Docker images to use as base name for tags | |
| images: | | |
| trieve/sync_qdrant | |
| tags: | | |
| type=raw,latest | |
| type=sha | |
| - name: Build and push Docker image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| platforms: ${{ matrix.platform }} | |
| cache-from: type=registry,ref=trieve/buildcache:sync_qdrant-${{matrix.runner}} | |
| cache-to: type=registry,ref=trieve/buildcache:sync_qdrant-${{matrix.runner}},mode=max | |
| context: server/ | |
| file: ./server/Dockerfile.sync-qdrant | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| queue_bm25: | |
| name: Push Queue BM25 Script | |
| runs-on: ${{ matrix.runner }} | |
| strategy: | |
| matrix: | |
| runner: [blacksmith-8vcpu-ubuntu-2204] | |
| platform: [linux/amd64] | |
| exclude: | |
| - runner: blacksmith-8vcpu-ubuntu-2204 | |
| platform: linux/arm64 | |
| - runner: blacksmith-8vcpu-ubuntu-2204-arm | |
| platform: linux/amd64 | |
| steps: | |
| - name: Checkout the repo | |
| uses: actions/checkout@v4 | |
| - name: Setup buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| # list of Docker images to use as base name for tags | |
| images: | | |
| trieve/queue-bm25-migration | |
| tags: | | |
| type=raw,latest | |
| type=sha | |
| - name: Build and push Docker image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| platforms: ${{ matrix.platform }} | |
| cache-from: type=registry,ref=trieve/buildcache:bm25-queue-${{matrix.runner}} | |
| cache-to: type=registry,ref=trieve/buildcache:bm25-queue-${{matrix.runner}},mode=max | |
| context: server/ | |
| file: ./server/Dockerfile.queue-bm25 | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| reindex-worker: | |
| name: Push Reindex Worker | |
| runs-on: ${{ matrix.runner }} | |
| strategy: | |
| matrix: | |
| runner: [blacksmith-8vcpu-ubuntu-2204] | |
| platform: [linux/amd64] | |
| exclude: | |
| - runner: blacksmith-8vcpu-ubuntu-2204 | |
| platform: linux/arm64 | |
| - runner: blacksmith-8vcpu-ubuntu-2204-arm | |
| platform: linux/amd64 | |
| steps: | |
| - name: Checkout the repo | |
| uses: actions/checkout@v4 | |
| - name: Setup buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| # list of Docker images to use as base name for tags | |
| images: | | |
| trieve/reindex-worker | |
| tags: | | |
| type=raw,latest | |
| type=sha | |
| - name: Build and push Docker image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| platforms: ${{ matrix.platform }} | |
| cache-from: type=registry,ref=trieve/buildcache:reindex-worker-${{matrix.runner}} | |
| cache-to: type=registry,ref=trieve/buildcache:reindex-worker-${{matrix.runner}},mode=max | |
| context: server/ | |
| file: ./server/Dockerfile.reindex-worker | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} |