fix(backend): cleanups part 1 #496
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # StorageHub: Network CI/CD Workflow | |
| # | |
| # Overview: | |
| # - Setup: detect whether to rebuild the Rust node and backend, determine image tag and force flags | |
| # - Build: conditionally build node/backend binaries and publish Docker images (if needed) | |
| # - Tests: | |
| # - Zombienet-based and network tests | |
| # - Integration test suites for solochain and parachain scenarios | |
| # | |
| # Notes: | |
| # - Requires Docker Hub credentials to push images | |
| # - Uses Blacksmith runners for performance | |
| # - Employs sccache and mold for fast Rust builds | |
| name: StorageHub Network CI/CD | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} | |
| cancel-in-progress: true | |
| on: | |
| pull_request: | |
| push: | |
| branches: [main] | |
| workflow_dispatch: | |
| inputs: | |
| image_tag: | |
| description: "Docker image tag to publish (e.g., v0.1.0)" | |
| required: false | |
| type: string | |
| release: | |
| types: [published] | |
| jobs: | |
| setup: | |
| runs-on: blacksmith-16vcpu-ubuntu-2404 | |
| outputs: | |
| node_changed: ${{ steps.node_check.outputs.changed }} | |
| backend_changed: ${{ steps.backend_check.outputs.changed }} | |
| image-tag: ${{ steps.set-tag.outputs.tag }} | |
| force_node: ${{ steps.flags.outputs.force_node }} | |
| force_backend: ${{ steps.flags.outputs.force_backend }} | |
| env: | |
| SKIP_BUILD_LABEL_PRESENT: ${{ contains(github.event.pull_request.labels.*.name, 'skip-node-build') }} | |
| SKIP_BACKEND_BUILD_LABEL_PRESENT: ${{ contains(github.event.pull_request.labels.*.name, 'skip-backend-build') }} | |
| FORCE_NODE_PUBLISH_LABEL_PRESENT: ${{ contains(github.event.pull_request.labels.*.name, 'force-node-publish') }} | |
| FORCE_BACKEND_PUBLISH_LABEL_PRESENT: ${{ contains(github.event.pull_request.labels.*.name, 'force-backend-publish') }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Check if the node needs rebuild | |
| id: node_check | |
| run: | | |
| if [[ "${{ github.event_name }}" == "release" ]]; then | |
| echo "changed=false" >> $GITHUB_OUTPUT | |
| exit 0 | |
| fi | |
| BASE_SHA="${{ github.event.pull_request.base.sha || github.event.before }}" | |
| HEAD_SHA="${{ github.sha }}" | |
| if [[ "${{ env.SKIP_BUILD_LABEL_PRESENT }}" != "true" ]] && git diff --name-only $BASE_SHA $HEAD_SHA | grep -E '^(client|node|pallets|runtime)/|^Cargo\.toml$'; then | |
| echo "changed=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "Comparing changes from $BASE_SHA to $HEAD_SHA" | |
| echo "changed=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Check if the MSP backend needs rebuild | |
| id: backend_check | |
| run: | | |
| if [[ "${{ github.event_name }}" == "release" ]]; then | |
| echo "changed=false" >> $GITHUB_OUTPUT | |
| exit 0 | |
| fi | |
| BASE_SHA="${{ github.event.pull_request.base.sha || github.event.before }}" | |
| HEAD_SHA="${{ github.sha }}" | |
| if [[ "${{ env.SKIP_BACKEND_BUILD_LABEL_PRESENT }}" != "true" ]] && git diff --name-only $BASE_SHA $HEAD_SHA | grep -E '^backend/|^Cargo\.toml$'; then | |
| echo "changed=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "Comparing changes from $BASE_SHA to $HEAD_SHA" | |
| echo "changed=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Set tag names for publishing | |
| id: set-tag | |
| run: | | |
| if [[ "${{ github.event_name }}" == 'release' ]]; then | |
| echo "tag=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT | |
| elif [[ "${{ github.event_name }}" == 'workflow_dispatch' && -n "${{ inputs.image_tag }}" ]]; then | |
| echo "tag=${{ inputs.image_tag }}" >> $GITHUB_OUTPUT | |
| elif [[ "${{ github.ref }}" == 'refs/heads/main' ]]; then | |
| echo "tag=latest" >> $GITHUB_OUTPUT | |
| elif [[ "${{ env.SKIP_BUILD_LABEL_PRESENT }}" == "true" && "${{ env.SKIP_BACKEND_BUILD_LABEL_PRESENT }}" == "true" ]]; then | |
| echo "tag=latest" >> $GITHUB_OUTPUT | |
| elif [[ "${{ steps.node_check.outputs.changed }}" == 'false' && "${{ steps.backend_check.outputs.changed }}" == 'false' && "${{ env.FORCE_NODE_PUBLISH_LABEL_PRESENT }}" != "true" && "${{ env.FORCE_BACKEND_PUBLISH_LABEL_PRESENT }}" != "true" ]]; then | |
| echo "tag=latest" >> $GITHUB_OUTPUT | |
| else | |
| echo "tag=sha-$(git log -1 --format='%H' | cut -c1-8)" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Set flags | |
| id: flags | |
| run: | | |
| if [[ "${{ github.event_name }}" == 'release' ]]; then | |
| echo "force_node=false" >> $GITHUB_OUTPUT | |
| echo "force_backend=true" >> $GITHUB_OUTPUT | |
| elif [[ "${{ github.event_name }}" == 'workflow_dispatch' && -n "${{ inputs.image_tag }}" ]]; then | |
| echo "force_node=false" >> $GITHUB_OUTPUT | |
| echo "force_backend=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "force_node=${{ env.FORCE_NODE_PUBLISH_LABEL_PRESENT }}" >> $GITHUB_OUTPUT | |
| echo "force_backend=${{ env.FORCE_BACKEND_PUBLISH_LABEL_PRESENT }}" >> $GITHUB_OUTPUT | |
| fi | |
| build_node: | |
| needs: [setup] | |
| name: "Build node binary if changed" | |
| runs-on: blacksmith-16vcpu-ubuntu-2404 | |
| env: | |
| SCCACHE_GHA_ENABLED: "true" | |
| RUSTC_WRAPPER: "sccache" | |
| CARGO_INCREMENTAL: "0" | |
| CARGO_TERM_COLOR: always | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Run sccache-cache | |
| uses: mozilla-actions/[email protected] | |
| - uses: actions-rust-lang/[email protected] | |
| with: | |
| cache: true | |
| - name: Cache cargo registry and git index | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo- | |
| - uses: rui314/setup-mold@v1 | |
| - name: Set build flags (use mold linker) | |
| run: echo "RUSTFLAGS=${{ env.RUSTFLAGS }} -C linker=clang -C link-arg=-fuse-ld=mold" >> $GITHUB_ENV | |
| - uses: ./.github/workflow-templates/setup-pnpm | |
| # Install required build dependencies (incl. binutils for readelf) | |
| - name: Install build deps | |
| run: sudo apt-get update && sudo apt-get install -y libpq-dev libclang-dev binutils | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build node binary | |
| if: needs.setup.outputs.node_changed == 'true' || needs.setup.outputs.force_node == 'true' | |
| run: cargo build -p storage-hub-node --release --locked | |
| - name: Build Stats | |
| run: sccache --show-stats || true | |
| - name: Check node binary built by mold | |
| if: needs.setup.outputs.node_changed == 'true' || needs.setup.outputs.force_node == 'true' | |
| run: readelf -p .comment target/release/storage-hub-node | |
| - name: Prepare artefacts | |
| if: needs.setup.outputs.node_changed == 'true' || needs.setup.outputs.force_node == 'true' | |
| run: | | |
| mkdir -p build | |
| mkdir -p runtimes | |
| cp target/release/storage-hub-node build/ | |
| cp target/release/wbuild/sh-*/sh*_runtime.compact.compressed.wasm runtimes/ | |
| - uses: actions/upload-artifact@v4 | |
| if: needs.setup.outputs.node_changed == 'true' || needs.setup.outputs.force_node == 'true' | |
| with: | |
| name: node | |
| path: build/storage-hub-node | |
| if-no-files-found: error | |
| build_backend: | |
| needs: [setup] | |
| name: "Build backend binary if changed" | |
| runs-on: blacksmith-16vcpu-ubuntu-2404 | |
| env: | |
| SCCACHE_GHA_ENABLED: "true" | |
| RUSTC_WRAPPER: "sccache" | |
| CARGO_INCREMENTAL: "0" | |
| CARGO_TERM_COLOR: always | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Run sccache-cache | |
| uses: mozilla-actions/[email protected] | |
| - uses: actions-rust-lang/[email protected] | |
| with: | |
| cache: true | |
| - name: Cache cargo registry and git index | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo- | |
| - uses: rui314/setup-mold@v1 | |
| - name: Set build flags (use mold linker) | |
| run: echo "RUSTFLAGS=${{ env.RUSTFLAGS }} -C linker=clang -C link-arg=-fuse-ld=mold" >> $GITHUB_ENV | |
| - uses: ./.github/workflow-templates/setup-pnpm | |
| - name: Install libpq-dev | |
| run: sudo apt-get update && sudo apt-get install -y libpq-dev libclang-dev | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build backend | |
| if: needs.setup.outputs.backend_changed == 'true' || needs.setup.outputs.force_backend == 'true' | |
| run: cargo build -p sh-msp-backend --release --locked | |
| - name: Build Stats | |
| run: sccache --show-stats || true | |
| - name: Check Backend Built By Mold | |
| if: needs.setup.outputs.backend_changed == 'true' || needs.setup.outputs.force_backend == 'true' | |
| run: readelf -p .comment target/release/sh-msp-backend | |
| - name: Prepare artefacts (backend) | |
| if: needs.setup.outputs.backend_changed == 'true' || needs.setup.outputs.force_backend == 'true' | |
| run: | | |
| mkdir -p build | |
| cp target/release/sh-msp-backend build/ | |
| - uses: actions/upload-artifact@v4 | |
| if: needs.setup.outputs.backend_changed == 'true' || needs.setup.outputs.force_backend == 'true' | |
| with: | |
| name: backend | |
| path: build/sh-msp-backend | |
| if-no-files-found: error | |
| push_node_image: | |
| needs: [build_node, setup] | |
| if: needs.setup.outputs.node_changed == 'true' || needs.setup.outputs.force_node == 'true' | |
| name: "Push node Docker image to Docker Hub" | |
| runs-on: blacksmith-16vcpu-ubuntu-2404 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download node binary from artefacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: node | |
| path: build/ | |
| - name: Prepare Docker tags | |
| id: prep | |
| run: | | |
| DOCKER_IMAGE=moonsonglabs/storage-hub | |
| TAGS="${DOCKER_IMAGE}:${{ needs.setup.outputs.image-tag }}" | |
| echo "tags=${TAGS}" >> $GITHUB_OUTPUT | |
| echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Setup Blacksmith Builder | |
| uses: useblacksmith/setup-docker-builder@v1 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_TOKEN }} | |
| - name: Build Docker image and push it to Docker Hub | |
| uses: useblacksmith/build-push-action@v2 | |
| with: | |
| context: . | |
| file: ./docker/storage-hub-node.Dockerfile | |
| push: true | |
| platforms: linux/amd64 | |
| tags: ${{ steps.prep.outputs.tags }} | |
| labels: | | |
| org.opencontainers.image.title=${{ github.event.repository.name }} | |
| org.opencontainers.image.description=${{ github.event.repository.description }} | |
| org.opencontainers.image.url=${{ github.event.repository.html_url }} | |
| org.opencontainers.image.source=${{ github.event.repository.clone_url }} | |
| org.opencontainers.image.revision=${{ github.sha }} | |
| org.opencontainers.image.licenses=${{ github.event.repository.license.spdx_id }} | |
| push_backend_image: | |
| needs: [build_backend, setup] | |
| if: needs.setup.outputs.backend_changed == 'true' || needs.setup.outputs.force_backend == 'true' | |
| name: "Push backend Docker image to Docker Hub" | |
| runs-on: blacksmith-16vcpu-ubuntu-2404 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download backend binary from artefacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: backend | |
| path: build/ | |
| - name: Prepare Docker tags | |
| id: prep | |
| run: | | |
| DOCKER_IMAGE=moonsonglabs/storage-hub-msp-backend | |
| TAGS="${DOCKER_IMAGE}:${{ needs.setup.outputs.image-tag }}" | |
| echo "tags=${TAGS}" >> $GITHUB_OUTPUT | |
| echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Blacksmith Builder | |
| uses: useblacksmith/setup-docker-builder@v1 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_TOKEN }} | |
| - name: Build Docker image and push it to Docker Hub | |
| uses: useblacksmith/build-push-action@v2 | |
| with: | |
| context: . | |
| file: ./docker/storage-hub-msp-backend.Dockerfile | |
| push: true | |
| platforms: linux/amd64 | |
| tags: ${{ steps.prep.outputs.tags }} | |
| labels: | | |
| org.opencontainers.image.title=${{ github.event.repository.name }}-msp-backend | |
| org.opencontainers.image.description=Storage Hub MSP Backend | |
| org.opencontainers.image.url=${{ github.event.repository.html_url }} | |
| org.opencontainers.image.source=${{ github.event.repository.clone_url }} | |
| org.opencontainers.image.revision=${{ github.sha }} | |
| org.opencontainers.image.licenses=${{ github.event.repository.license.spdx_id }} | |
| # zombie_test_k8: | |
| # needs: [build_node, setup] | |
| # name: "Test Simple Network" | |
| # runs-on: blacksmith-16vcpu-ubuntu-2404 | |
| # env: | |
| # SH_IMAGE: ${{ needs.setup.outputs.image-tag }} | |
| # defaults: | |
| # run: | |
| # working-directory: test | |
| # steps: | |
| # - uses: actions/checkout@v4 | |
| # - uses: ./.github/workflow-templates/setup-pnpm | |
| # - name: Pull Docker image | |
| # run: docker pull moonsonglabs/storage-hub:${{ needs.setup.outputs.image-tag }} | |
| # - name: Start local k8 cluster | |
| # uses: medyagh/setup-minikube@latest | |
| # with: | |
| # cache: true | |
| # driver: docker | |
| # cpus: 4 | |
| # memory: 12000 | |
| # - name: Run Zombienet Test! | |
| # run: | | |
| # pnpm install | |
| # pnpm zombienet test configs/simple.zndsl | |
| network_test: | |
| needs: [build_node, setup] | |
| name: "Network Test: Zombienet (Parachain + Relay)" | |
| runs-on: blacksmith-4vcpu-ubuntu-2404 | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/workflow-templates/setup-pnpm | |
| - run: mkdir -p target/release/ | |
| - name: Get node binary from artefacts if changed | |
| if: needs.setup.outputs.node_changed == 'true' || needs.setup.outputs.force_node == 'true' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: node | |
| path: target/release/ | |
| - name: Pull latest node Docker image and extract binary if not changed | |
| if: needs.setup.outputs.node_changed == 'false' && needs.setup.outputs.force_node == 'false' | |
| run: | | |
| docker pull moonsonglabs/storage-hub:latest | |
| docker create --name temp_storage_hub moonsonglabs/storage-hub:latest | |
| docker cp temp_storage_hub:/usr/local/bin/storage-hub-node target/release/storage-hub-node | |
| docker rm temp_storage_hub | |
| - name: Start the zombienet full network | |
| run: | | |
| chmod +x target/release/storage-hub-node | |
| chmod -R 777 docker/dev-keystores | |
| cd test | |
| pnpm install | |
| pnpm zombie:run:full:native & | |
| sleep 60 | |
| - name: Setup zombienet | |
| run: | | |
| cd test | |
| pnpm zombie:setup:native | |
| - name: Run zombienet tests | |
| run: | | |
| cd test | |
| pnpm test:full | |
| zombie_test: | |
| needs: [build_node, setup] | |
| name: "Network Test: Zombienet CLI" | |
| timeout-minutes: 30 | |
| runs-on: blacksmith-4vcpu-ubuntu-2404 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/workflow-templates/setup-pnpm | |
| - run: mkdir -p target/release/ | |
| - name: Get node binary from artefacts if changed | |
| if: needs.setup.outputs.node_changed == 'true' || needs.setup.outputs.force_node == 'true' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: node | |
| path: target/release/ | |
| - name: Pull latest node Docker image and extract binary if not changed | |
| if: needs.setup.outputs.node_changed == 'false' && needs.setup.outputs.force_node == 'false' | |
| run: | | |
| docker pull moonsonglabs/storage-hub:latest | |
| docker create --name temp_storage_hub moonsonglabs/storage-hub:latest | |
| docker cp temp_storage_hub:/usr/local/bin/storage-hub-node target/release/storage-hub-node | |
| docker rm temp_storage_hub | |
| - name: Setup zombienet | |
| run: | | |
| chmod +x target/release/storage-hub-node | |
| chmod -R 777 docker/dev-keystores | |
| cd test | |
| pnpm install | |
| pnpm tsx scripts/downloadPolkadot.ts stable2412 | |
| - name: Run zombienet tests using its CLI | |
| run: | | |
| cd test | |
| pnpm zombienet test --provider native configs/simpleNative.zndsl | |
| dev_node_test: | |
| needs: [build_node, setup] | |
| timeout-minutes: 30 | |
| name: "Node Tests: Solo Dev Node" | |
| runs-on: blacksmith-4vcpu-ubuntu-2404 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard: [1, 2] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/workflow-templates/setup-pnpm | |
| - run: mkdir -p target/release/ | |
| - name: Get node binary from artefacts if changed | |
| if: needs.setup.outputs.node_changed == 'true' || needs.setup.outputs.force_node == 'true' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: node | |
| path: target/release/ | |
| - name: Build local node Docker image | |
| if: needs.setup.outputs.node_changed == 'true' || needs.setup.outputs.force_node == 'true' | |
| run: | | |
| chmod +x target/release/storage-hub-node | |
| chmod -R 777 docker/dev-keystores | |
| cd test | |
| pnpm install | |
| pnpm docker:build | |
| - name: Pull latest node Docker image if not changed | |
| if: needs.setup.outputs.node_changed == 'false' && needs.setup.outputs.force_node == 'false' | |
| run: | | |
| docker pull moonsonglabs/storage-hub:latest | |
| docker tag moonsonglabs/storage-hub:latest storage-hub:local | |
| - name: Run typegen | |
| run: | | |
| cd test | |
| pnpm typegen | |
| - name: Run Dev Node tests | |
| run: | | |
| cd test | |
| node --import tsx \ | |
| --test \ | |
| --test-reporter=@reporters/github \ | |
| --test-reporter-destination=stdout \ | |
| --test-reporter=spec \ | |
| --test-reporter-destination=stdout \ | |
| --test-concurrency=1 \ | |
| --test-shard=${{ matrix.shard }}/2 \ | |
| ./suites/solo-node/**/**.test.ts | |
| full_net: | |
| needs: [build_node, setup] | |
| name: "Integration Tests: FullNet" | |
| runs-on: blacksmith-4vcpu-ubuntu-2404 | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard: [1, 2, 3, 4] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/workflow-templates/setup-pnpm | |
| with: | |
| node_version: 23 | |
| - run: mkdir -p target/release/ | |
| - name: Get node binary from artefacts if changed | |
| if: needs.setup.outputs.node_changed == 'true' || needs.setup.outputs.force_node == 'true' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: node | |
| path: target/release/ | |
| - name: Build local Docker image | |
| if: needs.setup.outputs.node_changed == 'true' || needs.setup.outputs.force_node == 'true' | |
| run: | | |
| chmod +x target/release/storage-hub-node | |
| chmod -R 777 docker/dev-keystores | |
| cd test | |
| pnpm install | |
| pnpm docker:build | |
| - name: Pull latest node Docker image if not changed | |
| if: needs.setup.outputs.node_changed == 'false' && needs.setup.outputs.force_node == 'false' | |
| run: | | |
| docker pull moonsonglabs/storage-hub:latest | |
| docker tag moonsonglabs/storage-hub:latest storage-hub:local | |
| - name: Run typegen | |
| run: | | |
| cd test | |
| pnpm typegen | |
| - uses: taiki-e/install-action@v2 | |
| with: | |
| tool: diesel_cli | |
| - name: Run FullNet tests | |
| id: run_tests | |
| run: | | |
| cd test | |
| pnpm i | |
| node --import tsx \ | |
| --test \ | |
| --test-reporter=@reporters/github \ | |
| --test-reporter-destination=stdout \ | |
| --test-reporter=spec \ | |
| --test-reporter-destination=stdout \ | |
| --test-concurrency=1 \ | |
| --test-shard=${{ matrix.shard }}/4 \ | |
| ./suites/integration/msp/**.test.ts | |
| - name: Collect Docker logs on failure | |
| if: failure() && steps.run_tests.outcome == 'failure' | |
| run: | | |
| echo "::group::Docker Container Status" | |
| docker ps -a | |
| echo "::endgroup::" | |
| mkdir -p test-logs | |
| for container in $(docker ps -a --filter "ancestor=moonsonglabs/storage-hub" --format "{{.Names}}"); do | |
| echo "Collecting logs from $container" | |
| docker logs $container > "test-logs/${container}.log" 2>&1 | |
| done | |
| SP_LOGS_DIR=$(find /tmp -maxdepth 1 -type d -name "bsp-logs-*" -printf '%T@ %p\n' | sort -n | tail -1 | cut -f2- -d" ") | |
| if [ ! -z "$SP_LOGS_DIR" ]; then | |
| echo "Found logs directory: $BSP_LOGS_DIR" | |
| cp -r $SP_LOGS_DIR/* test-logs/ || true | |
| else | |
| echo "No sp-logs directory found in /tmp" | |
| fi | |
| echo "Collected log files:" | |
| ls -la test-logs/ | |
| - name: Upload logs as artifact | |
| if: failure() && steps.run_tests.outcome == 'failure' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: sp-test-logs-shard-${{ matrix.shard }} | |
| path: test-logs/ | |
| retention-days: 5 | |
| if-no-files-found: warn | |
| bsp_net: | |
| needs: [build_node, setup] | |
| name: "Integration Tests: BSPNet" | |
| runs-on: blacksmith-4vcpu-ubuntu-2404 | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard: [1, 2, 3, 4, 5] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/workflow-templates/setup-pnpm | |
| with: | |
| node_version: 23 | |
| - run: mkdir -p target/release/ | |
| - name: Get node binary from artefacts if changed | |
| if: needs.setup.outputs.node_changed == 'true' || needs.setup.outputs.force_node == 'true' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: node | |
| path: target/release/ | |
| - name: Build local Docker image | |
| if: needs.setup.outputs.node_changed == 'true' || needs.setup.outputs.force_node == 'true' | |
| run: | | |
| chmod +x target/release/storage-hub-node | |
| chmod -R 777 docker/dev-keystores | |
| cd test | |
| pnpm install | |
| pnpm docker:build | |
| - name: Pull latest node Docker image if not changed | |
| if: needs.setup.outputs.node_changed == 'false' && needs.setup.outputs.force_node == 'false' | |
| run: | | |
| docker pull moonsonglabs/storage-hub:latest | |
| docker tag moonsonglabs/storage-hub:latest storage-hub:local | |
| - name: Run typegen | |
| run: | | |
| cd test | |
| pnpm typegen | |
| - name: Run BSPNet tests | |
| id: run_tests | |
| run: | | |
| cd test | |
| pnpm i | |
| node --import tsx \ | |
| --test \ | |
| --test-reporter=@reporters/github \ | |
| --test-reporter-destination=stdout \ | |
| --test-reporter=spec \ | |
| --test-reporter-destination=stdout \ | |
| --test-concurrency=1 \ | |
| --test-shard=${{ matrix.shard }}/5 \ | |
| ./suites/integration/bsp/**.test.ts | |
| - name: Collect Docker logs on failure | |
| if: failure() && steps.run_tests.outcome == 'failure' | |
| run: | | |
| echo "::group::Docker Container Status" | |
| docker ps -a | |
| echo "::endgroup::" | |
| mkdir -p test-logs | |
| for container in $(docker ps -a --filter "ancestor=moonsonglabs/storage-hub" --format "{{.Names}}"); do | |
| echo "Collecting logs from $container" | |
| docker logs $container > "test-logs/${container}.log" 2>&1 | |
| done | |
| BSP_LOGS_DIR=$(find /tmp -maxdepth 1 -type d -name "bsp-logs-*" -printf '%T@ %p\n' | sort -n | tail -1 | cut -f2- -d" ") | |
| if [ ! -z "$BSP_LOGS_DIR" ]; then | |
| echo "Found logs directory: $BSP_LOGS_DIR" | |
| cp -r $BSP_LOGS_DIR/* test-logs/ || true | |
| else | |
| echo "No bsp-logs directory found in /tmp" | |
| fi | |
| echo "Collected log files:" | |
| ls -la test-logs/ | |
| - name: Upload logs as artifact | |
| if: failure() && steps.run_tests.outcome == 'failure' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: bsp-test-logs-shard-${{ matrix.shard }} | |
| path: test-logs/ | |
| retention-days: 5 | |
| if-no-files-found: warn | |
| backend_test: | |
| needs: [build_node, build_backend, setup] | |
| name: "Integration Tests: Backend" | |
| runs-on: blacksmith-4vcpu-ubuntu-2404 | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard: [1] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/workflow-templates/setup-pnpm | |
| with: | |
| node_version: 23 | |
| - uses: actions-rust-lang/[email protected] | |
| - name: Install libpq-dev | |
| run: sudo apt-get update && sudo apt-get install -y libpq-dev | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - run: mkdir -p target/release/ | |
| - name: Get backend binary from artefacts if changed | |
| if: needs.setup.outputs.backend_changed == 'true' || needs.setup.outputs.force_backend == 'true' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: backend | |
| path: target/release/ | |
| - name: Build local backend Docker image | |
| if: needs.setup.outputs.backend_changed == 'true' || needs.setup.outputs.force_backend == 'true' | |
| run: | | |
| chmod +x target/release/sh-msp-backend | |
| chmod -R 777 docker/dev-keystores | |
| cd test | |
| pnpm i | |
| pnpm docker:build:backend | |
| - name: Pull latest backend Docker image if not changed | |
| if: needs.setup.outputs.backend_changed == 'false' && needs.setup.outputs.force_backend == 'false' | |
| run: | | |
| docker pull moonsonglabs/storage-hub-msp-backend:latest | |
| docker tag moonsonglabs/storage-hub-msp-backend:latest sh-msp-backend:local | |
| - name: Get node binary from artefacts if changed | |
| if: needs.setup.outputs.node_changed == 'true' || needs.setup.outputs.force_node == 'true' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: node | |
| path: target/release/ | |
| - name: Build local node Docker image | |
| if: needs.setup.outputs.node_changed == 'true' || needs.setup.outputs.force_node == 'true' | |
| run: | | |
| chmod +x target/release/storage-hub-node | |
| chmod -R 777 docker/dev-keystores | |
| cd test | |
| pnpm install | |
| pnpm docker:build | |
| - name: Pull latest node Docker image if not changed | |
| if: needs.setup.outputs.node_changed == 'false' && needs.setup.outputs.force_node == 'false' | |
| run: | | |
| docker pull moonsonglabs/storage-hub:latest | |
| docker tag moonsonglabs/storage-hub:latest storage-hub:local | |
| - uses: taiki-e/install-action@v2 | |
| with: | |
| tool: diesel_cli | |
| - name: Run backend tests | |
| id: run_tests | |
| run: | | |
| cd test | |
| pnpm i | |
| node --import tsx \ | |
| --test \ | |
| --test-reporter=@reporters/github \ | |
| --test-reporter-destination=stdout \ | |
| --test-reporter=spec \ | |
| --test-reporter-destination=stdout \ | |
| --test-concurrency=1 \ | |
| --test-shard=${{ matrix.shard }}/1 \ | |
| ./suites/integration/backend/**.test.ts | |
| - name: Collect Docker Logs on Failure | |
| if: failure() && steps.run_tests.outcome == 'failure' | |
| run: | | |
| echo "::group::Docker Container Status" | |
| docker ps -a | |
| echo "::endgroup::" | |
| mkdir -p test-logs | |
| # Collect logs from Storage Hub images | |
| for container in $(docker ps -a --filter "ancestor=moonsonglabs/storage-hub" --format "{{.Names}}"); do | |
| echo "Collecting logs from $container" | |
| docker logs $container > "test-logs/${container}.log" 2>&1 | |
| done | |
| # Collect logs from Backend image | |
| for container in $(docker ps -a --filter "ancestor=sh-msp-backend:local" --format "{{.Names}}"); do | |
| echo "Collecting logs from $container" | |
| docker logs $container > "test-logs/${container}.log" 2>&1 | |
| done | |
| SP_LOGS_DIR=$(find /tmp -maxdepth 1 -type d -name "bsp-logs-*" -printf '%T@ %p\n' | sort -n | tail -1 | cut -f2- -d" ") | |
| if [ ! -z "$SP_LOGS_DIR" ]; then | |
| echo "Found logs directory: $SP_LOGS_DIR" | |
| cp -r $SP_LOGS_DIR/* test-logs/ || true | |
| else | |
| echo "No sp-logs directory found in /tmp" | |
| fi | |
| echo "Collected log files:" | |
| ls -la test-logs/ | |
| - name: Upload logs as artifact | |
| if: failure() && steps.run_tests.outcome == 'failure' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: backend-test-logs | |
| path: test-logs/ | |
| retention-days: 5 | |
| if-no-files-found: warn | |
| user_net: | |
| needs: [build_node, setup] | |
| name: "Integration Tests: User" | |
| runs-on: blacksmith-4vcpu-ubuntu-2404 | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard: [1, 2] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/workflow-templates/setup-pnpm | |
| with: | |
| node_version: 23 | |
| - run: mkdir -p target/release/ | |
| - name: Get node binary from artefacts if changed | |
| if: needs.setup.outputs.node_changed == 'true' || needs.setup.outputs.force_node == 'true' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: node | |
| path: target/release/ | |
| - name: Build local node Docker image | |
| if: needs.setup.outputs.node_changed == 'true' || needs.setup.outputs.force_node == 'true' | |
| run: | | |
| chmod +x target/release/storage-hub-node | |
| chmod -R 777 docker/dev-keystores | |
| cd test | |
| pnpm install | |
| pnpm docker:build | |
| - name: Pull latest node Docker image if not changed | |
| if: needs.setup.outputs.node_changed == 'false' && needs.setup.outputs.force_node == 'false' | |
| run: | | |
| docker pull moonsonglabs/storage-hub:latest | |
| docker tag moonsonglabs/storage-hub:latest storage-hub:local | |
| - name: Run typegen | |
| run: | | |
| cd test | |
| pnpm typegen | |
| - uses: taiki-e/install-action@v2 | |
| with: | |
| tool: diesel_cli | |
| - name: Run user tests | |
| id: run_tests | |
| run: | | |
| cd test | |
| pnpm i | |
| node --import tsx \ | |
| --test \ | |
| --test-reporter=@reporters/github \ | |
| --test-reporter-destination=stdout \ | |
| --test-reporter=spec \ | |
| --test-reporter-destination=stdout \ | |
| --test-concurrency=1 \ | |
| --test-shard=${{ matrix.shard }}/2 \ | |
| ./suites/integration/user/**.test.ts | |
| - name: Collect Docker logs on failure | |
| if: failure() && steps.run_tests.outcome == 'failure' | |
| run: | | |
| echo "::group::Docker Container Status" | |
| docker ps -a | |
| echo "::endgroup::" | |
| mkdir -p test-logs | |
| for container in $(docker ps -a --filter "ancestor=moonsonglabs/storage-hub" --format "{{.Names}}"); do | |
| echo "Collecting logs from $container" | |
| docker logs $container > "test-logs/${container}.log" 2>&1 | |
| done | |
| SP_LOGS_DIR=$(find /tmp -maxdepth 1 -type d -name "bsp-logs-*" -printf '%T@ %p\n' | sort -n | tail -1 | cut -f2- -d" ") | |
| if [ ! -z "$SP_LOGS_DIR" ]; then | |
| echo "Found logs directory: $SP_LOGS_DIR" | |
| cp -r $SP_LOGS_DIR/* test-logs/ || true | |
| else | |
| echo "No bsp-logs directory found in /tmp" | |
| fi | |
| echo "Collected log files:" | |
| ls -la test-logs/ | |
| - name: Upload logs as artifact | |
| if: failure() && steps.run_tests.outcome == 'failure' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: user-test-logs-shard-${{ matrix.shard }} | |
| path: test-logs/ | |
| retention-days: 5 | |
| if-no-files-found: warn | |
| fisherman_test: | |
| needs: [build_node, setup] | |
| name: "Integration Tests: Fisherman" | |
| runs-on: blacksmith-4vcpu-ubuntu-2404 | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/workflow-templates/setup-pnpm | |
| with: | |
| node_version: 23 | |
| - run: mkdir -p target/release/ | |
| - name: Get node binary from artefacts if changed | |
| if: needs.setup.outputs.node_changed == 'true' || needs.setup.outputs.force_node == 'true' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: node | |
| path: target/release/ | |
| - name: Build local node Docker image | |
| if: needs.setup.outputs.node_changed == 'true' || needs.setup.outputs.force_node == 'true' | |
| run: | | |
| chmod +x target/release/storage-hub-node | |
| chmod -R 777 docker/dev-keystores | |
| cd test | |
| pnpm install | |
| pnpm docker:build | |
| - name: Pull latest node Docker image if not changed | |
| if: needs.setup.outputs.node_changed == 'false' && needs.setup.outputs.force_node == 'false' | |
| run: | | |
| docker pull moonsonglabs/storage-hub:latest | |
| docker tag moonsonglabs/storage-hub:latest storage-hub:local | |
| - name: Run typegen | |
| run: | | |
| cd test | |
| pnpm typegen | |
| - uses: taiki-e/install-action@v2 | |
| with: | |
| tool: diesel_cli | |
| - name: Run Fisherman tests | |
| id: run_tests | |
| run: | | |
| cd test | |
| pnpm i | |
| node --import tsx \ | |
| --test \ | |
| --test-reporter=@reporters/github \ | |
| --test-reporter-destination=stdout \ | |
| --test-reporter=spec \ | |
| --test-reporter-destination=stdout \ | |
| --test-concurrency=1 \ | |
| ./suites/integration/fisherman/**.test.ts | |
| - name: Collect Docker logs on failure | |
| if: failure() && steps.run_tests.outcome == 'failure' | |
| run: | | |
| echo "::group::Docker Container Status" | |
| docker ps -a | |
| echo "::endgroup::" | |
| mkdir -p test-logs | |
| for container in $(docker ps -a --filter "ancestor=moonsonglabs/storage-hub" --format "{{.Names}}"); do | |
| echo "Collecting logs from $container" | |
| docker logs $container > "test-logs/${container}.log" 2>&1 | |
| done | |
| SP_LOGS_DIR=$(find /tmp -maxdepth 1 -type d -name "bsp-logs-*" -printf '%T@ %p\n' | sort -n | tail -1 | cut -f2- -d" ") | |
| if [ ! -z "$SP_LOGS_DIR" ]; then | |
| echo "Found logs directory: $SP_LOGS_DIR" | |
| cp -r $SP_LOGS_DIR/* test-logs/ || true | |
| else | |
| echo "No sp-logs directory found in /tmp" | |
| fi | |
| echo "Collected log files:" | |
| ls -la test-logs/ | |
| - name: Upload logs as artifact | |
| if: failure() && steps.run_tests.outcome == 'failure' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: fisherman-test-logs | |
| path: test-logs/ | |
| retention-days: 5 | |
| if-no-files-found: warn | |
| solochain_evm: | |
| needs: [build_node, build_backend, setup] | |
| name: "Integration Tests: Solochain EVM" | |
| runs-on: blacksmith-4vcpu-ubuntu-2404 | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/workflow-templates/setup-pnpm | |
| with: | |
| node_version: 23 | |
| - uses: actions-rust-lang/[email protected] | |
| - name: Install libpq-dev | |
| run: sudo apt-get update && sudo apt-get install -y libpq-dev | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - run: mkdir -p target/release/ | |
| - name: Get backend binary from artefacts if changed | |
| if: needs.setup.outputs.backend_changed == 'true' || needs.setup.outputs.force_backend == 'true' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: backend | |
| path: target/release/ | |
| - name: Build local backend Docker image | |
| if: needs.setup.outputs.backend_changed == 'true' || needs.setup.outputs.force_backend == 'true' | |
| run: | | |
| chmod +x target/release/sh-msp-backend | |
| chmod -R 777 docker/dev-keystores | |
| cd test | |
| pnpm i | |
| pnpm docker:build:backend | |
| - name: Pull latest backend Docker image if not changed | |
| if: needs.setup.outputs.backend_changed == 'false' && needs.setup.outputs.force_backend == 'false' | |
| run: | | |
| docker pull moonsonglabs/storage-hub-msp-backend:latest | |
| docker tag moonsonglabs/storage-hub-msp-backend:latest sh-msp-backend:local | |
| - name: Get node binary from artefacts if changed | |
| if: needs.setup.outputs.node_changed == 'true' || needs.setup.outputs.force_node == 'true' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: node | |
| path: target/release/ | |
| - name: Build local node Docker image | |
| if: needs.setup.outputs.node_changed == 'true' || needs.setup.outputs.force_node == 'true' | |
| run: | | |
| chmod +x target/release/storage-hub-node | |
| chmod -R 777 docker/dev-keystores | |
| cd test | |
| pnpm install | |
| pnpm docker:build | |
| - name: Pull latest node Docker image if not changed | |
| if: needs.setup.outputs.node_changed == 'false' && needs.setup.outputs.force_node == 'false' | |
| run: | | |
| docker pull moonsonglabs/storage-hub:latest | |
| docker tag moonsonglabs/storage-hub:latest storage-hub:local | |
| - name: Run typegen | |
| run: | | |
| cd test | |
| pnpm typegen | |
| - name: Setup Rust toolchain (for SDK wasm build) | |
| uses: actions-rust-lang/[email protected] | |
| with: | |
| target: wasm32-unknown-unknown | |
| cache: false | |
| - name: Install wasm-pack (for SDK build) | |
| run: curl -sSf https://rustwasm.github.io/wasm-pack/installer/init.sh | sh | |
| - name: Build SDK | |
| run: | | |
| cd sdk | |
| pnpm install --frozen-lockfile | |
| pnpm build | |
| - uses: taiki-e/install-action@v2 | |
| with: | |
| tool: diesel_cli | |
| - name: Run Solochain EVM tests | |
| id: run_tests | |
| run: | | |
| cd test | |
| pnpm i | |
| node --import tsx \ | |
| --test \ | |
| --test-reporter=@reporters/github \ | |
| --test-reporter-destination=stdout \ | |
| --test-reporter=spec \ | |
| --test-reporter-destination=stdout \ | |
| --test-concurrency=1 \ | |
| ./suites/integration/solochain-evm/**.test.ts | |
| - name: Collect Docker logs on failure | |
| if: failure() && steps.run_tests.outcome == 'failure' | |
| run: | | |
| echo "::group::Docker Container Status" | |
| docker ps -a | |
| echo "::endgroup::" | |
| mkdir -p test-logs | |
| for container in $(docker ps -a --filter "ancestor=moonsonglabs/storage-hub" --format "{{.Names}}"); do | |
| echo "Collecting logs from $container" | |
| docker logs $container > "test-logs/${container}.log" 2>&1 | |
| done | |
| # Collect logs from Backend image | |
| for container in $(docker ps -a --filter "ancestor=sh-msp-backend:local" --format "{{.Names}}"); do | |
| echo "Collecting logs from $container" | |
| docker logs $container > "test-logs/${container}.log" 2>&1 | |
| done | |
| SP_LOGS_DIR=$(find /tmp -maxdepth 1 -type d -name "bsp-logs-*" -printf '%T@ %p\n' | sort -n | tail -1 | cut -f2- -d" ") | |
| if [ ! -z "$SP_LOGS_DIR" ]; then | |
| echo "Found logs directory: $SP_LOGS_DIR" | |
| cp -r $SP_LOGS_DIR/* test-logs/ || true | |
| else | |
| echo "No sp-logs directory found in /tmp" | |
| fi | |
| echo "Collected log files:" | |
| ls -la test-logs/ | |
| - name: Upload logs as artifact | |
| if: failure() && steps.run_tests.outcome == 'failure' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: solochain-evm-test-logs | |
| path: test-logs/ | |
| retention-days: 5 | |
| if-no-files-found: warn | |
| typegen_check: | |
| needs: [build_node, setup] | |
| name: "Check Rust/TS bindings are up to date" | |
| runs-on: blacksmith-4vcpu-ubuntu-2404 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/workflow-templates/setup-pnpm | |
| with: | |
| node_version: 22 | |
| - run: mkdir -p target/release/ | |
| - name: Get node binary from artefacts if changed | |
| if: needs.setup.outputs.node_changed == 'true' || needs.setup.outputs.force_node == 'true' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: node | |
| path: target/release/ | |
| - name: Build local node Docker image | |
| if: needs.setup.outputs.node_changed == 'true' || needs.setup.outputs.force_node == 'true' | |
| run: | | |
| chmod +x target/release/storage-hub-node | |
| chmod -R 777 docker/dev-keystores | |
| cd test | |
| pnpm install | |
| pnpm docker:build | |
| - name: Pull latest node Docker image if not changed | |
| if: needs.setup.outputs.node_changed == 'false' && needs.setup.outputs.force_node == 'false' | |
| run: | | |
| docker pull moonsonglabs/storage-hub:latest | |
| docker tag moonsonglabs/storage-hub:latest storage-hub:local | |
| - name: Run typegen | |
| run: | | |
| cd test | |
| pnpm typegen | |
| - name: Check for changes | |
| run: | | |
| cd api-augment | |
| if [ -n "$(git status --porcelain .)" ]; then | |
| echo "Typegen produced changes. Please run 'pnpm typegen' locally and commit the changes." | |
| exit 1 | |
| else | |
| echo "No changes" | |
| exit 0 | |
| fi |