Plugging the shard throughput limit configuration to the ingest (#5193) #2447
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Code coverage | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| - trigger-coverage-workflow | |
| paths: | |
| - quickwit/Cargo.toml | |
| - quickwit/Cargo.lock | |
| - quickwit/quickwit-*/** | |
| env: | |
| AWS_REGION: us-east-1 | |
| AWS_ACCESS_KEY_ID: "placeholder" | |
| AWS_SECRET_ACCESS_KEY: "placeholder" | |
| CARGO_INCREMENTAL: 0 | |
| PUBSUB_EMULATOR_HOST: "localhost:9898" | |
| QW_DISABLE_TELEMETRY: 1 | |
| QW_S3_ENDPOINT: "http://localhost:4566" # Services are exposed as localhost because we are not running coverage in a container. | |
| QW_S3_FORCE_PATH_STYLE_ACCESS: 1 | |
| QW_TEST_DATABASE_URL: postgres://quickwit-dev:quickwit-dev@localhost:5432/quickwit-metastore-dev | |
| RUSTFLAGS: -Dwarnings --cfg tokio_unstable | |
| jobs: | |
| test: | |
| name: Coverage | |
| runs-on: gh-ubuntu-arm64 | |
| timeout-minutes: 40 | |
| # Setting a containing will require to fix the QW_S3_ENDPOINT to http://localstack:4566 | |
| services: | |
| localstack: | |
| image: localstack/localstack:latest | |
| ports: | |
| - "4566:4566" | |
| - "4571:4571" | |
| - "8080:8080" | |
| env: | |
| SERVICES: kinesis,s3 | |
| options: >- | |
| --health-cmd "curl -k https://localhost:4566" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| postgres: | |
| image: postgres:latest | |
| ports: | |
| - "5432:5432" | |
| env: | |
| POSTGRES_USER: quickwit-dev | |
| POSTGRES_PASSWORD: quickwit-dev | |
| POSTGRES_DB: quickwit-metastore-dev | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| kafka-broker: | |
| image: confluentinc/cp-kafka:7.2.1 | |
| ports: | |
| - "9092:9092" | |
| - "9101:9101" | |
| env: | |
| KAFKA_BROKER_ID: 1 | |
| KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" | |
| KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT | |
| KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-broker:29092,PLAINTEXT_HOST://localhost:9092 | |
| KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 | |
| KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 | |
| KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 | |
| KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 | |
| KAFKA_JMX_PORT: 9101 | |
| KAFKA_JMX_HOSTNAME: localhost | |
| KAFKA_HEAP_OPTS: -Xms256M -Xmx256M | |
| options: >- | |
| --health-cmd "cub kafka-ready -b localhost:9092 1 5" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| zookeeper: | |
| image: confluentinc/cp-zookeeper:7.2.1 | |
| ports: | |
| - "2181:2181" | |
| env: | |
| KAFKA_HEAP_OPTS: -Xms256M -Xmx256M | |
| ZOOKEEPER_CLIENT_PORT: 2181 | |
| ZOOKEEPER_TICK_TIME: 2000 | |
| options: >- | |
| --health-cmd "cub zk-ready localhost:2181 5" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| gcp-pubsub-emulator: | |
| image: thekevjames/gcloud-pubsub-emulator:7555256f2c | |
| ports: | |
| - "9898:8681" | |
| env: | |
| PUBSUB_PROJECT1: "quickwit-emulator,emulator_topic:emulator_subscription" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install lib libsasl2 | |
| run: | | |
| sudo apt update | |
| sudo apt install libsasl2-dev | |
| sudo apt install libsasl2-2 | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/git | |
| ~/.cargo/registry | |
| key: ${{ runner.os }}-cargo-test-${{ hashFiles('Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo-test-${{ hashFiles('Cargo.lock') }} | |
| ${{ runner.os }}-cargo-test | |
| - name: Install awslocal | |
| run: pip install awscli-local | |
| - name: Prepare LocalStack S3 | |
| run: ./quickwit-cli/tests/prepare_tests.sh | |
| working-directory: ./quickwit | |
| # GitHub Actions does not allow services to be started with a custom command, | |
| # so we are running Azurite as a container manually. | |
| - name: Run Azurite service | |
| run: DOCKER_SERVICES=azurite make docker-compose-up | |
| # GitHub Actions does not allow services to be started with a custom command, | |
| # so we are running fake gcs server as a container manually. | |
| - name: Run Fake GCS Server service | |
| run: DOCKER_SERVICES=fake-gcs-server make docker-compose-up | |
| - name: Run Pulsar service | |
| run: DOCKER_SERVICES=pulsar make docker-compose-up | |
| - name: Install Rust | |
| run: rustup update stable | |
| - name: Install cargo-llvm-cov, cargo-nextest, and protoc | |
| uses: taiki-e/install-action@v2 | |
| with: | |
| tool: cargo-llvm-cov,nextest,protoc | |
| # We limit the number of jobs to 4 to avoid OOM errors when linking the binary. | |
| - name: Generate code coverage | |
| run: | | |
| cargo llvm-cov clean --workspace | |
| cargo llvm-cov nextest --no-report --test failpoints --features fail/failpoints --retries 4 | |
| # increase stack size for test_all_with_s3_localstack_cli, see quickwit#4963 | |
| RUST_MIN_STACK=67108864 CARGO_BUILD_JOBS=4 cargo llvm-cov nextest --no-report --all-features --retries 4 | |
| cargo llvm-cov report --lcov --output-path lcov.info | |
| working-directory: ./quickwit | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos | |
| files: ./quickwit/lcov.info | |
| on-failure: | |
| if: ${{ github.repository_owner == 'quickwit-oss' && failure() }} | |
| name: On Failure | |
| needs: [test] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Send Message | |
| uses: sarisia/actions-status-discord@v1 | |
| with: | |
| webhook: ${{ secrets.DISCORD_WEBHOOK }} | |
| nodetail: true | |
| color: "#FF0000" | |
| title: "" | |
| description: | | |
| ### ❌ [${{ github.event.pull_request.title }}](${{ github.event.pull_request.html_url }}) | |
| @${{ github.actor }} quickwit coverage CI failed on your PR. | |
| Coverage CI contains tests that are not running in the regular CI because they are too lengthy. | |
| For this reason it is possible for it to break even if the tests were passing on your PR. | |
| This is not a catastrophy, but you are responsible for fixing it! | |
| You can run the full test suite locally with `make test-all`. | |
| Please report in this channel that you are working on it/fixed it/or if it is a flaky test/ | |
| or if you need help. | |
| **[View logs](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})** |