Fix broken traces + add minor enhancements (#3794) #1778
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Code coverage | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| - trigger-coverage-workflow | |
| paths: | |
| - quickwit/Cargo.toml | |
| - quickwit/Cargo.lock | |
| - quickwit/quickwit-*/** | |
| env: | |
| AWS_REGION: us-east-1 | |
| AWS_ACCESS_KEY_ID: "placeholder" | |
| AWS_SECRET_ACCESS_KEY: "placeholder" | |
| CARGO_INCREMENTAL: 0 | |
| PUBSUB_EMULATOR_HOST: "localhost:9898" | |
| QW_DISABLE_TELEMETRY: 1 | |
| QW_S3_ENDPOINT: "http://localhost:4566" # Services are exposed as localhost because we are not running coverage in a container. | |
| QW_S3_FORCE_PATH_STYLE_ACCESS: 1 | |
| QW_TEST_DATABASE_URL: postgres://quickwit-dev:quickwit-dev@localhost:5432/quickwit-metastore-dev | |
| jobs: | |
| test: | |
| name: Coverage | |
| runs-on: buildjet-8vcpu-ubuntu-2204 # ubuntu-latest | |
| # Setting a containing will require to fix the QW_S3_ENDPOINT to http://localstack:4566 | |
| services: | |
| localstack: | |
| image: localstack/localstack:latest | |
| ports: | |
| - "4566:4566" | |
| - "4571:4571" | |
| - "8080:8080" | |
| env: | |
| SERVICES: kinesis,s3 | |
| options: >- | |
| --health-cmd "curl -k https://localhost:4566" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| credentials: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} | |
| postgres: | |
| image: postgres:latest | |
| ports: | |
| - "5432:5432" | |
| env: | |
| POSTGRES_USER: quickwit-dev | |
| POSTGRES_PASSWORD: quickwit-dev | |
| POSTGRES_DB: quickwit-metastore-dev | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| credentials: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} | |
| kafka-broker: | |
| image: confluentinc/cp-kafka:7.2.1 | |
| ports: | |
| - "9092:9092" | |
| - "9101:9101" | |
| env: | |
| KAFKA_BROKER_ID: 1 | |
| KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" | |
| KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT | |
| KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-broker:29092,PLAINTEXT_HOST://localhost:9092 | |
| KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 | |
| KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 | |
| KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 | |
| KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 | |
| KAFKA_JMX_PORT: 9101 | |
| KAFKA_JMX_HOSTNAME: localhost | |
| KAFKA_HEAP_OPTS: -Xms256M -Xmx256M | |
| options: >- | |
| --health-cmd "cub kafka-ready -b localhost:9092 1 5" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| credentials: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} | |
| zookeeper: | |
| image: confluentinc/cp-zookeeper:7.2.1 | |
| ports: | |
| - "2181:2181" | |
| env: | |
| KAFKA_HEAP_OPTS: -Xms256M -Xmx256M | |
| ZOOKEEPER_CLIENT_PORT: 2181 | |
| ZOOKEEPER_TICK_TIME: 2000 | |
| options: >- | |
| --health-cmd "cub zk-ready localhost:2181 5" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| credentials: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} | |
| gcp-pubsub-emulator: | |
| image: thekevjames/gcloud-pubsub-emulator:7555256f2c | |
| ports: | |
| - "9898:8681" | |
| env: | |
| PUBSUB_PROJECT1: "quickwit-emulator,emulator_topic:emulator_subscription" | |
| credentials: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install lib libsasl2 | |
| run: | | |
| sudo apt update | |
| sudo apt install libsasl2-dev | |
| sudo apt install libsasl2-2 | |
| - uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.cargo/git | |
| ~/.cargo/registry | |
| key: ${{ runner.os }}-cargo-test-${{ hashFiles('Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo-test-${{ hashFiles('Cargo.lock') }} | |
| ${{ runner.os }}-cargo-test | |
| - name: Install awslocal | |
| run: pip install awscli-local | |
| - name: Prepare LocalStack S3 | |
| run: ./quickwit-cli/tests/prepare_tests.sh | |
| working-directory: ./quickwit | |
| # GitHub Actions does not allow services to be started with a custom command, | |
| # so we are running Azurite as a container manually. | |
| - name: Run Azurite service | |
| run: DOCKER_SERVICES=azurite make docker-compose-up | |
| - name: Run Pulsar service | |
| run: DOCKER_SERVICES=pulsar make docker-compose-up | |
| - name: Install Rust | |
| run: rustup update stable | |
| - uses: taiki-e/install-action@v2 | |
| with: | |
| tool: cargo-llvm-cov,nextest,protoc | |
| # We limit the number of jobs to 4 to avoid OOM errors when linking the binary. | |
| - name: Generate code coverage | |
| run: | | |
| cargo llvm-cov clean --workspace | |
| cargo llvm-cov nextest --no-report --test failpoints --features fail/failpoints --retries 2 | |
| CARGO_BUILD_JOBS=4 cargo llvm-cov nextest --no-report --all-features --retries 2 | |
| cargo llvm-cov report --lcov --output-path lcov.info | |
| working-directory: ./quickwit | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos | |
| files: ./quickwit/lcov.info |