Ingester: Set forcedCompactionInProgress for TSDBs during Compactions Triggered by Idleness
#43527
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - r[0-9]+ # Trigger builds after a push to weekly branches | |
| tags: | |
| # The following regex matches the Mimir release tag. Tag filters not as strict due to different regex system on Github Actions. | |
| - mimir-[0-9]+.[0-9]+.[0-9]+** | |
| pull_request: | |
| permissions: | |
| contents: read | |
| concurrency: | |
| # Cancel any running workflow for the same branch when new commits are pushed. | |
| # We group both by ref_name (available when CI is triggered by a push to a branch/tag) | |
| # and head_ref (available when CI is triggered by a PR). | |
| group: "${{ github.ref_name }}-${{ github.head_ref }}" | |
| cancel-in-progress: true | |
| jobs: | |
| prepare: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Get build image from Makefile | |
| id: build_image_step | |
| run: echo "build_image=$(make print-build-image)" >> "$GITHUB_OUTPUT" | |
| outputs: | |
| build_image: ${{ steps.build_image_step.outputs.build_image }} | |
| # Determine if we will deploy (aka push) the image to the registry. | |
| is_deploy: ${{ (startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/r')) && github.event_name == 'push' && github.repository == 'grafana/mimir' }} | |
| goversion: | |
| runs-on: ubuntu-latest | |
| needs: prepare | |
| container: | |
| image: ${{ needs.prepare.outputs.build_image }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Run Git Config | |
| run: git config --global --add safe.directory '*' | |
| - name: Get Go Version | |
| id: go-version | |
| run: | | |
| echo "version=$(make BUILD_IN_CONTAINER=false print-go-version)" >> "$GITHUB_OUTPUT" | |
| outputs: | |
| version: ${{ steps.go-version.outputs.version }} | |
| lint: | |
| runs-on: ubuntu-latest | |
| needs: prepare | |
| container: | |
| image: ${{ needs.prepare.outputs.build_image }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Run Git Config | |
| run: git config --global --add safe.directory '*' | |
| # Commands in the Makefile are hardcoded with an assumed file structure of the CI container | |
| # Symlink ensures paths specified in previous commands don’t break | |
| - name: Symlink Expected Path to Workspace | |
| run: | | |
| mkdir -p /go/src/github.com/grafana/mimir | |
| ln -s $GITHUB_WORKSPACE/* /go/src/github.com/grafana/mimir | |
| - name: Lint | |
| run: make BUILD_IN_CONTAINER=false lint | |
| - name: Check Vendor Directory | |
| run: make BUILD_IN_CONTAINER=false mod-check | |
| - name: Check Protos | |
| run: make BUILD_IN_CONTAINER=false check-protos | |
| - name: Check Generated Documentation | |
| run: make BUILD_IN_CONTAINER=false check-doc | |
| - name: Check White Noise | |
| run: make BUILD_IN_CONTAINER=false check-white-noise | |
| - name: Check License Header | |
| run: make BUILD_IN_CONTAINER=false check-license | |
| - name: Check Docker-Compose YAML | |
| run: make BUILD_IN_CONTAINER=false check-mimir-microservices-mode-docker-compose-yaml check-mimir-read-write-mode-docker-compose-yaml | |
| - name: Check Generated OTLP Code | |
| run: make BUILD_IN_CONTAINER=false check-generated-otlp-code | |
| lint-jsonnet: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - prepare | |
| container: | |
| image: ${{ needs.prepare.outputs.build_image }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Run Git Config | |
| run: git config --global --add safe.directory '*' | |
| # Commands in the Makefile are hardcoded with an assumed file structure of the CI container | |
| # Symlink ensures paths specified in previous commands don’t break | |
| - name: Symlink Expected Path to Workspace | |
| run: | | |
| mkdir -p /go/src/github.com/grafana/mimir | |
| ln -s $GITHUB_WORKSPACE/* /go/src/github.com/grafana/mimir | |
| - name: Check Mixin | |
| run: make BUILD_IN_CONTAINER=false check-mixin | |
| - name: Check Mixin Tests | |
| run: make BUILD_IN_CONTAINER=false check-mixin-tests | |
| - name: Check Mixin with Mimirtool rules check | |
| run: make BUILD_IN_CONTAINER=false check-mixin-mimirtool-rules | |
| - name: Check Jsonnet Manifests | |
| run: make BUILD_IN_CONTAINER=false check-jsonnet-manifests | |
| - name: Check Jsonnet Getting Started | |
| run: make BUILD_IN_CONTAINER=false check-jsonnet-getting-started | |
| - name: Check Jsonnet Tests | |
| run: make BUILD_IN_CONTAINER=false check-jsonnet-tests | |
| lint-helm: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - prepare | |
| container: | |
| image: ${{ needs.prepare.outputs.build_image }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Run Git Config | |
| run: git config --global --add safe.directory '*' | |
| # Commands in the Makefile are hardcoded with an assumed file structure of the CI container | |
| # Symlink ensures paths specified in previous commands don’t break | |
| - name: Symlink Expected Path to Workspace | |
| run: | | |
| mkdir -p /go/src/github.com/grafana/mimir | |
| ln -s $GITHUB_WORKSPACE/* /go/src/github.com/grafana/mimir | |
| - name: Set up Helm | |
| uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.0 | |
| with: | |
| version: v3.17.2 | |
| - name: Check Helm Tests | |
| run: make BUILD_IN_CONTAINER=false check-helm-tests | |
| test: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| # Do not abort other groups when one fails. | |
| fail-fast: false | |
| # Split tests into 4 groups. | |
| matrix: | |
| test_group_id: [0, 1, 2, 3] | |
| test_group_total: [4] | |
| needs: | |
| - prepare | |
| container: | |
| image: ${{ needs.prepare.outputs.build_image }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Run Git Config | |
| run: git config --global --add safe.directory '*' | |
| - name: Symlink Expected Path to Workspace | |
| run: | | |
| mkdir -p /go/src/github.com/grafana/mimir | |
| ln -s $GITHUB_WORKSPACE/* /go/src/github.com/grafana/mimir | |
| # We use retry logic for tests to handle flaky tests that might fail intermittently. | |
| # The retry action will run the test command up to max_attempts times if it fails. | |
| # This helps reduce CI failures caused by temporary issues, timing problems, or race conditions. | |
| - name: Run Tests with retry | |
| uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 | |
| with: | |
| timeout_minutes: 180 # 3 hours so that we rely on the timeout in the makefile | |
| max_attempts: 2 | |
| retry_wait_seconds: 0 | |
| command: | | |
| echo "Running unit tests (group ${{ matrix.test_group_id }} of ${{ matrix.test_group_total }}) with Go version: $(go version)" | |
| ./.github/workflows/scripts/run-unit-tests-group.sh --index ${{ matrix.test_group_id }} --total ${{ matrix.test_group_total }} | |
| test-docs: | |
| uses: ./.github/workflows/test-docs.yml | |
| build: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - prepare | |
| container: | |
| image: ${{ needs.prepare.outputs.build_image }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Run Git Config | |
| run: git config --global --add safe.directory '*' | |
| - name: Install Docker Client | |
| run: ./.github/workflows/scripts/install-docker.sh | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 | |
| - name: Set up Docker Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 | |
| - name: Symlink Expected Path to Workspace | |
| run: | | |
| mkdir -p /go/src/github.com/grafana/mimir | |
| ln -s $GITHUB_WORKSPACE/* /go/src/github.com/grafana/mimir | |
| - name: Build Multiarch Docker Images Locally | |
| # Ignore mimir-build-image and mimir-rules-action. | |
| run: | | |
| ./.github/workflows/scripts/build-images.sh /tmp/images $(make list-image-targets | grep -v -E '/mimir-build-image/|/mimir-rules-action/') | |
| - name: Build Archive With Docker Images | |
| run: | | |
| tar cvf images.tar /tmp/images | |
| - name: Upload Archive with Docker Images | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: Docker Images | |
| path: ./images.tar | |
| - name: Build Mimir with race-detector | |
| run: | | |
| make BUILD_IN_CONTAINER=false cmd/mimir/.uptodate_race | |
| export IMAGE_TAG_RACE=$(make image-tag-race) | |
| export MIMIR_DISTROLESS_IMAGE="grafana/mimir:$IMAGE_TAG_RACE" | |
| docker save $MIMIR_DISTROLESS_IMAGE -o ./mimir_race_image_distroless | |
| - name: Upload archive with race-enabled Mimir | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: Race-enabled Mimir | |
| path: | | |
| ./mimir_race_image_distroless | |
| integration: | |
| needs: [goversion, build, prepare] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| # Do not abort other groups when one fails. | |
| fail-fast: false | |
| # Split tests into 6 groups. | |
| matrix: | |
| test_group_id: [0, 1, 2, 3, 4, 5] | |
| test_group_total: [6] | |
| steps: | |
| - name: Upgrade golang | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: ${{ needs.goversion.outputs.version }} | |
| cache: false # We manage caching ourselves below to maintain consistency with the other jobs that don't use setup-go. | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Run Git Config | |
| run: git config --global --add safe.directory '*' | |
| - name: Symlink Expected Path to Workspace | |
| run: | | |
| sudo mkdir -p /go/src/github.com/grafana/mimir | |
| sudo ln -s $GITHUB_WORKSPACE/* /go/src/github.com/grafana/mimir | |
| - name: Download Archive with Docker Images | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: Docker Images | |
| - name: Extract Docker Images from Archive | |
| run: tar xvf images.tar -C / | |
| - name: Load Mimirtool Image into Docker | |
| run: | | |
| export IMAGE_TAG=$(make image-tag) | |
| # skopeo will by default load system-specific version of the image (linux/amd64). | |
| # note that this doesn't use skopeo version from our build-image, because we don't use build-image when running integration tests. | |
| # that's why we use docker run to run latest version. | |
| docker run -v /tmp/images:/tmp/images -v /var/run/docker.sock:/var/run/docker.sock quay.io/skopeo/stable:v1.15.1 copy oci-archive:/tmp/images/mimirtool.oci "docker-daemon:grafana/mimirtool:$IMAGE_TAG" | |
| - name: Download Archive with Docker Images | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: Race-enabled Mimir | |
| - name: Load race-enabled mimir into Docker | |
| run: | | |
| export IMAGE_TAG_RACE=$(make image-tag-race) | |
| docker load -i ./mimir_race_image_distroless | |
| docker run "grafana/mimir:$IMAGE_TAG_RACE" --version | |
| - name: Preload Images | |
| # We download docker images used by integration tests so that all images are available | |
| # locally and the download time doesn't account in the test execution time, which is subject | |
| # to a timeout | |
| run: go run ./tools/pre-pull-images | xargs -n1 -P4 docker pull | |
| # We use retry logic for integration tests to handle flaky tests that might fail intermittently. | |
| # The retry action will run the test command up to max_attempts times if it fails. | |
| # This helps reduce CI failures caused by temporary issues, network problems, or timing conditions | |
| # that are common in integration tests with multiple services interacting. | |
| - name: Integration Tests with retry | |
| uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 | |
| with: | |
| timeout_minutes: 180 # 3 hours so that we rely on the timeout in the makefile | |
| max_attempts: 2 | |
| retry_wait_seconds: 0 | |
| command: | | |
| export IMAGE_TAG_RACE=$(make image-tag-race) | |
| export MIMIR_IMAGE="grafana/mimir:$IMAGE_TAG_RACE" | |
| export IMAGE_TAG=$(make image-tag) | |
| export MIMIRTOOL_IMAGE="grafana/mimirtool:$IMAGE_TAG" | |
| export MIMIR_CHECKOUT_DIR="/go/src/github.com/grafana/mimir" | |
| echo "Running integration tests with image: $MIMIR_IMAGE (Mimir), $MIMIRTOOL_IMAGE (Mimirtool)" | |
| echo "Running integration tests (group ${{ matrix.test_group_id }} of ${{ matrix.test_group_total }}) with Go version: $(go version)" | |
| ./.github/workflows/scripts/run-integration-tests-group.sh --index ${{ matrix.test_group_id }} --total ${{ matrix.test_group_total }} | |
| deploy: | |
| needs: [prepare, build, test, lint, integration] | |
| # Only deploy images on pushes to the grafana/mimir repo, which either are tag pushes or weekly release branch pushes. | |
| if: needs.prepare.outputs.is_deploy == 'true' | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ${{ needs.prepare.outputs.build_image }} | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Run Git Config | |
| run: git config --global --add safe.directory '*' | |
| - name: Install Docker Client | |
| run: ./.github/workflows/scripts/install-docker.sh | |
| - name: Symlink Expected Path to Workspace | |
| run: | | |
| mkdir -p /go/src/github.com/grafana/mimir | |
| ln -s $GITHUB_WORKSPACE/* /go/src/github.com/grafana/mimir | |
| - name: Download Archive with Docker Images | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: Docker Images | |
| - name: Extract Docker Images from Archive | |
| run: tar xvf images.tar -C / | |
| - name: Login to DockerHub | |
| uses: grafana/shared-workflows/actions/dockerhub-login@13fb504e3bfe323c1188bf244970d94b2d336e86 # v1.0.1 | |
| - name: Push Docker Images | |
| run: | | |
| ./.github/workflows/scripts/push-images.sh /tmp/images grafana/ $(make image-tag) |