diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index eafe2e7eab7..874d4d0489c 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -2,7 +2,7 @@ name: Daily Core check on: schedule: - - cron: '0 5 * * *' + - cron: '0 4 * * 1,2,3,4,5' jobs: check: diff --git a/.github/workflows/codeql-analysis.old b/.github/workflows/codeql-analysis.yml similarity index 87% rename from .github/workflows/codeql-analysis.old rename to .github/workflows/codeql-analysis.yml index 8d069b7238a..19d0db3dcaf 100644 --- a/.github/workflows/codeql-analysis.old +++ b/.github/workflows/codeql-analysis.yml @@ -6,11 +6,11 @@ name: "CodeQL" on: - push: - branches: [develop] schedule: - cron: '0 5 * * 1' + workflow_dispatch: {} + jobs: analyze: name: Analyze @@ -51,13 +51,23 @@ jobs: # Set up JDK - name: Set up JDK uses: actions/setup-java@v4 + if: ${{ matrix.language == 'java' }} with: distribution: 'temurin' java-version: 21 + - name: Setup gradle + if: ${{ matrix.language == 'java' }} + uses: gradle/actions/setup-gradle@v4 + + - name: Build with Gradle + if: ${{ matrix.language == 'java' }} + run: ./gradlew testClasses -x :ui:installFrontend -x :ui:assembleFrontend + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild + if: ${{ matrix.language != 'java' }} uses: github/codeql-action/autobuild@v3 # ℹī¸ Command-line programs to run using the OS shell. diff --git a/.github/workflows/release-plugins.yml b/.github/workflows/gradle-release-plugins.yml similarity index 88% rename from .github/workflows/release-plugins.yml rename to .github/workflows/gradle-release-plugins.yml index a65e56a0533..59a4934009a 100644 --- a/.github/workflows/release-plugins.yml +++ b/.github/workflows/gradle-release-plugins.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: releaseVersion: - description: 'The release version (e.g., 0.21.0-RC1)' + description: 'The release version (e.g., 0.21.0-rc1)' required: true type: string nextVersion: @@ -59,8 +59,9 @@ jobs: env: GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }} run: | - chmod +x ./release-plugins.sh; - ./release-plugins.sh \ + chmod +x ./dev-tools/release-plugins.sh; + + ./dev-tools/release-plugins.sh \ --release-version=${{github.event.inputs.releaseVersion}} \ --next-version=${{github.event.inputs.nextVersion}} \ --yes \ @@ -71,8 +72,9 @@ jobs: env: GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }} run: | - chmod +x ./release-plugins.sh; - ./release-plugins.sh \ + chmod +x ./dev-tools/release-plugins.sh; + + ./dev-tools/release-plugins.sh \ --release-version=${{github.event.inputs.releaseVersion}} \ --next-version=${{github.event.inputs.nextVersion}} \ --dry-run \ diff --git a/.github/workflows/gradle-release.yml b/.github/workflows/gradle-release.yml new file mode 100644 index 00000000000..bdedb40c178 --- /dev/null +++ b/.github/workflows/gradle-release.yml @@ -0,0 +1,89 @@ +name: Run Gradle Release +run-name: "Releasing Kestra ${{ github.event.inputs.releaseVersion }} 🚀" +on: + workflow_dispatch: + inputs: + releaseVersion: + description: 'The release version (e.g., 0.21.0-rc1)' + required: true + type: string + nextVersion: + description: 'The next version (e.g., 0.22.0-SNAPSHOT)' + required: true + type: string +env: + RELEASE_VERSION: "${{ github.event.inputs.releaseVersion }}" + NEXT_VERSION: "${{ github.event.inputs.nextVersion }}" +jobs: + release: + name: Release Kestra + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/develop' + steps: + # Checks + - name: Check Inputs + run: | + if ! [[ "$RELEASE_VERSION" =~ ^[0-9]+(\.[0-9]+)\.0-rc[01](-SNAPSHOT)?$ ]]; then + echo "Invalid release version. Must match regex: ^[0-9]+(\.[0-9]+)\.0-rc[01](-SNAPSHOT)?$" + exit 1 + fi + + if ! [[ "$NEXT_VERSION" =~ ^[0-9]+(\.[0-9]+)\.0-SNAPSHOT$ ]]; then + echo "Invalid next version. Must match regex: ^[0-9]+(\.[0-9]+)\.0-SNAPSHOT$" + exit 1; + fi + # Checkout + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # Checkout GitHub Actions + - uses: actions/checkout@v4 + with: + repository: kestra-io/actions + path: actions + ref: main + + # Setup build + - uses: ./actions/.github/actions/setup-build + id: build + with: + java-enabled: true + node-enabled: true + python-enabled: true + caches-enabled: true + + - name: Configure Git + run: | + git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + + # Execute + - name: Run Gradle Release + env: + GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }} + run: | + # Extract the major and minor versions + BASE_VERSION=$(echo "$RELEASE_VERSION" | sed -E 's/^([0-9]+\.[0-9]+)\..*/\1/') + PUSH_RELEASE_BRANCH="releases/v${BASE_VERSION}.x" + + # Create and push release branch + git checkout -b "$PUSH_RELEASE_BRANCH"; + git push -u origin "$PUSH_RELEASE_BRANCH"; + + # Run gradle release + git checkout develop; + + if [[ "$RELEASE_VERSION" == *"-SNAPSHOT" ]]; then + # -SNAPSHOT qualifier maybe used to test release-candidates + ./gradlew release -Prelease.useAutomaticVersion=true \ + -Prelease.releaseVersion="${RELEASE_VERSION}" \ + -Prelease.newVersion="${NEXT_VERSION}" \ + -Prelease.pushReleaseVersionBranch="${PUSH_RELEASE_BRANCH}" \ + -Prelease.failOnSnapshotDependencies=false + else + ./gradlew release -Prelease.useAutomaticVersion=true \ + -Prelease.releaseVersion="${RELEASE_VERSION}" \ + -Prelease.newVersion="${NEXT_VERSION}" \ + -Prelease.pushReleaseVersionBranch="${PUSH_RELEASE_BRANCH}" + fi \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d7d0bb76d3e..c8ba0f0b800 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -48,12 +48,12 @@ jobs: plugins: ${{ steps.plugins-list.outputs.plugins }} steps: - - name: Checkout current ref + - name: Checkout - Current ref uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Checkout GitHub Actions + - name: Checkout - GitHub Actions uses: actions/checkout@v4 with: repository: kestra-io/actions @@ -61,13 +61,14 @@ jobs: ref: main # Npm - - name: Npm install + - name: Setup - Npm install shell: bash working-directory: ui run: npm ci # Setup build - - uses: ./actions/.github/actions/setup-build + - name: Setup - Build + uses: ./actions/.github/actions/setup-build id: build with: java-enabled: true @@ -75,7 +76,7 @@ jobs: caches-enabled: true # Get Plugins List - - name: Get Plugins List + - name: Plugins - Get List uses: ./.github/actions/plugins-list if: "!startsWith(github.ref, 'refs/tags/v')" id: plugins-list @@ -83,7 +84,7 @@ jobs: plugin-version: ${{ env.PLUGIN_VERSION }} # Set Plugins List - - name: Set Plugin List + - name: Plugins - Set List id: plugins if: "!startsWith(github.ref, 'refs/tags/v')" run: | @@ -95,17 +96,8 @@ jobs: echo "plugins=--repositories=https://s01.oss.sonatype.org/content/repositories/snapshots $PLUGINS" >> $GITHUB_OUTPUT fi - # Build - - name: Build with Gradle - run: | - ./gradlew executableJar - - - name: Copy exe to image - run: | - cp build/executable/* docker/app/kestra && chmod +x docker/app/kestra - # Docker Tag - - name: Set up Vars + - name: Setup - Docker vars id: vars run: | TAG=${GITHUB_REF#refs/*/} @@ -124,15 +116,24 @@ jobs: echo "tag=${TAG}" >> $GITHUB_OUTPUT echo "artifact=docker-kestra-${TAG}" >> $GITHUB_OUTPUT + # Build + - name: Gradle - Build + run: | + ./gradlew executableJar + + - name: Artifacts - Copy exe to image + run: | + cp build/executable/* docker/app/kestra && chmod +x docker/app/kestra + # Docker setup - - name: Set up QEMU + - name: Docker - Setup QEMU uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx + - name: Docker - Setup Buildx uses: docker/setup-buildx-action@v3 # Docker Build - - name: Build & Export Docker Image + - name: Docker - Build & export image uses: docker/build-push-action@v6 if: "!startsWith(github.ref, 'refs/tags/v')" with: @@ -148,66 +149,96 @@ jobs: outputs: type=docker,dest=/tmp/${{ steps.vars.outputs.artifact }}.tar # Upload artifacts - - name: Upload JAR + - name: Artifacts - Upload JAR uses: actions/upload-artifact@v4 with: name: jar path: build/libs/ - - name: Upload Executable + - name: Artifacts - Upload Executable uses: actions/upload-artifact@v4 with: name: exe path: build/executable/ - - name: Upload Docker + - name: Artifacts - Upload Docker uses: actions/upload-artifact@v4 if: "!startsWith(github.ref, 'refs/tags/v')" with: name: ${{ steps.vars.outputs.artifact }} path: /tmp/${{ steps.vars.outputs.artifact }}.tar - check-frontend: name: Run frontend tests runs-on: ubuntu-latest + if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == ''}} + continue-on-error: true steps: - - uses: actions/checkout@v4 + # Setup + - name: Checkout - Current ref + uses: actions/checkout@v4 - - name: Npm install + - name: Npm - Install shell: bash working-directory: ui run: npm ci - - name: Npm lint + - name: Npm - Lint uses: reviewdog/action-eslint@v1 with: github_token: ${{ secrets.GITHUB_TOKEN }} - reporter: github-pr-review # Change reporter. + reporter: github-pr-review workdir: "ui" - - name: Run front-end unit tests + # Unit test + - name: Npm - Run unit tests shell: bash working-directory: ui - run: npm run test:unit + run: npm run test:cicd - - name: Install Playwright + # Build to send bundle stats to codecov + - name: Npm - Run build + shell: bash + working-directory: ui + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + run: npm run build + + # Storybook + - name: Storybook - Install Playwright shell: bash working-directory: ui run: npx playwright install --with-deps - - name: Build Storybook + - name: Storybook - Build shell: bash working-directory: ui run: npm run build-storybook --quiet - - name: Serve Storybook and run tests + - name: Storybook - Run tests shell: bash working-directory: ui run: | npx concurrently -k -s first -n "SB,TEST" -c "magenta,blue" \ "npx http-server storybook-static --port 6006 --silent" \ - "npx wait-on tcp:127.0.0.1:6006 && npm run test-storybook" + "npx wait-on tcp:127.0.0.1:6006 && npm run test:storybook" + + # Codecov + - name: Codecov - Upload coverage reports + uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} + continue-on-error: true + with: + token: ${{ secrets.CODECOV_TOKEN }} + flags: frontend + + - name: Codecov - Upload test results + uses: codecov/test-results-action@v1 + if: ${{ !cancelled() }} + continue-on-error: true + with: + token: ${{ secrets.CODECOV_TOKEN }} + flags: frontend # Run Reusable Workflow from QA repository check-e2e: @@ -227,6 +258,7 @@ jobs: secrets: GITHUB_AUTH_TOKEN: ${{ secrets.GH_PERSONAL_TOKEN }} GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} + check: env: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} @@ -236,19 +268,22 @@ jobs: timeout-minutes: 60 steps: # Checkout - - uses: actions/checkout@v4 + - name: Checkout - Current ref + uses: actions/checkout@v4 with: fetch-depth: 0 # Checkout GitHub Actions - - uses: actions/checkout@v4 + - name: Checkout - GitHub Actions + uses: actions/checkout@v4 with: repository: kestra-io/actions path: actions ref: main # Setup build - - uses: ./actions/.github/actions/setup-build + - name: Setup - Build + uses: ./actions/.github/actions/setup-build id: build with: java-enabled: true @@ -257,12 +292,12 @@ jobs: caches-enabled: true # Services - - name: Build the docker-compose stack + - name: Setup - Start docker compose run: docker compose -f docker-compose-ci.yml up -d if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }} # Gradle check - - name: Build with Gradle + - name: Gradle - Build if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }} env: GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} @@ -271,27 +306,47 @@ jobs: export GOOGLE_APPLICATION_CREDENTIALS=$HOME/.gcp-service-account.json ./gradlew check javadoc --parallel + # Test results + - name: Test - Publish Test Results + uses: dorny/test-reporter@v1 + if: always() + with: + name: Java Tests Report + reporter: java-junit + path: '**/build/test-results/test/TEST-*.xml' + list-suites: 'failed' + list-tests: 'failed' + fail-on-error: 'false' + # Sonar - - name: Analyze with Sonar + - name: Test - Analyze with Sonar if: ${{ env.SONAR_TOKEN != 0 && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} run: ./gradlew sonar --info - # Allure check - - name: Auth to Google Cloud + # GCP + - name: GCP - Auth with unit test account id: auth if: ${{ always() && env.GOOGLE_SERVICE_ACCOUNT != 0 }} + continue-on-error: true uses: "google-github-actions/auth@v2" with: credentials_json: "${{ secrets.GOOGLE_SERVICE_ACCOUNT }}" - - uses: rlespinasse/github-slug-action@v5 + - name: GCP - Setup Cloud SDK + if: ${{ env.GOOGLE_SERVICE_ACCOUNT != 0 && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }} + uses: "google-github-actions/setup-gcloud@v2" + + # Allure check + - name: Allure - Generate slug variables + uses: rlespinasse/github-slug-action@v5 - - name: Publish allure report + - name: Allure - Publish report uses: andrcuns/allure-publish-action@v2.9.0 if: ${{ always() && env.GOOGLE_SERVICE_ACCOUNT != 0 && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }} + continue-on-error: true env: GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} JAVA_HOME: /usr/lib/jvm/default-jvm/ @@ -305,29 +360,30 @@ jobs: ignoreMissingResults: true # Jacoco - - name: "Set up Cloud SDK" - if: ${{ env.GOOGLE_SERVICE_ACCOUNT != 0 && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }} - uses: "google-github-actions/setup-gcloud@v2" - - - name: "Copy jacoco files" + - name: Jacoco - Copy reports if: ${{ env.GOOGLE_SERVICE_ACCOUNT != 0 && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }} + continue-on-error: true run: | mv build/reports/jacoco/testCodeCoverageReport build/reports/jacoco/test/ mv build/reports/jacoco/test/testCodeCoverageReport.xml build/reports/jacoco/test/jacocoTestReport.xml gsutil -m rsync -d -r build/reports/jacoco/test/ gs://internal-kestra-host/${{ format('{0}/{1}', github.repository, 'jacoco') }} - # report test - - name: Test Report - uses: mikepenz/action-junit-report@v5 - if: success() || failure() + # Codecov + - name: Codecov - Upload coverage reports + uses: codecov/codecov-action@v5 + if: ${{ !cancelled() && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }} + continue-on-error: true with: - report_paths: "**/build/test-results/**/TEST-*.xml" + token: ${{ secrets.CODECOV_TOKEN }} + flags: backend - # Codecov - - uses: codecov/codecov-action@v5 - if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }} + - name: Codecov - Upload test results + uses: codecov/test-results-action@v1 + if: ${{ !cancelled() && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }} + continue-on-error: true with: token: ${{ secrets.CODECOV_TOKEN }} + flags: backend release: name: Github Release @@ -343,7 +399,7 @@ jobs: ) steps: # Download Exec - - name: Download executable + - name: Artifacts - Download executable uses: actions/download-artifact@v4 if: startsWith(github.ref, 'refs/tags/v') with: @@ -351,7 +407,7 @@ jobs: path: build/executable # GitHub Release - - name: Create GitHub release + - name: GitHub - Create release id: create_github_release uses: "marvinpinto/action-automatic-releases@latest" if: startsWith(github.ref, 'refs/tags/v') @@ -363,7 +419,7 @@ jobs: build/executable/* # Trigger gha workflow to bump helm chart version - - name: trigger the Helm chart version bump + - name: GitHub - Trigger the Helm chart version bump uses: peter-evans/repository-dispatch@v3 if: steps.create_github_release.conclusion == 'success' with: @@ -401,23 +457,25 @@ jobs: packages: python3 python3-venv python-is-python3 python3-pip nodejs npm curl zip unzip python-libraries: kestra steps: - - uses: actions/checkout@v4 + - name: Checkout - Current ref + uses: actions/checkout@v4 + # Docker setup - - name: Set up QEMU + - name: Docker - Setup QEMU uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx + - name: Docker - Setup Docker Buildx uses: docker/setup-buildx-action@v3 # Docker Login - - name: Login to DockerHub + - name: Docker - Login to DockerHub uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} # Vars - - name: Set image name + - name: Docker - Set image name id: vars run: | TAG=${GITHUB_REF#refs/*/} @@ -428,18 +486,18 @@ jobs: fi # Build Docker Image - - name: Download executable + - name: Artifacts - Download executable uses: actions/download-artifact@v4 with: name: exe path: build/executable - - name: Copy exe to image + - name: Docker - Copy exe to image run: | cp build/executable/* docker/app/kestra && chmod +x docker/app/kestra # Docker Build and push - - name: Build Docker Image + - name: Docker - Build image uses: docker/build-push-action@v6 with: context: . @@ -464,17 +522,20 @@ jobs: (needs.check-e2e.result == 'skipped' || needs.check-e2e.result == 'success') ) steps: - - uses: actions/checkout@v4 + - name: Checkout - Current ref + uses: actions/checkout@v4 # Checkout GitHub Actions - - uses: actions/checkout@v4 + - name: Checkout - GitHub Actions + uses: actions/checkout@v4 with: repository: kestra-io/actions path: actions ref: main # Setup build - - uses: ./actions/.github/actions/setup-build + - name: Setup - Build + uses: ./actions/.github/actions/setup-build id: build with: java-enabled: true @@ -482,7 +543,7 @@ jobs: caches-enabled: true # Publish - - name: Publish package to Sonatype + - name: Publish - Release package to Maven Central if: github.ref == 'refs/heads/develop' env: ORG_GRADLE_PROJECT_sonatypeUsername: ${{ secrets.SONATYPE_USER }} @@ -496,23 +557,13 @@ jobs: echo "signing.password=${SONATYPE_GPG_PASSWORD}" >> ~/.gradle/gradle.properties echo "signing.secretKeyRingFile=${HOME}/.gradle/secring.gpg" >> ~/.gradle/gradle.properties echo ${SONATYPE_GPG_FILE} | base64 -d > ~/.gradle/secring.gpg - ./gradlew publishToSonatype + ./gradlew publishToSonatype ${{ startsWith(github.ref, 'refs/tags/v') && 'closeAndReleaseSonatypeStagingRepository' || '' }} + + # Gradle dependency + - name: Java - Gradle dependency graph + if: ${{ github.ref == 'refs/heads/develop' }} + uses: gradle/actions/dependency-submission@v4 - # Release - - name: Release package to Maven Central - if: startsWith(github.ref, 'refs/tags/v') - env: - ORG_GRADLE_PROJECT_sonatypeUsername: ${{ secrets.SONATYPE_USER }} - ORG_GRADLE_PROJECT_sonatypePassword: ${{ secrets.SONATYPE_PASSWORD }} - SONATYPE_GPG_KEYID: ${{ secrets.SONATYPE_GPG_KEYID }} - SONATYPE_GPG_PASSWORD: ${{ secrets.SONATYPE_GPG_PASSWORD }} - SONATYPE_GPG_FILE: ${{ secrets.SONATYPE_GPG_FILE }} - run: | - echo "signing.keyId=${SONATYPE_GPG_KEYID}" > ~/.gradle/gradle.properties - echo "signing.password=${SONATYPE_GPG_PASSWORD}" >> ~/.gradle/gradle.properties - echo "signing.secretKeyRingFile=${HOME}/.gradle/secring.gpg" >> ~/.gradle/gradle.properties - echo ${SONATYPE_GPG_FILE} | base64 -d > ~/.gradle/secring.gpg - ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository end: runs-on: ubuntu-latest needs: @@ -526,7 +577,7 @@ jobs: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} steps: # Update - - name: Update internal + - name: Github - Update internal uses: benc-uk/workflow-dispatch@v1 if: github.ref == 'refs/heads/develop' && needs.docker.result == 'success' with: @@ -536,7 +587,7 @@ jobs: token: ${{ secrets.GH_PERSONAL_TOKEN }} # Slack - - name: Slack notification + - name: Slack - Notification uses: Gamesight/slack-workflow-status@master if: ${{ always() && env.SLACK_WEBHOOK_URL != 0 }} with: @@ -545,3 +596,4 @@ jobs: name: GitHub Actions icon_emoji: ":github-actions:" channel: "C02DQ1A7JLR" # _int_git channel + diff --git a/.github/workflows/tag-plugins.yml b/.github/workflows/setversion-tag-plugins.yml similarity index 73% rename from .github/workflows/tag-plugins.yml rename to .github/workflows/setversion-tag-plugins.yml index ddd4f7a3a3b..6c751d4fa6b 100644 --- a/.github/workflows/tag-plugins.yml +++ b/.github/workflows/setversion-tag-plugins.yml @@ -21,22 +21,6 @@ jobs: with: fetch-depth: 0 - # Checkout GitHub Actions - - uses: actions/checkout@v4 - with: - repository: kestra-io/actions - path: actions - ref: main - - # Setup build - - uses: ./actions/.github/actions/setup-build - id: build - with: - java-enabled: true - node-enabled: true - python-enabled: true - caches-enabled: true - # Get Plugins List - name: Get Plugins List uses: ./.github/actions/plugins-list @@ -55,8 +39,9 @@ jobs: env: GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }} run: | - chmod +x ./tag-release-plugins.sh; - ./tag-release-plugins.sh \ + chmod +x ./dev-tools/setversion-tag-plugins.sh; + + ./dev-tools/setversion-tag-plugins.sh \ --release-version=${{github.event.inputs.releaseVersion}} \ --yes \ ${{ steps.plugins-list.outputs.repositories }} @@ -66,8 +51,9 @@ jobs: env: GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }} run: | - chmod +x ./tag-release-plugins.sh; - ./tag-release-plugins.sh \ + chmod +x ./dev-tools/setversion-tag-plugins.sh; + + ./dev-tools/setversion-tag-plugins.sh \ --release-version=${{github.event.inputs.releaseVersion}} \ --dry-run \ --yes \ diff --git a/.github/workflows/setversion-tag.yml b/.github/workflows/setversion-tag.yml new file mode 100644 index 00000000000..52e7140e157 --- /dev/null +++ b/.github/workflows/setversion-tag.yml @@ -0,0 +1,58 @@ +name: Set Version and Tag +run-name: "Set version and Tag Kestra to ${{ github.event.inputs.releaseVersion }} 🚀" +on: + workflow_dispatch: + inputs: + releaseVersion: + description: 'The release version (e.g., 0.21.1)' + required: true + type: string +env: + RELEASE_VERSION: "${{ github.event.inputs.releaseVersion }}" +jobs: + release: + name: Release Kestra + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/heads/releases/v') + steps: + # Checks + - name: Check Inputs + run: | + if ! [[ "$RELEASE_VERSION" =~ ^[0-9]+(\.[0-9]+)(\.[0-9]+)(-rc[0-9])?(-SNAPSHOT)?$ ]]; then + echo "Invalid release version. Must match regex: ^[0-9]+(\.[0-9]+)(\.[0-9]+)-(rc[0-9])?(-SNAPSHOT)?$" + exit 1 + fi + + CURRENT_BRANCH="{{ github.ref }}" + + # Extract the major and minor versions + BASE_VERSION=$(echo "$RELEASE_VERSION" | sed -E 's/^([0-9]+\.[0-9]+)\..*/\1/') + RELEASE_BRANCH="refs/heads/releases/v${BASE_VERSION}.x" + + if ! [[ "$CURRENT_BRANCH" == "$RELEASE_BRANCH" ]]; then + echo "Invalid release branch. Expected $RELEASE_BRANCH, was $CURRENT_BRANCH" + exit 1 + fi + + # Checkout + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Configure Git + run: | + git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + + # Execute + - name: Run Gradle Release + env: + GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }} + run: | + # Update version + sed -i "s/^version=.*/version=$RELEASE_VERSION/" ./gradle.properties + git add ./gradle.properties + git commit -m"chore(version): update to version '$RELEASE_VERSION'" + git push + git tag -a "v$RELEASE_VERSION" -m"v$RELEASE_VERSION" + git push origin "v$RELEASE_VERSION" \ No newline at end of file diff --git a/.gitignore b/.gitignore index b2e5d79916e..51af3026c50 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ yarn.lock ui/coverage ui/stats.html ui/.frontend-gradle-plugin +ui/utils/CHANGELOG.md ### Docker /.env diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000000..3507343d323 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,76 @@ +component_management: + individual_components: + - component_id: cli + name: Cli + paths: + - cli/** + - component_id: core + name: Core + paths: + - core/** + - component_id: e2e-tests + name: End to End + paths: + - e2e-tests/** + - component_id: jdbc + name: Jdbc + paths: + - jdbc/** + - component_id: jdbc-h2 + name: Jdbc H2 + paths: + - jdbc-h2/** + - component_id: jdbc-mysql + name: Jdbc Mysql + paths: + - jdbc-mysql/** + - component_id: jdbc-postgres + name: Jdbc Postgres + paths: + - jdbc-postgres/** + - component_id: model + name: Model + paths: + - model/** + - component_id: processor + name: Processor + paths: + - processor/** + - component_id: repository-memory + name: Repository Memory + paths: + - repository-memory/** + - component_id: runner-memory + name: Runner Memory + paths: + - runner-memory/** + - component_id: script + name: Script + paths: + - script/** + - component_id: storage-local + name: Storage Local + paths: + - storage-local/** + - component_id: tests + name: Tests + paths: + - tests/** + - component_id: ui + name: Ui + paths: + - ui/** + - component_id: webserver + name: Webserver + paths: + - webserver/** + +flag_management: + default_rules: + carryforward: true + statuses: + - type: project + target: 80% + threshold: 1% + - type: patch + target: 90% diff --git a/core/src/main/java/io/kestra/core/http/client/configurations/HttpConfiguration.java b/core/src/main/java/io/kestra/core/http/client/configurations/HttpConfiguration.java index a9194b72a07..df768ce87fc 100644 --- a/core/src/main/java/io/kestra/core/http/client/configurations/HttpConfiguration.java +++ b/core/src/main/java/io/kestra/core/http/client/configurations/HttpConfiguration.java @@ -13,9 +13,11 @@ import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.temporal.ChronoUnit; +import lombok.extern.jackson.Jacksonized; @Builder(toBuilder = true) @Getter +@Jacksonized public class HttpConfiguration { @Schema(title = "The timeout configuration.") @PluginProperty diff --git a/core/src/main/java/io/kestra/core/models/QueryFilter.java b/core/src/main/java/io/kestra/core/models/QueryFilter.java new file mode 100644 index 00000000000..91a5abddac9 --- /dev/null +++ b/core/src/main/java/io/kestra/core/models/QueryFilter.java @@ -0,0 +1,250 @@ +package io.kestra.core.models; + +import io.kestra.core.utils.Enums; +import lombok.Builder; + +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Builder +public record QueryFilter( + Field field, + Op operation, + Object value +) { + public enum Op { + EQUALS("$eq"), + NOT_EQUALS("$ne"), + GREATER_THAN("$gte"), + LESS_THAN("$lte"), + IN("$in"), + NOT_IN("$notIn"), + STARTS_WITH("$startsWith"), + ENDS_WITH("$endsWith"), + CONTAINS("$contains"), + REGEX("$regex"); + + private static final Map BY_VALUE = Arrays.stream(values()) + .collect(Collectors.toMap(Op::value, Function.identity())); + + private final String value; + + Op(String value) { + this.value = value; + } + + public static Op fromString(String value) { + return Enums.fromString(value, BY_VALUE, "operation"); + } + + public String value() { + return value; + } + } + + public enum Field { + QUERY("q") { + @Override + public List supportedOp() { + return List.of(Op.EQUALS, Op.NOT_EQUALS, Op.REGEX); + } + }, + SCOPE("scope") { + @Override + public List supportedOp() { + return List.of(Op.EQUALS, Op.NOT_EQUALS); + } + }, + NAMESPACE("namespace") { + @Override + public List supportedOp() { + return List.of(Op.EQUALS, Op.NOT_EQUALS, Op.CONTAINS, Op.STARTS_WITH, Op.ENDS_WITH, Op.REGEX); + } + }, + LABELS("labels") { + @Override + public List supportedOp() { + return List.of(Op.EQUALS, Op.NOT_EQUALS); + } + }, + FLOW_ID("flowId") { + @Override + public List supportedOp() { + return List.of(Op.EQUALS, Op.NOT_EQUALS, Op.CONTAINS, Op.IN, Op.NOT_IN); + } + }, + START_DATE("startDate") { + @Override + public List supportedOp() { + return List.of(Op.GREATER_THAN, Op.LESS_THAN, Op.EQUALS, Op.NOT_EQUALS); + } + }, + END_DATE("endDate") { + @Override + public List supportedOp() { + return List.of(Op.GREATER_THAN, Op.LESS_THAN, Op.EQUALS, Op.NOT_EQUALS); + } + }, + STATE("state") { + @Override + public List supportedOp() { + return List.of(Op.EQUALS, Op.NOT_EQUALS, Op.IN, Op.NOT_IN); + } + }, + TIME_RANGE("timeRange") { + @Override + public List supportedOp() { + return List.of(Op.EQUALS, Op.NOT_EQUALS, Op.CONTAINS, Op.STARTS_WITH, + Op.ENDS_WITH, Op.IN, Op.NOT_IN, Op.REGEX); + } + }, + TRIGGER_EXECUTION_ID("triggerExecutionId") { + @Override + public List supportedOp() { + return List.of(Op.EQUALS, Op.NOT_EQUALS, Op.CONTAINS, Op.STARTS_WITH, Op.ENDS_WITH, Op.IN, Op.NOT_IN); + } + }, + TRIGGER_ID("triggerId") { + @Override + public List supportedOp() { + return List.of(Op.EQUALS, Op.NOT_EQUALS, Op.CONTAINS, Op.STARTS_WITH, Op.ENDS_WITH, Op.IN, Op.NOT_IN); + } + }, + CHILD_FILTER("childFilter") { + @Override + public List supportedOp() { + return List.of(Op.EQUALS, Op.NOT_EQUALS); + } + }, + WORKER_ID("workerId") { + @Override + public List supportedOp() { + return List.of(Op.EQUALS, Op.NOT_EQUALS, Op.CONTAINS, Op.STARTS_WITH, Op.ENDS_WITH, Op.IN, Op.NOT_IN); + } + }, + EXISTING_ONLY("existingOnly") { + @Override + public List supportedOp() { + return List.of(Op.EQUALS, Op.NOT_EQUALS); + } + }, + MIN_LEVEL("level") { + @Override + public List supportedOp() { + return List.of(Op.EQUALS, Op.NOT_EQUALS); + } + }; + + private static final Map BY_VALUE = Arrays.stream(values()) + .collect(Collectors.toMap(Field::value, Function.identity())); + + public abstract List supportedOp(); + + private final String value; + + Field(String value) { + this.value = value; + } + + public static Field fromString(String value) { + return Enums.fromString(value, BY_VALUE, "field"); + } + + public String value() { + return value; + } + } + + + public enum Resource { + FLOW { + @Override + public List supportedField() { + return List.of(Field.LABELS, Field.NAMESPACE, Field.QUERY, Field.SCOPE); + } + }, + NAMESPACE { + @Override + public List supportedField() { + return List.of(Field.EXISTING_ONLY); + } + }, + EXECUTION { + @Override + public List supportedField() { + return List.of( + Field.QUERY, Field.SCOPE, Field.FLOW_ID, Field.START_DATE, Field.END_DATE, Field.TIME_RANGE, + Field.STATE, Field.LABELS, Field.TRIGGER_EXECUTION_ID, Field.CHILD_FILTER, + Field.NAMESPACE + ); + } + }, + LOG { + @Override + public List supportedField() { + return List.of(Field.NAMESPACE, Field.START_DATE, Field.END_DATE, + Field.FLOW_ID, Field.TRIGGER_ID, Field.MIN_LEVEL + ); + } + }, + TASK { + @Override + public List supportedField() { + return List.of(Field.NAMESPACE, Field.QUERY, Field.END_DATE, Field.FLOW_ID, Field.START_DATE, + Field.STATE, Field.LABELS, Field.TRIGGER_EXECUTION_ID, Field.CHILD_FILTER + ); + } + }, + TEMPLATE { + @Override + public List supportedField() { + return List.of(Field.NAMESPACE, Field.QUERY); + } + }, + TRIGGER { + @Override + public List supportedField() { + return List.of(Field.QUERY, Field.NAMESPACE, Field.WORKER_ID, Field.FLOW_ID + ); + } + }; + + public abstract List supportedField(); + + /** + * Converts {@code Resource} enums to a list of {@code ResourceField}, + * including fields and their supported operations. + * + * @return List of {@code ResourceField} with resource names, fields, and operations. + */ + public static List asResourceList() { + return Arrays.stream(values()) + .map(Resource::toResourceField) + .toList(); + } + + private static ResourceField toResourceField(Resource resource) { + List fieldOps = resource.supportedField().stream() + .map(Resource::toFieldInfo) + .toList(); + return new ResourceField(resource.name().toLowerCase(), fieldOps); + } + + private static FieldOp toFieldInfo(Field field) { + List operations = field.supportedOp().stream() + .map(Resource::toOperation) + .toList(); + return new FieldOp(field.name().toLowerCase(), field.value(), operations); + } + + private static Operation toOperation(Op op) { + return new Operation(op.name(), op.value()); + } + } + + public record ResourceField(String name, List fields) {} + public record FieldOp(String name, String value, List operations) {} + public record Operation(String name, String value) {} + +} diff --git a/core/src/main/java/io/kestra/core/repositories/ExecutionRepositoryInterface.java b/core/src/main/java/io/kestra/core/repositories/ExecutionRepositoryInterface.java index 6c121b23e1d..d0483529b37 100644 --- a/core/src/main/java/io/kestra/core/repositories/ExecutionRepositoryInterface.java +++ b/core/src/main/java/io/kestra/core/repositories/ExecutionRepositoryInterface.java @@ -1,5 +1,6 @@ package io.kestra.core.repositories; +import io.kestra.core.models.QueryFilter; import io.kestra.core.models.executions.Execution; import io.kestra.core.models.executions.TaskRun; import io.kestra.core.models.executions.statistics.DailyExecutionStatistics; @@ -59,19 +60,9 @@ default Optional findById(String tenantId, String id) { ArrayListTotal find( Pageable pageable, - @Nullable String query, @Nullable String tenantId, - @Nullable List scope, - @Nullable String namespace, - @Nullable String flowId, - @Nullable ZonedDateTime startDate, - @Nullable ZonedDateTime endDate, - @Nullable List state, - @Nullable Map labels, - @Nullable String triggerExecutionId, - @Nullable ChildFilter childFilter + @Nullable List filters ); - default Flux find( @Nullable String query, @Nullable String tenantId, @@ -103,18 +94,11 @@ Flux find( boolean allowDeleted ); + ArrayListTotal findTaskRun( Pageable pageable, - @Nullable String query, @Nullable String tenantId, - @Nullable String namespace, - @Nullable String flowId, - @Nullable ZonedDateTime startDate, - @Nullable ZonedDateTime endDate, - @Nullable List states, - @Nullable Map labels, - @Nullable String triggerExecutionId, - @Nullable ChildFilter childFilter + List filters ); Execution delete(Execution execution); diff --git a/core/src/main/java/io/kestra/core/repositories/FlowRepositoryInterface.java b/core/src/main/java/io/kestra/core/repositories/FlowRepositoryInterface.java index 89d6b57c953..06e8e53e29c 100644 --- a/core/src/main/java/io/kestra/core/repositories/FlowRepositoryInterface.java +++ b/core/src/main/java/io/kestra/core/repositories/FlowRepositoryInterface.java @@ -1,5 +1,6 @@ package io.kestra.core.repositories; +import io.kestra.core.models.QueryFilter; import io.kestra.core.models.SearchResult; import io.kestra.core.models.executions.Execution; import io.kestra.core.models.flows.Flow; @@ -143,6 +144,12 @@ ArrayListTotal find( @Nullable Map labels ); + ArrayListTotal find( + Pageable pageable, + @Nullable String tenantId, + @Nullable List filters + ); + List findWithSource( @Nullable String query, @Nullable String tenantId, diff --git a/core/src/main/java/io/kestra/core/repositories/LogRepositoryInterface.java b/core/src/main/java/io/kestra/core/repositories/LogRepositoryInterface.java index b4bdd74a4c2..2bb5753d9bd 100644 --- a/core/src/main/java/io/kestra/core/repositories/LogRepositoryInterface.java +++ b/core/src/main/java/io/kestra/core/repositories/LogRepositoryInterface.java @@ -1,5 +1,6 @@ package io.kestra.core.repositories; +import io.kestra.core.models.QueryFilter; import io.kestra.core.models.executions.Execution; import io.kestra.core.models.executions.LogEntry; import io.kestra.core.models.executions.statistics.LogStatistics; @@ -76,15 +77,9 @@ public interface LogRepositoryInterface extends SaveRepositoryInterface find( Pageable pageable, - @Nullable String query, @Nullable String tenantId, - @Nullable String namespace, - @Nullable String flowId, - @Nullable String triggerId, - @Nullable Level minLevel, - @Nullable ZonedDateTime startDate, - @Nullable ZonedDateTime endDate - ); + List filters + ); Flux findAsync( @Nullable String tenantId, diff --git a/core/src/main/java/io/kestra/core/repositories/TriggerRepositoryInterface.java b/core/src/main/java/io/kestra/core/repositories/TriggerRepositoryInterface.java index aade9d39e95..5b25d3ce6a3 100644 --- a/core/src/main/java/io/kestra/core/repositories/TriggerRepositoryInterface.java +++ b/core/src/main/java/io/kestra/core/repositories/TriggerRepositoryInterface.java @@ -1,5 +1,6 @@ package io.kestra.core.repositories; +import io.kestra.core.models.QueryFilter; import io.kestra.core.models.executions.Execution; import io.kestra.core.models.triggers.Trigger; import io.kestra.core.models.triggers.TriggerContext; @@ -29,6 +30,7 @@ public interface TriggerRepositoryInterface { Trigger lock(String triggerUid, Function function); ArrayListTotal find(Pageable from, String query, String tenantId, String namespace, String flowId, String workerId); + ArrayListTotal find(Pageable from, String tenantId, List filters); /** * Counts the total number of triggers. diff --git a/core/src/main/java/io/kestra/core/runners/FlowInputOutput.java b/core/src/main/java/io/kestra/core/runners/FlowInputOutput.java index ad2d995110e..9d8e63862ef 100644 --- a/core/src/main/java/io/kestra/core/runners/FlowInputOutput.java +++ b/core/src/main/java/io/kestra/core/runners/FlowInputOutput.java @@ -69,7 +69,6 @@ public class FlowInputOutput { private static final Pattern URI_PATTERN = Pattern.compile("^[a-z]+:\\/\\/(?:www\\.)?[-a-zA-Z0-9@:%._\\+~#=]{1,256}\\.[a-zA-Z0-9()]{1,6}\\b(?:[-a-zA-Z0-9()@:%_\\+.~#?&\\/=]*)$"); private static final ObjectMapper YAML_MAPPER = JacksonMapper.ofYaml(); - private static final ObjectMapper JSON_MAPPER = JacksonMapper.ofJson(); private final StorageInterface storageInterface; private final Optional secretKey; @@ -95,11 +94,12 @@ public FlowInputOutput( * @return The list of {@link InputAndValue}. */ public Mono> validateExecutionInputs(final List> inputs, - final Execution execution, - final Publisher data) { + final Flow flow, + final Execution execution, + final Publisher data) { if (ListUtils.isEmpty(inputs)) return Mono.just(Collections.emptyList()); - return readData(inputs, execution, data, false).map(inputData -> resolveInputs(inputs, execution, inputData)); + return readData(inputs, execution, data, false).map(inputData -> resolveInputs(inputs, flow, execution, inputData)); } /** @@ -111,9 +111,9 @@ public Mono> validateExecutionInputs(final List> in * @return The Map of typed inputs. */ public Mono> readExecutionInputs(final Flow flow, - final Execution execution, - final Publisher data) { - return this.readExecutionInputs(flow.getInputs(), execution, data); + final Execution execution, + final Publisher data) { + return this.readExecutionInputs(flow.getInputs(), flow, execution, data); } /** @@ -125,9 +125,10 @@ public Mono> readExecutionInputs(final Flow flow, * @return The Map of typed inputs. */ public Mono> readExecutionInputs(final List> inputs, + final Flow flow, final Execution execution, final Publisher data) { - return readData(inputs, execution, data, true).map(inputData -> this.readExecutionInputs(inputs, execution, inputData)); + return readData(inputs, execution, data, true).map(inputData -> this.readExecutionInputs(inputs, flow, execution, inputData)); } private Mono> readData(List> inputs, Execution execution, Publisher data, boolean uploadFiles) { @@ -192,15 +193,16 @@ public Map readExecutionInputs( final Execution execution, final Map data ) { - return readExecutionInputs(flow.getInputs(), execution, data); + return readExecutionInputs(flow.getInputs(), flow, execution, data); } private Map readExecutionInputs( final List> inputs, + final Flow flow, final Execution execution, final Map data ) { - Map resolved = this.resolveInputs(inputs, execution, data) + Map resolved = this.resolveInputs(inputs, flow, execution, data) .stream() .filter(InputAndValue::enabled) .map(it -> { @@ -225,6 +227,7 @@ private Map readExecutionInputs( @VisibleForTesting public List resolveInputs( final List> inputs, + final Flow flow, final Execution execution, final Map data ) { @@ -240,7 +243,7 @@ public List resolveInputs( }) .collect(Collectors.toMap(it -> it.get().input().getId(), Function.identity(), (o1, o2) -> o1, LinkedHashMap::new))); - resolvableInputMap.values().forEach(input -> resolveInputValue(input, execution, resolvableInputMap)); + resolvableInputMap.values().forEach(input -> resolveInputValue(input, flow, execution, resolvableInputMap)); return resolvableInputMap.values().stream().map(ResolvableInput::get).toList(); } @@ -248,6 +251,7 @@ public List resolveInputs( @SuppressWarnings({"unchecked", "rawtypes"}) private InputAndValue resolveInputValue( final @NotNull ResolvableInput resolvable, + final Flow flow, final @NotNull Execution execution, final @NotNull Map inputs) { @@ -258,8 +262,8 @@ private InputAndValue resolveInputValue( try { // resolve all input dependencies and check whether input is enabled - final Map dependencies = resolveAllDependentInputs(input, execution, inputs); - final RunContext runContext = buildRunContextForExecutionAndInputs(execution, dependencies); + final Map dependencies = resolveAllDependentInputs(input, flow, execution, inputs); + final RunContext runContext = buildRunContextForExecutionAndInputs(flow, execution, dependencies); boolean isInputEnabled = dependencies.isEmpty() || dependencies.values().stream().allMatch(InputAndValue::enabled); @@ -325,15 +329,15 @@ private InputAndValue resolveInputValue( return resolvable.get(); } - private RunContext buildRunContextForExecutionAndInputs(Execution execution, Map dependencies) { + private RunContext buildRunContextForExecutionAndInputs(final Flow flow, final Execution execution, Map dependencies) { Map flattenInputs = MapUtils.flattenToNestedMap(dependencies.entrySet() .stream() .collect(HashMap::new, (m, v) -> m.put(v.getKey(), v.getValue().value()), HashMap::putAll) ); - return runContextFactory.of(null, execution, vars -> vars.withInputs(flattenInputs)); + return runContextFactory.of(flow, execution, vars -> vars.withInputs(flattenInputs)); } - private Map resolveAllDependentInputs(final Input input, final Execution execution, final Map inputs) { + private Map resolveAllDependentInputs(final Input input, final Flow flow, final Execution execution, final Map inputs) { return Optional.ofNullable(input.getDependsOn()) .map(DependsOn::inputs) .stream() @@ -341,7 +345,7 @@ private Map resolveAllDependentInputs(final Input inpu .filter(id -> !id.equals(input.getId())) .map(inputs::get) .filter(Objects::nonNull) // input may declare unknown or non-necessary dependencies. Let's ignore. - .map(it -> resolveInputValue(it, execution, inputs)) + .map(it -> resolveInputValue(it, flow, execution, inputs)) .collect(Collectors.toMap(it -> it.input().getId(), Function.identity())); } diff --git a/core/src/main/java/io/kestra/core/runners/VariableRenderer.java b/core/src/main/java/io/kestra/core/runners/VariableRenderer.java index ed30759bd42..797745249a1 100644 --- a/core/src/main/java/io/kestra/core/runners/VariableRenderer.java +++ b/core/src/main/java/io/kestra/core/runners/VariableRenderer.java @@ -15,8 +15,6 @@ import lombok.Getter; import java.io.IOException; -import java.io.StringWriter; -import java.io.Writer; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -56,8 +54,7 @@ public static IllegalVariableEvaluationException properPebbleException(PebbleExc return new IllegalVariableEvaluationException( "Unable to find `" + current.getAttributeName() + "` used in the expression `" + current.getFileName() + - "` at line " + current.getLineNumber(), - e + "` at line " + current.getLineNumber() ); } diff --git a/core/src/main/java/io/kestra/core/runners/pebble/functions/SecretFunction.java b/core/src/main/java/io/kestra/core/runners/pebble/functions/SecretFunction.java index 0d286972d0f..e97ab23023a 100644 --- a/core/src/main/java/io/kestra/core/runners/pebble/functions/SecretFunction.java +++ b/core/src/main/java/io/kestra/core/runners/pebble/functions/SecretFunction.java @@ -1,9 +1,9 @@ package io.kestra.core.runners.pebble.functions; -import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.runners.RunVariables; import io.kestra.core.secret.SecretNotFoundException; import io.kestra.core.secret.SecretService; +import io.kestra.core.services.FlowService; import io.pebbletemplates.pebble.error.PebbleException; import io.pebbletemplates.pebble.extension.Function; import io.pebbletemplates.pebble.template.EvaluationContext; @@ -15,7 +15,6 @@ import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.function.Consumer; @Slf4j @@ -24,18 +23,32 @@ public class SecretFunction implements Function { @Inject private SecretService secretService; + @Inject + private FlowService flowService; + @Override public List getArgumentNames() { - return List.of("key"); + return List.of("key", "namespace"); } @SuppressWarnings("unchecked") @Override public Object execute(Map args, PebbleTemplate self, EvaluationContext context, int lineNumber) { String key = getSecretKey(args, self, lineNumber); + String namespace = (String) args.get("namespace"); + Map flow = (Map) context.getVariable("flow"); + String flowNamespace = flow.get("namespace"); + String flowTenantId = flow.get("tenantId"); + + if (namespace == null) { + namespace = flowNamespace; + } else { + flowService.checkAllowedNamespace(flowTenantId, namespace, flowTenantId, flowNamespace); + } + try { - String secret = secretService.findSecret(flow.get("tenantId"), flow.get("namespace"), key); + String secret = secretService.findSecret(flowTenantId, namespace, key); try { Consumer addSecretConsumer = (Consumer) context.getVariable(RunVariables.SECRET_CONSUMER_VARIABLE_NAME); diff --git a/core/src/main/java/io/kestra/core/services/ExecutionLogService.java b/core/src/main/java/io/kestra/core/services/ExecutionLogService.java index 2d070a28fb2..838ab40dd29 100644 --- a/core/src/main/java/io/kestra/core/services/ExecutionLogService.java +++ b/core/src/main/java/io/kestra/core/services/ExecutionLogService.java @@ -1,22 +1,14 @@ package io.kestra.core.services; import io.kestra.core.models.executions.LogEntry; -import io.kestra.core.queues.QueueFactoryInterface; -import io.kestra.core.queues.QueueInterface; import io.kestra.core.repositories.LogRepositoryInterface; -import io.micronaut.http.sse.Event; import jakarta.inject.Inject; -import jakarta.inject.Named; import jakarta.inject.Singleton; import org.slf4j.event.Level; -import reactor.core.publisher.Flux; -import reactor.core.publisher.FluxSink; -import reactor.core.scheduler.Schedulers; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.util.List; -import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -25,56 +17,9 @@ */ @Singleton public class ExecutionLogService { - @Inject private LogRepositoryInterface logRepository; - @Inject - @Named(QueueFactoryInterface.WORKERTASKLOG_NAMED) - protected QueueInterface logQueue; - - public Flux> streamExecutionLogs(final String tenantId, - final String executionId, - final Level minLevel, - final boolean withAccessControl) { - - final AtomicReference disposable = new AtomicReference<>(); - - return Flux.>create(emitter -> { - - // send a first "empty" event so the SSE is correctly initialized in the frontend in case there are no logs - emitter.next(Event.of(LogEntry.builder().build()).id("start")); - - // fetch repository first - getExecutionLogs(tenantId, executionId, minLevel, List.of(), withAccessControl) - .forEach(logEntry -> emitter.next(Event.of(logEntry).id("progress"))); - - final List levels = LogEntry.findLevelsByMin(minLevel).stream().map(Enum::name).toList(); - - // consume in realtime - disposable.set(this.logQueue.receive(either -> { - if (either.isRight()) { - return; - } - - LogEntry current = either.getLeft(); - - if (current.getExecutionId() != null && current.getExecutionId().equals(executionId)) { - if (levels.contains(current.getLevel().name())) { - emitter.next(Event.of(current).id("progress")); - } - } - })); - }, FluxSink.OverflowStrategy.BUFFER) - .doFinally(ignored -> { - Schedulers.boundedElastic().schedule(() -> { - if (disposable.get() != null) { - disposable.get().run(); - } - }); - }); - } - public InputStream getExecutionLogsAsStream(String tenantId, String executionId, Level minLevel, diff --git a/core/src/main/java/io/kestra/core/services/ExecutionService.java b/core/src/main/java/io/kestra/core/services/ExecutionService.java index 48f047c3208..c078a43dc2d 100644 --- a/core/src/main/java/io/kestra/core/services/ExecutionService.java +++ b/core/src/main/java/io/kestra/core/services/ExecutionService.java @@ -486,7 +486,7 @@ public Mono> validateForResume(final Execution execution, Fl return getFirstPausedTaskOr(execution, flow) .flatMap(task -> { if (task.isPresent() && task.get() instanceof Pause pauseTask) { - return Mono.just(flowInputOutput.resolveInputs(pauseTask.getOnResume(), execution, Map.of())); + return Mono.just(flowInputOutput.resolveInputs(pauseTask.getOnResume(), flow, execution, Map.of())); } else { return Mono.just(Collections.emptyList()); } @@ -507,7 +507,7 @@ public Mono> validateForResume(final Execution execution, Fl return getFirstPausedTaskOr(execution, flow) .flatMap(task -> { if (task.isPresent() && task.get() instanceof Pause pauseTask) { - return flowInputOutput.validateExecutionInputs(pauseTask.getOnResume(), execution, inputs); + return flowInputOutput.validateExecutionInputs(pauseTask.getOnResume(), flow, execution, inputs); } else { return Mono.just(Collections.emptyList()); } @@ -528,7 +528,7 @@ public Mono resume(final Execution execution, Flow flow, State.Type n return getFirstPausedTaskOr(execution, flow) .flatMap(task -> { if (task.isPresent() && task.get() instanceof Pause pauseTask) { - return flowInputOutput.readExecutionInputs(pauseTask.getOnResume(), execution, inputs); + return flowInputOutput.readExecutionInputs(pauseTask.getOnResume(), flow, execution, inputs); } else { return Mono.just(Collections.emptyMap()); } diff --git a/core/src/main/java/io/kestra/core/services/LabelService.java b/core/src/main/java/io/kestra/core/services/LabelService.java index 05c54c9d335..c6094554616 100644 --- a/core/src/main/java/io/kestra/core/services/LabelService.java +++ b/core/src/main/java/io/kestra/core/services/LabelService.java @@ -6,6 +6,7 @@ import io.kestra.core.models.triggers.AbstractTrigger; import io.kestra.core.runners.RunContext; import io.kestra.core.utils.ListUtils; +import jakarta.annotation.Nullable; import java.util.*; @@ -54,9 +55,9 @@ private static String renderLabelValue(RunContext runContext, Label label) { } } - public static boolean containsAll(List