Upload instrumentation report regardless if tests failed #14983
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Android - Build and test | |
| on: | |
| pull_request: | |
| paths: | |
| - '**' | |
| - '!.github/workflows/**' | |
| - '.github/workflows/android-app.yml' | |
| - '!.github/CODEOWNERS' | |
| - '!audits/**' | |
| - '!ci/**' | |
| - '!dist-assets/**' | |
| - '!docs/**' | |
| - '!graphics/**' | |
| - '!gui/**' | |
| - '!ios/**' | |
| - '!test/**' | |
| - '!scripts/**' | |
| - '!windows/**' | |
| - '!**/**.md' | |
| - '!**/osv-scanner.toml' | |
| schedule: | |
| # At 00:00 UTC every day. | |
| # Notifications for scheduled workflows are sent to the user who last modified the cron | |
| # syntax in the workflow file. If you update this you must have notifications for | |
| # Github Actions enabled, so these don't go unnoticed. | |
| # https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/notifications-for-workflow-runs | |
| - cron: '0 0 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| override_container_image: | |
| description: Override container image | |
| type: string | |
| required: false | |
| run_firebase_tests: | |
| description: Run firebase tests | |
| type: boolean | |
| required: false | |
| mockapi_test_repeat: | |
| description: Mockapi test repeat(self hosted) | |
| default: '1' | |
| required: true | |
| type: string | |
| e2e_test_repeat: | |
| description: e2e test repeat(self hosted) | |
| default: '0' | |
| required: true | |
| type: string | |
| # Build if main is updated to ensure up-to-date caches are available | |
| push: | |
| branches: [main] | |
| permissions: {} | |
| jobs: | |
| prepare: | |
| name: Prepare | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Use custom container image if specified | |
| if: ${{ github.event.inputs.override_container_image != '' }} | |
| run: echo "inner_container_image=${{ github.event.inputs.override_container_image }}" | |
| >> $GITHUB_ENV | |
| - name: Use default container image and resolve digest | |
| if: ${{ github.event.inputs.override_container_image == '' }} | |
| run: | | |
| echo "inner_container_image=$(cat ./building/android-container-image.txt)" >> $GITHUB_ENV | |
| outputs: | |
| container_image: ${{ env.inner_container_image }} | |
| generate-debug-keystore: | |
| name: Generate debug keystore | |
| needs: prepare | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Generate keystore | |
| run: >- | |
| keytool -genkey | |
| -keystore debug.keystore | |
| -storepass android | |
| -alias androiddebugkey | |
| -keypass android | |
| -keyalg RSA | |
| -keysize 2048 | |
| -validity 10000 | |
| -dname "CN=Android Debug,O=Android,C=US" | |
| - name: Upload keystore | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: debug-keystore | |
| path: debug.keystore | |
| if-no-files-found: error | |
| retention-days: 7 | |
| generate-relay-list: | |
| name: Generate relay list | |
| needs: prepare | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ${{ needs.prepare.outputs.container_image }} | |
| steps: | |
| # Fix for HOME path overridden by GH runners when building in containers, see: | |
| # https://github.com/actions/runner/issues/863 | |
| - name: Fix HOME path | |
| run: echo "HOME=/root" >> $GITHUB_ENV | |
| - name: Get date | |
| id: get-date | |
| shell: bash | |
| run: echo "date=$(/bin/date -u "+%Y%m%d")" >> $GITHUB_OUTPUT | |
| - name: Cache | |
| uses: actions/cache@v4 | |
| id: cache-relay-list | |
| with: | |
| path: android/app/build/extraAssets/relays.json | |
| key: relay-list-${{ steps.get-date.outputs.date }} | |
| - name: Checkout repository | |
| if: steps.cache-relay-list.outputs.cache-hit != 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - name: Generate | |
| if: steps.cache-relay-list.outputs.cache-hit != 'true' | |
| env: | |
| RUSTFLAGS: --deny warnings | |
| run: | | |
| mkdir -p android/app/build/extraAssets | |
| cargo run --bin relay_list > android/app/build/extraAssets/relays.json | |
| - name: Upload | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: relay-list | |
| path: android/app/build/extraAssets/relays.json | |
| if-no-files-found: error | |
| retention-days: 7 | |
| build-native: | |
| name: Build native | |
| needs: prepare | |
| runs-on: ubuntu-latest | |
| container: | |
| image: "${{ needs.prepare.outputs.container_image }}" | |
| strategy: | |
| matrix: | |
| include: | |
| - arch: "x86_64" | |
| abi: "x86_64" | |
| target: "x86_64-linux-android" | |
| - arch: "i686" | |
| abi: "x86" | |
| target: "i686-linux-android" | |
| - arch: "aarch64" | |
| abi: "arm64-v8a" | |
| target: "aarch64-linux-android" | |
| - arch: "armv7" | |
| abi: "armeabi-v7a" | |
| target: "armv7-linux-androideabi" | |
| steps: | |
| # Fix for HOME path overridden by GH runners when building in containers, see: | |
| # https://github.com/actions/runner/issues/863 | |
| - name: Fix HOME path | |
| run: echo "HOME=/root" >> $GITHUB_ENV | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - name: Checkout wireguard-go-rs recursively | |
| run: | | |
| git config --global --add safe.directory '*' | |
| git submodule update --init --recursive --depth=1 wireguard-go-rs | |
| - name: Calculate native lib cache hash | |
| id: native-lib-cache-hash | |
| shell: bash | |
| run: | | |
| git config --global --add safe.directory $(pwd) | |
| non_android_hash="$(git grep --cached -l '' -- ':!android/' \ | |
| | xargs -d '\n' sha1sum \ | |
| | sha1sum \ | |
| | awk '{print $1}')" | |
| echo "native_lib_hash=$non_android_hash" >> $GITHUB_OUTPUT | |
| - name: Cache native libraries | |
| uses: actions/cache@v4 | |
| id: cache-native-libs | |
| env: | |
| cache_hash: ${{ steps.native-lib-cache-hash.outputs.native_lib_hash }} | |
| with: | |
| path: ./android/app/build/extraJni | |
| key: android-native-libs-${{ runner.os }}-${{ matrix.abi }}-${{ env.cache_hash }} | |
| - name: Build native libraries | |
| if: steps.cache-native-libs.outputs.cache-hit != 'true' | |
| env: | |
| RUSTFLAGS: --deny warnings | |
| BUILD_TYPE: debug | |
| run: | | |
| ARCHITECTURES="${{ matrix.abi }}" | |
| UNSTRIPPED_LIB_PATH="$CARGO_TARGET_DIR/${{ matrix.target }}/$BUILD_TYPE/libmullvad_jni.so" | |
| STRIPPED_LIB_PATH="./android/app/build/extraJni/${{ matrix.abi }}/libmullvad_jni.so" | |
| NDK_TOOLCHAIN_STRIP_TOOL="$NDK_TOOLCHAIN_DIR/llvm-strip" | |
| cargo build --target ${{ matrix.target }} --verbose --package mullvad-jni --features api-override | |
| $NDK_TOOLCHAIN_STRIP_TOOL --strip-debug --strip-unneeded -o "$STRIPPED_LIB_PATH" "$UNSTRIPPED_LIB_PATH" | |
| - name: Upload native libs | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: native-libs-${{ matrix.arch }} | |
| path: android/app/build/extraJni | |
| if-no-files-found: error | |
| retention-days: 7 | |
| run-lint-and-tests: | |
| name: Run lint and test tasks | |
| needs: [prepare] | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ${{ needs.prepare.outputs.container_image }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - gradle-task: | | |
| testDebugUnitTest -x :test:arch:testDebugUnitTest | |
| :app:testOssProdDebugUnitTest | |
| :service:testOssProdDebugUnitTest | |
| :lib:billing:testDebugUnitTest | |
| :lib:daemon-grpc:testDebugUnitTest | |
| :lib:shared:testDebugUnitTest | |
| - gradle-task: :test:arch:test --rerun-tasks | |
| - gradle-task: detekt | |
| - gradle-task: lint | |
| steps: | |
| # Fix for HOME path overridden by GH runners when building in containers, see: | |
| # https://github.com/actions/runner/issues/863 | |
| - name: Fix HOME path | |
| run: echo "HOME=/root" >> $GITHUB_ENV | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - name: Run gradle task | |
| uses: burrunan/gradle-cache-action@v1 | |
| with: | |
| job-id: jdk17 | |
| arguments: ${{ matrix.gradle-task }} | |
| gradle-version: wrapper | |
| build-root-directory: android | |
| execution-only-caches: false | |
| # Disable if logs are hard to follow. | |
| concurrent: true | |
| read-only: ${{ github.ref != 'refs/heads/main' }} | |
| build-app: | |
| name: Build app | |
| needs: [prepare, generate-debug-keystore] | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ${{ needs.prepare.outputs.container_image }} | |
| steps: | |
| # Fix for HOME path overridden by GH runners when building in containers, see: | |
| # https://github.com/actions/runner/issues/863 | |
| - name: Fix HOME path | |
| run: echo "HOME=/root" >> $GITHUB_ENV | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: debug-keystore | |
| path: /root/.android | |
| - name: Compile app | |
| uses: burrunan/gradle-cache-action@v1 | |
| with: | |
| job-id: jdk17 | |
| arguments: compileOssProdDebugKotlin | |
| gradle-version: wrapper | |
| build-root-directory: android | |
| execution-only-caches: false | |
| # Disable if logs are hard to follow. | |
| concurrent: true | |
| read-only: ${{ github.ref != 'refs/heads/main' }} | |
| - name: Wait for other jobs (native, relay list) | |
| uses: kachick/[email protected] | |
| with: | |
| wait-list: | | |
| [ | |
| { | |
| "workflowFile": "android-app.yml", | |
| "jobName": "build-native" | |
| }, | |
| { | |
| "workflowFile": "android-app.yml", | |
| "jobName": "generate-relay-list" | |
| } | |
| ] | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: native-libs-* | |
| path: android/app/build/extraJni | |
| merge-multiple: true | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: relay-list | |
| path: android/app/build/extraAssets | |
| - name: Copy maybenot machines to asset directory | |
| run: cp dist-assets/maybenot_machines android/app/build/extraAssets/maybenot_machines | |
| - name: Build app | |
| uses: burrunan/gradle-cache-action@v1 | |
| with: | |
| job-id: jdk17 | |
| arguments: assembleOssProdDebug | |
| gradle-version: wrapper | |
| build-root-directory: android | |
| execution-only-caches: true | |
| # Disable if logs are hard to follow. | |
| concurrent: true | |
| read-only: ${{ github.ref != 'refs/heads/main' }} | |
| - name: Build stagemole app | |
| uses: burrunan/gradle-cache-action@v1 | |
| if: github.event_name == 'schedule' || github.event.inputs.run_firebase_tests == 'true' | |
| with: | |
| job-id: jdk17 | |
| arguments: assemblePlayStagemoleDebug | |
| gradle-version: wrapper | |
| build-root-directory: android | |
| execution-only-caches: true | |
| # Disable if logs are hard to follow. | |
| concurrent: true | |
| read-only: ${{ github.ref != 'refs/heads/main' }} | |
| - name: Upload apks | |
| # Using v3 due to v4 being very slow for this artifact. | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: apks | |
| path: android/app/build/outputs/apk | |
| if-no-files-found: error | |
| retention-days: 7 | |
| build-instrumented-tests: | |
| name: Build instrumented test packages | |
| needs: [prepare, generate-debug-keystore] | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ${{ needs.prepare.outputs.container_image }} | |
| strategy: | |
| matrix: | |
| include: | |
| - test-type: app | |
| assemble-command: assembleOssProdAndroidTest | |
| artifact-path: android/app/build/outputs/apk | |
| - test-type: mockapi | |
| assemble-command: :test:mockapi:assemble | |
| artifact-path: android/test/mockapi/build/outputs/apk | |
| - test-type: e2e | |
| assemble-command: :test:e2e:assemble | |
| artifact-path: android/test/e2e/build/outputs/apk | |
| steps: | |
| # Fix for HOME path overridden by GH runners when building in containers, see: | |
| # https://github.com/actions/runner/issues/863 | |
| - name: Fix HOME path | |
| run: echo "HOME=/root" >> $GITHUB_ENV | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: debug-keystore | |
| path: /root/.android | |
| - name: Assemble instrumented test apk | |
| uses: burrunan/gradle-cache-action@v1 | |
| with: | |
| job-id: jdk17 | |
| arguments: ${{ matrix.assemble-command }} | |
| gradle-version: wrapper | |
| build-root-directory: android | |
| execution-only-caches: false | |
| # Disable if logs are hard to follow. | |
| concurrent: true | |
| read-only: ${{ github.ref != 'refs/heads/main' }} | |
| - name: Upload apks | |
| # Using v3 due to v4 being very slow for this artifact. | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ matrix.test-type }}-instrumentation-apks | |
| path: ${{ matrix.artifact-path }} | |
| if-no-files-found: error | |
| retention-days: 7 | |
| instrumented-tests: | |
| name: Run instrumented tests | |
| runs-on: [self-hosted, android-device] | |
| needs: [build-app, build-instrumented-tests] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - test-type: app | |
| path: android/app/build/outputs/apk | |
| test-repeat: 1 | |
| - test-type: mockapi | |
| path: android/test/mockapi/build/outputs/apk | |
| test-repeat: ${{ github.event_name == 'schedule' && 10 || github.event.inputs.mockapi_test_repeat || 1 }} | |
| steps: | |
| - name: Prepare report dir | |
| if: ${{ matrix.test-repeat != 0 }} | |
| id: prepare-report-dir | |
| env: | |
| INNER_REPORT_DIR: /tmp/${{ matrix.test-type }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| run: | | |
| mkdir -p $INNER_REPORT_DIR | |
| echo "report_dir=$INNER_REPORT_DIR" >> $GITHUB_OUTPUT | |
| - name: Checkout repository | |
| if: ${{ matrix.test-repeat != 0 }} | |
| uses: actions/checkout@v4 | |
| # Using v3 due to v4 being very slow for this artifact. | |
| - uses: actions/download-artifact@v3 | |
| if: ${{ matrix.test-repeat != 0 }} | |
| with: | |
| name: apks | |
| path: android/app/build/outputs/apk | |
| # Using v3 due to v4 being very slow for this artifact. | |
| - uses: actions/download-artifact@v3 | |
| if: ${{ matrix.test-repeat != 0 }} | |
| with: | |
| name: ${{ matrix.test-type }}-instrumentation-apks | |
| path: ${{ matrix.path }} | |
| - name: Calculate timeout | |
| id: calculate-timeout | |
| run: echo "timeout=$(( ${{ matrix.test-repeat }} * 10 ))" >> $GITHUB_OUTPUT | |
| shell: bash | |
| - name: Run instrumented test script | |
| if: ${{ matrix.test-repeat != 0 }} | |
| timeout-minutes: ${{ fromJSON(steps.calculate-timeout.outputs.timeout) }} | |
| shell: bash -ieo pipefail {0} | |
| env: | |
| AUTO_FETCH_TEST_HELPER_APKS: true | |
| TEST_TYPE: ${{ matrix.test-type }} | |
| BILLING_FLAVOR: oss | |
| INFRA_FLAVOR: prod | |
| REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }} | |
| run: ./android/scripts/run-instrumented-tests-repeat.sh ${{ matrix.test-repeat }} | |
| - name: Upload instrumentation report (${{ matrix.test-type }}) | |
| uses: actions/upload-artifact@v4 | |
| if: always() && matrix.test-repeat != 0 | |
| with: | |
| name: ${{ matrix.test-type }}-instrumentation-report | |
| path: ${{ steps.prepare-report-dir.outputs.report_dir }} | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| instrumented-e2e-tests: | |
| name: Run instrumented e2e tests | |
| # Temporary workaround for targeting the runner android-runner-v1 | |
| runs-on: [self-hosted, android-device, android-emulator] | |
| if: github.event_name == 'schedule' || ${{ github.event.inputs.e2e_test_repeat }} != '0' | |
| needs: [build-app, build-instrumented-tests] | |
| strategy: | |
| matrix: | |
| include: | |
| - test-repeat: ${{ github.event.inputs.e2e_test_repeat || 1 }} | |
| steps: | |
| - name: Prepare report dir | |
| if: ${{ matrix.test-repeat != 0 }} | |
| id: prepare-report-dir | |
| env: | |
| INNER_REPORT_DIR: /tmp/${{ github.run_id }}-${{ github.run_attempt }} | |
| run: | | |
| mkdir -p $INNER_REPORT_DIR | |
| echo "report_dir=$INNER_REPORT_DIR" >> $GITHUB_OUTPUT | |
| - name: Checkout repository | |
| if: ${{ matrix.test-repeat != 0 }} | |
| uses: actions/checkout@v4 | |
| # Using v3 due to v4 being very slow for this artifact. | |
| - uses: actions/download-artifact@v3 | |
| if: ${{ matrix.test-repeat != 0 }} | |
| with: | |
| name: apks | |
| path: android/app/build/outputs/apk | |
| # Using v3 due to v4 being very slow for this artifact. | |
| - uses: actions/download-artifact@v3 | |
| if: ${{ matrix.test-repeat != 0 }} | |
| with: | |
| name: e2e-instrumentation-apks | |
| path: android/test/e2e/build/outputs/apk | |
| - name: Calculate timeout | |
| id: calculate-timeout | |
| run: echo "timeout=$(( ${{ matrix.test-repeat }} * 10 ))" >> $GITHUB_OUTPUT | |
| shell: bash | |
| - name: Run instrumented test script | |
| if: ${{ matrix.test-repeat != 0 }} | |
| timeout-minutes: ${{ fromJSON(steps.calculate-timeout.outputs.timeout) }} | |
| shell: bash -ieo pipefail {0} | |
| env: | |
| AUTO_FETCH_TEST_HELPER_APKS: true | |
| TEST_TYPE: e2e | |
| BILLING_FLAVOR: oss | |
| INFRA_FLAVOR: prod | |
| VALID_TEST_ACCOUNT_NUMBER: ${{ secrets.ANDROID_PROD_TEST_ACCOUNT }} | |
| INVALID_TEST_ACCOUNT_NUMBER: '0000000000000000' | |
| ENABLE_HIGHLY_RATE_LIMITED_TESTS: ${{ github.event_name == 'schedule' && 'true' || 'false' }} | |
| ENABLE_ACCESS_TO_LOCAL_API_TESTS: true | |
| REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }} | |
| run: ./android/scripts/run-instrumented-tests-repeat.sh ${{ matrix.test-repeat }} | |
| firebase-tests: | |
| name: Run firebase tests | |
| if: github.event_name == 'schedule' || github.event.inputs.run_firebase_tests == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| needs: [build-app, build-instrumented-tests] | |
| env: | |
| FIREBASE_ENVIRONMENT_VARIABLES: "\ | |
| clearPackageData=true,\ | |
| runnerBuilder=de.mannodermaus.junit5.AndroidJUnit5Builder,\ | |
| invalid_test_account_number=0000000000000000,\ | |
| ENABLE_HIGHLY_RATE_LIMITED_TESTS=${{ github.event_name == 'schedule' && 'true' || 'false' }},\ | |
| partner_auth=${{ secrets.STAGEMOLE_PARTNER_AUTH }},\ | |
| ENABLE_ACCESS_TO_LOCAL_API_TESTS=false" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - test-type: mockapi | |
| arg-spec-file: mockapi-oss.yml | |
| path: android/test/mockapi/build/outputs/apk | |
| - test-type: e2e | |
| arg-spec-file: e2e-play-stagemole.yml | |
| path: android/test/e2e/build/outputs/apk | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| # Using v3 due to v4 being very slow for this artifact. | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: apks | |
| path: android/app/build/outputs/apk | |
| # Using v3 due to v4 being very slow for this artifact. | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: ${{ matrix.test-type }}-instrumentation-apks | |
| path: ${{ matrix.path }} | |
| - name: Run tests on Firebase Test Lab | |
| uses: asadmansr/[email protected] | |
| env: | |
| SERVICE_ACCOUNT: ${{ secrets.FIREBASE_SERVICE_ACCOUNT }} | |
| with: | |
| arg-spec: | | |
| android/test/firebase/${{ matrix.arg-spec-file }}:default | |
| --environment-variables ${{ env.FIREBASE_ENVIRONMENT_VARIABLES }} |