From 894c7d871cd20387c2b095e60bfb4d0d933a3ea1 Mon Sep 17 00:00:00 2001 From: David Arnold Date: Tue, 26 Nov 2024 18:11:08 +0100 Subject: [PATCH] ci: prepare shared action for concrete downstream use (#28572) * ci: cleanup setup action * ci: make migration & setup downstream-ready * ci: replace obscure bench remove-app with an equivalent rm -rf --- .github/actions/setup/action.yml | 232 ++++++++++++----------- .github/workflows/_base-migration.yml | 156 ++++++++++----- .github/workflows/_base-server-tests.yml | 10 +- .github/workflows/_base-ui-tests.yml | 21 +- pyproject.toml | 3 + 5 files changed, 249 insertions(+), 173 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 43256b9ed10c..3e8e9a94b4c9 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -28,9 +28,6 @@ inputs: disable-socketio: required: false default: false - disable-redis-socketio: - required: false - default: false db: required: false default: mariadb @@ -40,18 +37,17 @@ inputs: runs: using: "composite" steps: - - name: Clone - uses: actions/checkout@v4 - with: - path: ${{ github.event.repository.name }} + - shell: bash -e {0} + run: | + # Add 'test_site' to /etc/hosts & setup git config + echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts + git config --global init.defaultBranch main + git config --global advice.detachedHead false - - name: Checkout Frappe + - name: Clone uses: actions/checkout@v4 with: - repository: ${{ env.FRAPPE_GH_ORG || github.repository_owner }}/frappe - ref: ${{ github.event.client_payload.frappe_sha || github.base_ref || github.ref_name }} - path: frappe - if: github.event.repository.name != 'frappe' + path: apps/${{ github.event.repository.name }} - name: Setup Python uses: actions/setup-python@v5 @@ -67,18 +63,64 @@ runs: exit 1 fi + - name: Fetch tombl util + shell: bash -e {0} + run: | + # Fetch tombl util + pushd $RUNNER_TEMP + curl -LO https://github.com/snyball/tombl/releases/download/v0.2.3/tombl-v0.2.3.tar.gz + curl -LO https://github.com/snyball/tombl/releases/download/v0.2.3/tombl-v0.2.3.sha256sum + sha256sum -c tombl-v0.2.3.sha256sum + tar -xzf tombl-v0.2.3.tar.gz + chmod +x tombl-v0.2.3/tombl + popd + echo "$RUNNER_TEMP/tombl-v0.2.3" >> $GITHUB_PATH + + - name: Checkout Frappe + uses: actions/checkout@v4 + with: + repository: ${{ env.FRAPPE_GH_ORG || github.repository_owner }}/frappe + ref: ${{ github.event.client_payload.frappe_sha || github.base_ref || github.ref_name }} + path: apps/frappe + if: github.event.repository.name != 'frappe' + + - name: Maybe clone additional apps + shell: bash -e {0} + env: + org: ${{ env.FRAPPE_GH_ORG || github.repository_owner }} + ref: ${{ github.event.client_payload.frappe_sha || github.base_ref || github.ref_name }} + run: | + # Maybe clone additional apps + eval "$(tombl -e CHECKOUTS=tool.frappe-ci.setup.app-checkouts ${{ github.event.repository.name }}/pyproject.toml)" || exit 0 + start_time=$(date +%s) + + for spec in "${CHECKOUTS[@]}"; do + spec_basename=$(basename "$spec") + if [[ "$spec" == *"/"* ]] || [[ "$spec" == http* ]]; then + remote_url="$spec" + else + remote_url="$org/$spec_basename" + fi + if [[ "$remote_url" != http* ]]; then + remote_url="https://github.com/$remote_url" + fi + mkdir "apps/$spec_basename" + pushd "apps/$spec_basename" + git init + git remote add origin "$remote_url" + git fetch origin "$ref" --depth 1 + git checkout FETCH_HEAD --quiet + popd + done + end_time=$(date +%s) + echo -e "\033[33mClone additional apps: $((end_time - start_time)) seconds\033[0m" + + - uses: actions/setup-node@v4 with: node-version: ${{ inputs.node-version }} check-latest: true - - shell: bash -e {0} - run: | - # Add 'test_site' to /etc/hosts - echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts - git config --global init.defaultBranch main - git config --global advice.detachedHead false - - name: Cache pip uses: actions/cache@v4 with: @@ -120,68 +162,48 @@ runs: echo -e "\033[33mInstall System Dependencies: $((end_time - start_time)) seconds\033[0m" - shell: bash -e {0} + env: + DB: ${{ inputs.db }} run: | - # Install Bench - start_time=$(date +%s) - - cd ~ || exit - pip install frappe-bench - - end_time=$(date +%s) - echo -e "\033[33mInstall Bench: $((end_time - start_time)) seconds\033[0m" - - - shell: bash -e {0} - run: | - # Init Bench + # Init Bench & test_site start_time=$(date +%s) + mkdir ${GITHUB_WORKSPACE}/{sites,config,logs,config/pids,sites/test_site} + python -m venv ${GITHUB_WORKSPACE}/env + source ${GITHUB_WORKSPACE}/env/bin/activate + pip install --quiet --upgrade pip + + # pip install --quiet frappe-bench + # revert after merge: https://github.com/frappe/bench/pull/1600 + pip install --quiet git+https://github.com/blaggacao/bench.git@feat/add-direct-config-module-calling + + python -m bench.config.common_site_config ${GITHUB_WORKSPACE} + python -m bench.config.redis ${GITHUB_WORKSPACE} + args=() + if [[ "${{ inputs.enable-watch }}" != "true" ]]; then + args+=("--skip-watch") + fi - cd ~ || exit - verbosity="${BENCH_INIT_VERBOSITY_FLAG:-}" - bench $verbosity init frappe-bench --skip-assets --python "$(which python)" --frappe-path "${GITHUB_WORKSPACE}/frappe" - - end_time=$(date +%s) - echo -e "\033[33mInit Bench: $((end_time - start_time)) seconds\033[0m" - - - shell: bash -e {0} - run: | - # Install App(s) - start_time=$(date +%s) + if [[ "${{ inputs.enable-schedule }}" != "true" ]]; then + args+=("--skip-schedule") + fi - cd ~/frappe-bench || exit - verbosity="${BENCH_INIT_VERBOSITY_FLAG:-}" + if [[ "${{ inputs.disable-socketio }}" == "true" ]]; then + args+=("--skip-socketio") + fi - for app in ${GITHUB_WORKSPACE}/*/; do - if [ -f "${app}setup.py" ] || [ -f "${app}pyproject.toml" ]; then - start_time=$(date +%s) - echo "Installing app in ${app}" - pip install --upgrade -e "${app}[dev,test]" - end_time=$(date +%s) - echo "Time taken to Install ${app} requirements: $((end_time - start_time)) seconds" - fi - done - # collect old style tools.bench.dev-dependencies - bench $verbosity setup requirements --dev; - if [ "$TYPE" == "ui" ] - then - bench $verbosity setup requirements --node; + if [[ "${{ inputs.disable-web }}" == "true" ]]; then + args+=("--skip-web") fi + if [[ "${{ inputs.enable-coverage }}" == "true" ]]; then + args+=("--with-coverage") + fi + python -m bench.config.procfile ${GITHUB_WORKSPACE} "${args[@]}" end_time=$(date +%s) - echo -e "\033[33mInstall App(s): $((end_time - start_time)) seconds\033[0m" - env: - TYPE: server - - - shell: bash -e {0} - run: | - # Setup Test Site - start_time=$(date +%s) - - cd ~/frappe-bench || exit - - mkdir ~/frappe-bench/sites/test_site - + echo -e "\033[33mInit Bench: $((end_time - start_time)) seconds\033[0m" + cat ${GITHUB_WORKSPACE}/Procfile | awk '{print "\033[0;34m" $0 "\033[0m"}' # Attempt to copy the configuration file - if cp "${GITHUB_WORKSPACE}/${{ github.event.repository.name }}/.github/helper/db/$DB.json" ~/frappe-bench/sites/test_site/site_config.json; then + if cp "${GITHUB_WORKSPACE}/apps/${{ github.event.repository.name }}/.github/helper/db/$DB.json" ${GITHUB_WORKSPACE}/sites/test_site/site_config.json; then echo "Successfully copied ${DB}.json to site_config.json." else echo "Error: The configuration file ${GITHUB_WORKSPACE}/${{ github.event.repository.name }}/.github/helper/db/$DB.json does not exist." @@ -203,45 +225,43 @@ runs: echo "${{ inputs.db-root-password }}" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe WITH PASSWORD 'test_frappe'" -U postgres; fi - end_time=$(date +%s) - echo -e "\033[33mSetup Test Site: $((end_time - start_time)) seconds\033[0m" - env: - DB: ${{ inputs.db }} - - shell: bash -e {0} run: | - # Modify Procfile - cd ~/frappe-bench || exit - if ${{ inputs.enable-watch != 'true' }}; then - sed -i 's/^watch:/# watch:/g' Procfile - fi - if ${{ inputs.enable-schedule != 'true'}}; then - sed -i 's/^schedule:/# schedule:/g' Procfile - fi - if ${{ inputs.disable-socketio }}; then - sed -i 's/^socketio:/# socketio:/g' Procfile - fi - if ${{ inputs.disable-redis-socketio }}; then - sed -i 's/^redis_socketio:/# redis_socketio:/g' Procfile - fi - if ${{ inputs.enable-coverage }}; then - sed -i 's/^web: bench serve/web: bench serve --with-coverage/g' Procfile - fi - if ${{ inputs.disable-web }}; then - sed -i 's/^web:/# web:/g' Procfile - fi + # Install App(s) + step_start_time=$(date +%s) + source ${GITHUB_WORKSPACE}/env/bin/activate - - shell: bash -e {0} - run: | - # Display modified Procfile - cd ~/frappe-bench || exit - cat Procfile | awk '{print "\033[0;34m" $0 "\033[0m"}' + for app in ${GITHUB_WORKSPACE}/apps/*/; do + app_name="$(basename $app)" + if [ -f "${app}setup.py" ] || [ -f "${app}pyproject.toml" ]; then + start_time=$(date +%s) + echo -e "\033[36mInstalling python app from ${app}\033[0m" + pip install --upgrade -e "${app}[dev,test]" + end_time=$(date +%s) + echo -e "\033[36mTime taken to Install python ${app}: $((end_time - start_time)) seconds\033[0m" + fi + if [ "${{ inputs.build-assets }}" == "true" ] && [ -f "${app}package.json" ]; then + start_time=$(date +%s) + echo -e "\033[36mInstalling js app dependencies from ${app}\033[0m" + pushd "$app" + yarn --check-files + popd + end_time=$(date +%s) + echo -e "\033[36mTime taken to Install js ${app}: $((end_time - start_time)) seconds\033[0m" + fi + echo "$app_name" >> sites/apps.txt + echo -e "\033[32mAdded $app_name to $PWD/sites/apps.txt\033[0m" + done + step_end_time=$(date +%s) + echo -e "\033[33mInstall App(s): $((step_end_time - step_start_time)) seconds\033[0m" + env: + TYPE: server - shell: bash -e {0} run: | # Start Bench - cd ~/frappe-bench || exit - bench start &> ~/frappe-bench/bench_start.log & + source ${GITHUB_WORKSPACE}/env/bin/activate + bench start &> ${GITHUB_WORKSPACE}/bench_start.log & - shell: bash -e {0} if: ${{ inputs.build-assets == 'true' }} @@ -249,7 +269,7 @@ runs: # Build Assets start_time=$(date +%s) - cd ~/frappe-bench || exit + source ${GITHUB_WORKSPACE}/env/bin/activate CI=Yes bench build end_time=$(date +%s) @@ -260,7 +280,7 @@ runs: # Reinstall Test Site start_time=$(date +%s) - cd ~/frappe-bench || exit + source ${GITHUB_WORKSPACE}/env/bin/activate bench --site test_site reinstall --yes end_time=$(date +%s) diff --git a/.github/workflows/_base-migration.yml b/.github/workflows/_base-migration.yml index 3fd339863c55..4d6c0b498100 100644 --- a/.github/workflows/_base-migration.yml +++ b/.github/workflows/_base-migration.yml @@ -44,79 +44,135 @@ jobs: disable-web: true db-root-password: ${{ env.DB_ROOT_PASSWORD }} - - name: Recover v13 database artifact + - name: Download database artifact + env: + current-base-ref: ${{ github.base_ref || github.ref_name }} + current-head-ref: ${{ github.head_ref || github.ref_name }} run: | - cd ~/frappe-bench/ - wget https://frappeframework.com/files/v13-frappe.sql.gz - bench --site test_site --force restore ~/frappe-bench/v13-frappe.sql.gz + eval "$(tombl -e MIGRATION_DB=tool.frappe-ci.setup.migration-db ${GITHUB_WORKSPACE}/apps/${{ github.event.repository.name }}/pyproject.toml)" || exit 0 + source ${GITHUB_WORKSPACE}/env/bin/activate + wget "$MIGRATION_DB" + bench --site test_site --force restore ${GITHUB_WORKSPACE}/$(basename "$MIGRATION_DB") - source env/bin/activate - cd apps/frappe/ - git remote set-url upstream https://github.com/frappe/frappe.git - - - name: Update to v14 - run: | - cd ~/frappe-bench/apps/frappe/ function update_to_version() { - version=$1 + eval "$(tombl -e FRAPPE_DEPENDENCIES=tool.bench.frappe-dependencies ${GITHUB_WORKSPACE}/apps/${{ github.event.repository.name }}/pyproject.toml)" || true + version="$1" + if [ -z "$version" ]; then + base_ref="${{ env.current-base-ref }}" + base_ref="${{ env.current-head-ref }}" + else + base_ref="version-$version-hotfix" + head_ref="version-$version-hotfix" + fi - branch_name="version-$version-hotfix" - echo "Updating to v$version" - git fetch --depth 1 upstream $branch_name:$branch_name - git checkout -q -f $branch_name + source ${GITHUB_WORKSPACE}/env/bin/activate + echo "Updating to version ${version:-$base_ref}" + + # Fetch and checkout branches + for app in ${GITHUB_WORKSPACE}/apps/*/; do + app_name=$(basename "$app") + echo "Processing app: $app_name" - pgrep honcho | xargs kill - sleep 3 - rm -rf ~/frappe-bench/env - bench -v setup env - bench start &>> ~/frappe-bench/bench_start.log & + if [[ ! " ${FRAPPE_DEPENDENCIES[@]} " =~ " $app_name " && "$app_name" != "${{ github.event.repository.name }}" ]]; then + rm -rf $app + echo "Removed $app_name as it's not part of tool.bench.frappe-dependencies" + else + if [[ "$app_name" == "${{ github.event.repository.name }}" ]]; then + git -C "$app" fetch --depth 1 origin $head_ref:$head_ref + if git -C "$app" checkout --quiet --force $head_ref; then + echo "Checked out $head_ref successfully for $app" + else + echo "Failed to checkout $ref for $app" >&2 + return 1 + fi + else + git -C "$app" fetch --depth 1 origin $base_ref:$base_ref + if git -C "$app" checkout --quiet --force $base_ref; then + echo "Checked out $base_ref successfully for $app" + else + echo "Failed to checkout $base_ref for $app" >&2 + return 1 + fi + fi + fi + done - bench --site test_site migrate - } + # Resetup env and install apps + if pgrep honcho > /dev/null; then + echo "Stopping honcho process..." + pgrep honcho | xargs kill + sleep 3 + fi - update_to_version 14 + echo "Setting up environment..." + if rm -rf ${GITHUB_WORKSPACE}/env && python -m venv ${GITHUB_WORKSPACE}/env; then + source ${GITHUB_WORKSPACE}/env/bin/activate + pip install --quiet --upgrade pip + # pip install --quiet frappe-bench + # revert after merge: https://github.com/frappe/bench/pull/1600 + pip install --quiet git+https://github.com/blaggacao/bench.git@feat/add-direct-config-module-calling + echo "Environment setup completed." + else + echo "Environment setup failed." >&2 + return 1 + fi - - name: Update to v15 - run: | - cd ~/frappe-bench/apps/frappe/ - function update_to_version() { - version=$1 + echo "Installing apps..." + for app in ${GITHUB_WORKSPACE}/apps/*/; do + if pip install --upgrade -e "$app"; then + echo "Installed $app successfully." + else + echo "Failed to install $app." >&2 + return 1 + fi + done - branch_name="version-$version-hotfix" - echo "Updating to v$version" - git fetch --depth 1 upstream $branch_name:$branch_name - git checkout -q -f $branch_name + echo "Starting bench..." + bench start &>> ${GITHUB_WORKSPACE}/bench_start.log & - pgrep honcho | xargs kill - sleep 3 - rm -rf ~/frappe-bench/env - bench -v setup env - bench start &>> ~/frappe-bench/bench_start.log & + echo "Running migrations on test_site..." + if bench --site test_site migrate; then + echo "Migration completed successfully." + else + echo "Migration failed." >&2 + return 1 + fi - bench --site test_site migrate + echo "Update to version ${version:-$base_ref} completed." } + + # Save this script into a file for later use. + declare -f update_to_version > "$RUNNER_TEMP/migrate" + + - name: Update to v14 + run: | + source $RUNNER_TEMP/migrate + update_to_version 14 + exit $? + + - name: Update to v15 + run: | + source $RUNNER_TEMP/migrate update_to_version 15 + exit $? - name: Update to last commit run: | - cd ~/frappe-bench/apps/frappe/ - echo "Updating to last commit" - pgrep honcho | xargs kill - sleep 3 - rm -rf ~/frappe-bench/env - git checkout -q -f "$GITHUB_SHA" - bench -v setup env - bench start &>> ~/frappe-bench/bench_start.log & - bench --site test_site migrate + source $RUNNER_TEMP/migrate + update_to_version + exit $? bench --site test_site execute frappe.tests.utils.check_orpahned_doctypes + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + if: ${{ failure() && contains( github.event.pull_request.labels.*.name, 'debug-gha') }} + - name: Show bench output if: ${{ always() }} run: | - cd ~/frappe-bench cat bench_start.log || true cd logs - for f in ./*.log*; do + for f in ${GITHUB_WORKSPACE}/*.log*; do echo "Printing log: $f"; cat $f done diff --git a/.github/workflows/_base-server-tests.yml b/.github/workflows/_base-server-tests.yml index b54804b2f0c6..718689ac55a9 100644 --- a/.github/workflows/_base-server-tests.yml +++ b/.github/workflows/_base-server-tests.yml @@ -59,7 +59,7 @@ jobs: # noisy 3rd party library warnings PYTHONWARNINGS: "module,ignore:::babel.messages.extract" DB_ROOT_PASSWORD: db_root - COVERAGE_RCFILE: ~/frappe-bench/apps/frappe/.coveragerc + COVERAGE_RCFILE: ./apps/frappe/.coveragerc strategy: fail-fast: false @@ -97,7 +97,6 @@ jobs: python-version: ${{ inputs.python-version }} node-version: ${{ inputs.node-version }} disable-socketio: true - disable-redis-socketio: true db-root-password: ${{ env.DB_ROOT_PASSWORD }} db: ${{ matrix.db }} env: @@ -105,7 +104,7 @@ jobs: - name: Run Tests run: | - cd ~/frappe-bench || exit + source ${GITHUB_WORKSPACE}/env/bin/activate bench --site test_site \ run-parallel-tests \ --app "${{ github.event.repository.name }}" \ @@ -142,7 +141,7 @@ jobs: if: inputs.enable-coverage with: name: coverage-${{ matrix.db }}-${{ matrix.index }} - path: ~/frappe-bench/sites/*-coverage*.xml + path: ./sites/*-coverage*.xml - name: Setup tmate session uses: mxschmitt/action-tmate@v3 @@ -151,10 +150,9 @@ jobs: - name: Show bench output if: ${{ always() }} run: | - cd ~/frappe-bench || exit cat bench_start.log || true cd logs - for f in ./*.log*; do + for f in ${GITHUB_WORKSPACE}/*.log*; do echo "Printing log: $f"; cat $f done diff --git a/.github/workflows/_base-ui-tests.yml b/.github/workflows/_base-ui-tests.yml index 0d9161f85358..03a240b78883 100644 --- a/.github/workflows/_base-ui-tests.yml +++ b/.github/workflows/_base-ui-tests.yml @@ -48,7 +48,7 @@ jobs: # noisy 3rd party library warnings PYTHONWARNINGS: "ignore" DB_ROOT_PASSWORD: db_root - COVERAGE_RCFILE: ~/frappe-bench/apps/frappe/.coveragerc + COVERAGE_RCFILE: ./apps/frappe/.coveragerc strategy: fail-fast: false @@ -76,7 +76,7 @@ jobs: - name: Verify yarn.lock run: | - cd ~/frappe-bench/apps/${{ github.event.repository.name }} + cd ${GITHUB_WORKSPACE}/apps/${{ github.event.repository.name }} git diff --exit-code yarn.lock - name: Cache cypress binary @@ -87,7 +87,7 @@ jobs: - name: Instrument Source Code run: | - cd ~/frappe-bench/apps/${{ github.event.repository.name }} + cd ${GITHUB_WORKSPACE}/apps/${{ github.event.repository.name }} npx nyc instrument \ -x '${{ github.event.repository.name }}/public/dist/**' \ -x '${{ github.event.repository.name }}/public/js/lib/**' \ @@ -95,18 +95,18 @@ jobs: - name: Build run: | - cd ~/frappe-bench/ + source ${GITHUB_WORKSPACE}/env/bin/activate bench build --apps ${{ github.event.repository.name }} - name: Site Setup run: | - cd ~/frappe-bench/ + source ${GITHUB_WORKSPACE}/env/bin/activate bench --site test_site execute frappe.utils.install.complete_setup_wizard bench --site test_site execute frappe.tests.ui_test_helpers.create_test_user - name: Run Tests run: | - cd ~/frappe-bench/ + source ${GITHUB_WORKSPACE}/env/bin/activate bench --site test_site \ run-ui-tests ${{ github.event.repository.name }} \ --with-coverage \ @@ -121,29 +121,28 @@ jobs: run: | ps -ef | grep "[f]rappe serve" | awk '{print $2}' | xargs kill -s SIGINT sleep 5 - ( tail -f ~/frappe-bench/sites/*-coverage*.xml & ) | grep -q "\/coverage" + ( tail -f ${GITHUB_WORKSPACE}/sites/*-coverage*.xml & ) | grep -q "\/coverage" - name: Upload JS coverage data uses: actions/upload-artifact@v3 if: inputs.enable-coverage with: name: coverage-js-${{ matrix.index }} - path: ~/frappe-bench/apps/${{ github.event.repository.name }}/.cypress-coverage/clover.xml + path: ./apps/${{ github.event.repository.name }}/.cypress-coverage/clover.xml - name: Upload coverage data uses: actions/upload-artifact@v3 if: inputs.enable-coverage with: name: coverage-py-${{ matrix.index }} - path: ~/frappe-bench/sites/*-coverage*.xml + path: ./sites/*-coverage*.xml - name: Show bench output if: ${{ always() }} run: | - cd ~/frappe-bench || exit cat bench_start.log || true cd logs - for f in ./*.log*; do + for f in ${GITHUB_WORKSPACE}/*.log*; do echo "Printing log: $f"; cat $f done diff --git a/pyproject.toml b/pyproject.toml index 26522772144e..6a2560494312 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -149,6 +149,9 @@ responses = "==0.23.1" freezegun = "~=1.2.2" pdbpp = "~=0.10.3" +[tool.frappe-ci.setup] +migration-db = "https://frappeframework.com/files/v13-frappe.sql.gz" + [tool.ruff] line-length = 110 target-version = "py310"