diff --git a/.github/foss_project_collector.py b/.github/foss_project_collector.py new file mode 100644 index 00000000..3a819fcb --- /dev/null +++ b/.github/foss_project_collector.py @@ -0,0 +1,49 @@ +# Copyright 2023 Ericsson AB +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import json +import sys +import glob + + +IGNORED_FOLDER_LIST = [ + "templates", + "__pycache__", + ".pytest_cache", +] + +def main(): + projects_dir = os.getenv("PROJECTS_DIR") + if not projects_dir: + print("[ERROR] PROJECTS_DIR not set!", file=sys.stderr) + sys.exit(1) + + project_list = [] + + for project_folder in glob.glob(os.path.join(projects_dir, "*/")): + project_name = os.path.basename(os.path.normpath(project_folder)) + + if project_name not in IGNORED_FOLDER_LIST: + project_list.append( + {"name": project_name, "folder": project_folder} + ) + + final_matrix_json = json.dumps(project_list) + + print(f"matrix_json={final_matrix_json}") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/foss.yaml b/.github/workflows/foss.yaml index 55481f39..442df830 100644 --- a/.github/workflows/foss.yaml +++ b/.github/workflows/foss.yaml @@ -24,10 +24,38 @@ concurrency: cancel-in-progress: true jobs: - # TODO: Parallelize the running of projects + # Prepares matrix used to generate jobs in foss_{$platform}_test + # This is a best-practice solution detailed in: + # https://docs.github.com/en/actions/how-tos/write-workflows/choose-what-workflows-do/run-job-variations + # This job assumes that every project initializer script is + # in `test/foss/project` and script name must be `init.sh` + # patches must clone their repository into folder: test-proj + prepare_project_matrix: + runs-on: ubuntu-24.04 + name: Collecting Projects + outputs: + project_configurations: ${{ steps.generate_matrix.outputs.matrix_json }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Generate dynamic project matrix + id: generate_matrix + env: + PROJECTS_DIR: "./test/foss/" + run: | + python3 ./.github/foss_project_collector.py >> "$GITHUB_OUTPUT" + shell: bash + foss_ubuntu_test: - name: "Test rules on FOSS projects (ubuntu)" + name: "Test rules on FOSS project: ${{ matrix.project.name }} (Ubuntu)" runs-on: ubuntu-24.04 + needs: prepare_project_matrix + strategy: + fail-fast: false + max-parallel: 2 # limit number of concurrent jobs + matrix: + project: ${{ fromJson(needs.prepare_project_matrix.outputs.project_configurations) }} steps: - name: Checkout repository @@ -36,14 +64,28 @@ jobs: - name: Setup environment uses: ./.github/platform_environment_setup/ubuntu - - name: Run Test On Opensource Projects - working-directory: test - run: python3 -m unittest foss/test_foss.py -vvv + - name: Setup project + working-directory: ${{ matrix.project.folder }} + run: sh init.sh + + - name: Run Bazel CodeChecker + working-directory: ${{ matrix.project.folder }}/test-proj + run: bazel build :codechecker_test + + - name: Run Per-File Bazel CodeChecker + working-directory: ${{ matrix.project.folder }}/test-proj + run: bazel build :per_file_test foss_rhel_test: - name: "Test rules on FOSS projects (RHEL)" + name: "Test rules on FOSS project: ${{ matrix.project.name }} (RHEL)" runs-on: ubuntu-24.04 container: redhat/ubi9:latest + needs: prepare_project_matrix + strategy: + fail-fast: false + max-parallel: 2 # limit number of concurrent jobs + matrix: + project: ${{ fromJson(needs.prepare_project_matrix.outputs.project_configurations) }} steps: - name: Checkout repository @@ -52,6 +94,14 @@ jobs: - name: Setup environment uses: ./.github/platform_environment_setup/rhel9 - - name: Run Test On Opensource Projects - working-directory: test - run: python3 -m unittest foss/test_foss.py -vvv + - name: Setup project + working-directory: ${{ matrix.project.folder }} + run: sh init.sh + + - name: Run Bazel CodeChecker + working-directory: ${{ matrix.project.folder }}/test-proj + run: bazel build :codechecker_test + + - name: Run Per-File Bazel CodeChecker + working-directory: ${{ matrix.project.folder }}/test-proj + run: bazel build :per_file_test