Merge pull request #13987 from rhmdnd/CMP-3540-remove-sdn-assertions-… #412
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sync CaC content to OSCAL content | |
| permissions: | |
| contents: write | |
| pull-requests: read | |
| on: | |
| push: | |
| branches: | |
| - master | |
| jobs: | |
| sync-cac-updates-to-oscal-content: | |
| runs-on: ubuntu-latest | |
| steps: | |
| # Step 1: Set up Python 3 | |
| - name: Set up Python 3 | |
| uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 | |
| with: | |
| python-version: '3.9' | |
| # Step 2: Install Git | |
| - name: Install Git | |
| run: sudo apt-get update && sudo apt-get install -y git | |
| # Step 3: Checkout the CaC repo | |
| - name: Checkout CaC repo | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| repository: ${{ github.repository }} | |
| path: cac-content | |
| - name: Get the commit message and PR number | |
| run: | | |
| cd cac-content | |
| # Get the latest commit message | |
| COMMIT_MSG=$(git log -1 --pretty=%B) | |
| # Extract the PR number from the commit message (if it's a merge commit) | |
| PR_NUMBER=$(echo "$COMMIT_MSG" | grep -oP '#\K\d+') | |
| if [ -n "$PR_NUMBER" ]; then | |
| echo "Found PR number: $PR_NUMBER" | |
| echo "PR_NUMBER=$PR_NUMBER" >> $GITHUB_ENV | |
| echo "SKIP=false" >> $GITHUB_ENV | |
| PR_INFO=$(curl -s "https://api.github.com/repos/${{ github.repository }}/pulls/${PR_NUMBER}") | |
| # Extract PR title from the response | |
| PR_TITLE=$(echo "$PR_INFO" | jq -r .title) | |
| echo "PR Title: $PR_TITLE" | |
| if [[ "$PR_TITLE" == *"Auto-generated PR from OSCAL"* ]]; then | |
| echo "The PR comes from OSCAL content. The task of Sync CaC content to OSCAL will exit." | |
| echo "Skipping further checks." | |
| echo "SKIP=true" >> $GITHUB_ENV | |
| fi | |
| fi | |
| # Step 4: Get the access token for content write permission to OSCAL content | |
| - name: Get GitHub app token | |
| if: ${{ env.SKIP == 'false' }} | |
| uses: actions/create-github-app-token@67018539274d69449ef7c02e8e71183d1719ab42 # v2.1.4 | |
| id: app-token | |
| with: | |
| app-id: ${{ secrets.APP_ID }} | |
| private-key: ${{ secrets.PRIVATE_KEY }} | |
| owner: ${{ github.repository_owner }} | |
| repositories: | | |
| content | |
| oscal-content | |
| # Step 5: Checkout complyscribe and setup the environment | |
| - name: Checkout complyscribe repo | |
| if: ${{ env.SKIP == 'false' }} | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| repository: complytime/complyscribe | |
| path: complyscribe | |
| - name: Setup complyscribe | |
| if: ${{ env.SKIP == 'false' }} | |
| run: | | |
| cd complyscribe && python3 -m venv venv && source venv/bin/activate | |
| python3 -m pip install --no-cache-dir "poetry==1.7.1" | |
| poetry install | |
| # Step 6: Detect the updates of CAC content | |
| - name: Detect files changed by PR | |
| if: ${{ env.SKIP == 'false' }} | |
| id: changed-files | |
| run: | | |
| OWNER="ComplianceAsCode" | |
| REPO="content" | |
| # Fetch all pages of the files for the pull request | |
| url="repos/$OWNER/$REPO/pulls/${{ env.PR_NUMBER }}/files" | |
| response=$(gh api "$url" --paginate) | |
| echo "$response" | jq -r '.[].filename' > filenames.txt | |
| echo "CHANGE_FOUND=false" >> $GITHUB_ENV | |
| source complyscribe/venv/bin/activate | |
| cd cac-content | |
| has_change() { | |
| local file="$1" | |
| local key="$2" | |
| ! python utils/compare_rule_var.py \ | |
| --owner "$OWNER" \ | |
| --repo "$REPO" \ | |
| "${{ env.PR_NUMBER }}" \ | |
| "$file" \ | |
| "$key" | |
| } | |
| while IFS= read -r line; do | |
| # Exclude lines containing 'tests/data/' | |
| if [[ "$line" == *tests/data/* ]]; then | |
| continue | |
| fi | |
| case "$line" in | |
| *controls/*|*.profile*) | |
| echo "$line" >> updated_filenames.txt | |
| ;; | |
| *rule.yml) | |
| if has_change "$line" "title"; then | |
| echo "Change detected: The title in '$line' was updated." | |
| echo "$line" >> updated_filenames.txt | |
| fi | |
| ;; | |
| *.var) | |
| if has_change "$line" "description" || has_change "$line" "options"; then | |
| echo "Change detected: The description or options in '$line' were updated." | |
| echo "$line" >> updated_filenames.txt | |
| fi | |
| ;; | |
| esac | |
| done < ../filenames.txt | |
| if [[ -f updated_filenames.txt ]]; then | |
| echo "Shows updated_filenames:" | |
| cat updated_filenames.txt | |
| echo "CHANGE_FOUND=true" >> $GITHUB_ENV | |
| fi | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| - name: Checkout OSCAL content repo | |
| if: ${{ env.CHANGE_FOUND == 'true' }} | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| repository: ComplianceAsCode/oscal-content | |
| path: oscal-content | |
| token: ${{ steps.app-token.outputs.token }} | |
| fetch-depth: 0 | |
| - name: Set RH_PRODUCTS | |
| if: ${{ env.CHANGE_FOUND == 'true' }} | |
| run: | | |
| echo "RH_PRODUCTS=(rhel8 rhel9 rhel10 ocp4 fedora)" >> $GITHUB_ENV | |
| # Step 7: Get profiles, controls and level mapping | |
| - name: Get profiles, controls and level mapping | |
| if: ${{ env.CHANGE_FOUND == 'true' }} | |
| run: | | |
| cd complyscribe && source venv/bin/activate | |
| RH_PRODUCTS=${{ env.RH_PRODUCTS }} | |
| for product in "${RH_PRODUCTS[@]}"; do | |
| echo "The map for $product..." | |
| map_file=$product"_map.json" | |
| python -W ignore scripts/get_mappings_profile_control_levels.py $product "$GITHUB_WORKSPACE/cac-content" > $map_file 2>&1 | |
| cat $map_file | |
| done | |
| # Step 8: Get product available controls | |
| - name: Get product controls | |
| if: ${{ env.CHANGE_FOUND == 'true' }} | |
| run: | | |
| cd complyscribe && source venv/bin/activate | |
| RH_PRODUCTS=${{ env.RH_PRODUCTS }} | |
| for product in "${RH_PRODUCTS[@]}"; do | |
| echo "All available controls of $product..." | |
| controls_file=$product"_controls" | |
| python -W ignore scripts/get_product_controls.py $product "$GITHUB_WORKSPACE/cac-content" > $controls_file 2>&1 | |
| cat $controls_file | |
| done | |
| # Step 9: Handle the detected updates | |
| # 1. Get the updated controls | |
| # 2. Get the updated profiles | |
| # 3. Get the controls and profiles are impacted by rules and vars | |
| - name: Handle the detected updates | |
| if: ${{ env.CHANGE_FOUND == 'true' }} | |
| run: | | |
| python cac-content/utils/handle_detected_updates.py 'cac-content/updated_filenames.txt' > updates 2>&1 | |
| cd complyscribe && source venv/bin/activate | |
| RH_PRODUCTS=${{ env.RH_PRODUCTS }} | |
| i=0 | |
| while IFS= read -r line; do | |
| i=$((i + 1)) | |
| if [[ $i -eq 1 ]]; then | |
| # 1. Get the updated controls | |
| echo $line > updated_controls | |
| elif [[ $i -eq 2 ]]; then | |
| # 2. Get the updated profiles | |
| profiles=($(echo "$line"| sed "s/{'/{\"/g; s/': '/\":\"/g; s/', '/\",\"/g; s/'}/\"}/g;")) | |
| for profile in "${profiles[@]}"; do | |
| product=$(echo "$profile" | jq -r '.product') | |
| profile_name=$(echo "$profile" | jq -r '.profile_name') | |
| echo $profile_name >> $product"_updated_profiles" | |
| done | |
| elif [[ $i -eq 3 ]]; then | |
| # 3. Get the updated rule and variables, | |
| # then convert them to impacted controls and profiles | |
| rules=($line) | |
| for rule in "${rules[@]}"; do | |
| python -W ignore scripts/get_rule_impacted_files.py rh-products "$GITHUB_WORKSPACE/cac-content" $rule control >> rule_impacted_controls 2>&1 | |
| for product in "${RH_PRODUCTS[@]}"; do | |
| python -W ignore scripts/get_rule_impacted_files.py $product "$GITHUB_WORKSPACE/cac-content" $rule profile >> $product"_rule_impacted_profiles" 2>&1 | |
| done | |
| done | |
| fi | |
| done < "$GITHUB_WORKSPACE"/updates | |
| # Step 10: Check if there is any existing open PR | |
| - name: Check if there is any existing open PR | |
| if: ${{ env.CHANGE_FOUND == 'true' }} | |
| run: | | |
| cd oscal-content | |
| # Use the GitHub CLI to search for an open PR. | |
| # The 'jq' query filters for PRs where the branch name contains "sync_cac_pr". | |
| # We take the first result found. | |
| PR_BRANCH=$(gh pr list --state open --json headRefName --jq '.[] | select(.headRefName | contains("sync_cac_pr")) | .headRefName' | head -n 1) | |
| if [[ -n "$PR_BRANCH" ]]; then | |
| echo "Found matching PR branch: $PR_BRANCH" | |
| # Set the branch name as the PR_BRANCH. | |
| echo "BRANCH_NAME=$PR_BRANCH" >> $GITHUB_ENV | |
| else | |
| echo "No open PR found with 'sync_cac_pr' in the branch name." | |
| BRANCH_NAME="sync_cac_pr${{ env.PR_NUMBER }}" | |
| echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV | |
| fi | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| # Step 11: Check if the OSCAL content branch exists | |
| - name: Check if the OSCAL content branch exists | |
| if: ${{ env.CHANGE_FOUND == 'true' }} | |
| run: | | |
| cd oscal-content | |
| git fetch --all | |
| if git show-ref --verify --quiet refs/remotes/origin/"${{ env.BRANCH_NAME }}"; then | |
| git checkout -b "${{ env.BRANCH_NAME }}" origin/${{ env.BRANCH_NAME }} | |
| else | |
| echo "OSCAL content branch $BRANCH_NAME doesn't exist" | |
| fi | |
| # Get the base commit HASH | |
| base_commit=$(git log -1 --format=%H) | |
| echo "base_commit=$base_commit" >> $GITHUB_ENV | |
| # Step 12: Sync updated controls to OSCAL content | |
| - name: Sync updated controls to OSCAL content | |
| if: ${{ env.CHANGE_FOUND == 'true' }} | |
| run: | | |
| cd complyscribe && source venv/bin/activate | |
| RH_PRODUCTS=${{ env.RH_PRODUCTS }} | |
| # 1.1 Get the updated controls to array | |
| file="updated_controls" | |
| if [ -f "$file" ] && [ -s "$file" ]; then | |
| updated_controls=($(cat "$file")) | |
| # Output all the updated controls in the PR | |
| echo "The updated controls: ${updated_controls[@]}" | |
| fi | |
| # 1.2 Sync the updated controls to OSCAL content | |
| for product in "${RH_PRODUCTS[@]}"; do | |
| for policy_id in "${updated_controls[@]}"; do | |
| # This sync depends on the specific product available controls | |
| available_controls=($(cat $product"_controls")) # Get all the available controls of the product | |
| for pc in "${available_controls[@]}"; do | |
| if [[ "$pc" == "$policy_id" ]]; then | |
| # 1.2.1 Sync the updated controls to OSCAL catalog | |
| poetry run complyscribe sync-cac-content catalog --repo-path ../oscal-content --committer-email "[email protected]" --committer-name "openscap-ci" --branch "${{ env.BRANCH_NAME }}" --cac-content-root "$GITHUB_WORKSPACE/cac-content" --cac-policy-id "$policy_id" --oscal-catalog "$policy_id" | |
| # 1.2.2 Sync the updated controls to OSCAL profile | |
| poetry run complyscribe sync-cac-content profile --repo-path ../oscal-content --committer-email "[email protected]" --committer-name "openscap-ci" --branch "${{ env.BRANCH_NAME }}" --cac-content-root "$GITHUB_WORKSPACE/cac-content" --product "$product" --cac-policy-id "$policy_id" --oscal-catalog "$policy_id" | |
| fi | |
| done | |
| # 1.2.3 Sync the updated controls to OSCAL component-definition | |
| # This sync depends on the control assoicated profile and levels | |
| sh ../cac-content/utils/complyscribe-cli-compd.sh true $policy_id $product ${{ env.BRANCH_NAME }} $GITHUB_WORKSPACE $product"_map.json" | |
| done | |
| done | |
| # Step 13: Sync updated profiles to OSCAL content | |
| - name: Sync updated profiles to OSCAL content | |
| if: ${{ env.CHANGE_FOUND == 'true' }} | |
| run: | | |
| cd complyscribe && source venv/bin/activate | |
| RH_PRODUCTS=${{ env.RH_PRODUCTS }} | |
| pr_number="${{ env.PR_NUMBER }}" | |
| # 1. Get the updated profiles | |
| # 2. Sync the updated profiles to OSCAL profile and component-definition | |
| for product in "${RH_PRODUCTS[@]}"; do | |
| file=$product"_updated_profiles" | |
| if [ -f "$file" ] && [ -s "$file" ]; then | |
| updated_profiles=($(cat "$file" | tr ' ' '\n' | sort | uniq | tr '\n' ' ' | sed 's/ $//')) | |
| echo "The updated profiles for product $product: ${updated_profiles[@]}" | |
| for profile in "${updated_profiles[@]}"; do | |
| # 2.1 Sync CaC profile to OSCAL prifile | |
| while IFS= read -r line; do | |
| map=${line//\'/\"} | |
| policy_id=$(echo "$map" | jq -r '.policy_id') | |
| profile_name=$(echo "$map" | jq -r '.profile_name') | |
| if [[ "$profile" == "$profile_name" ]]; then | |
| poetry run complyscribe sync-cac-content profile --repo-path ../oscal-content --committer-email "[email protected]" --committer-name "openscap-ci" --branch "${{ env.BRANCH_NAME }}" --cac-content-root "$GITHUB_WORKSPACE/cac-content" --product "$product" --cac-policy-id "$policy_id" --oscal-catalog "$policy_id" | |
| fi | |
| done < $product"_map.json" | |
| # 2.2 Sync CaC profile to OSCAL component-definition | |
| sh ../cac-content/utils/complyscribe-cli-compd.sh false $profile $product ${{ env.BRANCH_NAME }} $GITHUB_WORKSPACE $product"_map.json" | |
| done | |
| fi | |
| done | |
| # Step 14: Sync rule updates to OSCAL component-definition | |
| - name: Sync rule updates to OSCAL component-definition | |
| if: ${{ env.CHANGE_FOUND == 'true' }} | |
| run: | | |
| cd complyscribe && source venv/bin/activate | |
| RH_PRODUCTS=${{ env.RH_PRODUCTS }} | |
| pr_number="${{ env.PR_NUMBER }}" | |
| # 1. Get the rule impacted controls | |
| file="rule_impacted_controls" | |
| if [ -f "$file" ] && [ -s "$file" ]; then | |
| rule_impacted_controls=($(cat "$file" | tr ' ' '\n' | sort | uniq | tr '\n' ' ' | sed 's/ $//')) | |
| echo "The rule impacted controls: ${rule_impacted_controls[@]}" | |
| # 2. Sync the rule impacted controls to OSCAL component-definition | |
| for product in "${RH_PRODUCTS[@]}"; do | |
| # Sync CAC controls' updates to OSCAL content | |
| for policy_id in "${rule_impacted_controls[@]}"; do | |
| sh ../cac-content/utils/complyscribe-cli-compd.sh true $policy_id $product ${{ env.BRANCH_NAME }} $GITHUB_WORKSPACE $product"_map.json" | |
| done | |
| done | |
| fi | |
| # 3. Get the rule impacted profiles | |
| for product in "${RH_PRODUCTS[@]}"; do | |
| file=$product"_rule_impacted_profiles" | |
| if [ -f "$file" ] && [ -s "$file" ]; then | |
| rule_impacted_profiles=($(cat $file | tr ' ' '\n' | sort | uniq | tr '\n' ' ' | sed 's/ $//')) | |
| echo "The rule impacted profiles for $product: ${rule_impacted_profiles[@]}" | |
| # 4. Sync the rule impacted profiles to OSCAL component-definition | |
| for profile in "${rule_impacted_profiles[@]}"; do | |
| sh ../cac-content/utils/complyscribe-cli-compd.sh false $profile $product ${{ env.BRANCH_NAME }} $GITHUB_WORKSPACE $product"_map.json" | |
| done | |
| fi | |
| done | |
| # Step 15: Squash multiple commits for each run | |
| - name: Squash multiple commits | |
| if: ${{ env.CHANGE_FOUND == 'true' }} | |
| run: | | |
| cd oscal-content | |
| git config user.name "openscap-ci" | |
| git config user.email "[email protected]" | |
| echo "PR_SKIP=false" >> $GITHUB_ENV | |
| if [ "$(git branch --show-current)" == "${{ env.BRANCH_NAME }}" ]; then | |
| SQUASH_COUNT=$(git rev-list --count ${{ env.base_commit }}..HEAD) | |
| echo "SQUASH_COUNT=$SQUASH_COUNT" >> $GITHUB_ENV | |
| if [[ "$SQUASH_COUNT" -eq 0 ]]; then | |
| echo "No commit from the CAC PR ${{ env.PR_NUMBER }}." | |
| echo "PR_SKIP=true" >> $GITHUB_ENV | |
| elif [[ "$SQUASH_COUNT" -eq 1 ]]; then | |
| echo "::notice::Branch has 1 commit. No squashing needed." | |
| else | |
| # Call the squash script using the commit count | |
| $GITHUB_WORKSPACE/complyscribe/scripts/squash.sh "$SQUASH_COUNT" | |
| fi | |
| else | |
| echo "PR_SKIP=true" >> $GITHUB_ENV | |
| echo "No branch ${{ env.BRANCH_NAME }}. Skipping squash and create PR." | |
| fi | |
| shell: bash | |
| env: | |
| GH_TOKEN: ${{ env.INSTALLATION_TOKEN }} | |
| # Step 16: Create PR or update PR in OSCAL content | |
| - name: Create a Pull Request in OSCAL content | |
| if: ${{ env.PR_SKIP == 'false' }} | |
| run: | | |
| cd oscal-content | |
| OWNER="ComplianceAsCode" | |
| REPO="oscal-content" | |
| CAC_PR_URL="https://github.com/$OWNER/content/pull/${{ env.PR_NUMBER }}" | |
| commit=$(git log -1 --format=%H) | |
| PR_BODY="This is an auto-generated commit $commit from CAC PR [${{ env.PR_NUMBER }}]("$CAC_PR_URL")" | |
| if [[ "$(git branch --show-current)" == "${{ env.BRANCH_NAME }}" ]]; then | |
| if [ "${{ env.SQUASH_COUNT }}" -eq 0 ]; then | |
| echo "No commits from the CAC PR ${{ env.PR_NUMBER }}. Skipping PR creation." | |
| else | |
| # Check if the PR exists | |
| PR_EXISTS=$(gh pr list --repo $OWNER/$REPO \ | |
| --head $BRANCH_NAME --state open --json id \ | |
| | jq length) | |
| if [ "$PR_EXISTS" -gt 0 ]; then | |
| echo "PR ${{ env.BRANCH_NAME }} already exists. Skipping PR creation." | |
| echo "Add a comment for the CAC PR ${{ env.PR_NUMBER }}." | |
| gh pr comment ${{ env.BRANCH_NAME }} --body "${PR_BODY}" | |
| else | |
| echo "Creating PR for new branch: ${{ env.BRANCH_NAME }}" | |
| gh pr create --repo $OWNER/$REPO \ | |
| --title "Auto-generated PR from CAC ${{ env.PR_NUMBER }}" \ | |
| --head "${{ env.BRANCH_NAME }}" \ | |
| --base "main" \ | |
| --body "${PR_BODY}" | |
| fi | |
| fi | |
| else | |
| echo "No branch ${{ env.BRANCH_NAME }}. Skipping PR creation." | |
| fi | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} |