Refactor shell scripts

.github/actions/cloc-repository/action.yaml → .github/actions/create-lines-of-code-report/action.yaml

@@ -26,16 +26,16 @@ runs:
       shell: bash
       run: |
         export BUILD_DATETIME=${{ inputs.build_datetime }}
-        ./scripts/reports/cloc-repository.sh
+        ./scripts/reports/create-lines-of-code-report.sh
     - name: "Compress CLOC report"
       shell: bash
-      run: zip cloc-report.json.zip cloc-report.json
+      run: zip lines-of-code-report.json.zip lines-of-code-report.json
     - name: "Upload CLOC report as an artefact"
       if: ${{ !env.ACT }}
       uses: actions/upload-artifact@v3
       with:
-        name: cloc-report.json.zip
-        path: ./cloc-report.json.zip
+        name: lines-of-code-report.json.zip
+        path: ./lines-of-code-report.json.zip
         retention-days: 21
     - name: "Check prerequisites for sending the report"
       shell: bash
@@ -53,5 +53,5 @@ runs:
       if: steps.check.outputs.secrets_exist == 'true'
       run: |
         aws s3 cp \
-          ./cloc-report.json.zip \
-          ${{ inputs.idp_aws_report_upload_bucket_endpoint }}/${{ inputs.build_timestamp }}-cloc-report.json.zip
+          ./lines-of-code-report.json.zip \
+          ${{ inputs.idp_aws_report_upload_bucket_endpoint }}/${{ inputs.build_timestamp }}-lines-of-code-report.json.zip

.github/actions/scan-dependencies/action.yaml

@@ -26,7 +26,7 @@ runs:
       shell: bash
       run: |
         export BUILD_DATETIME=${{ inputs.build_datetime }}
-        ./scripts/reports/generate-sbom.sh
+        ./scripts/reports/create-sbom-report.sh
     - name: "Compress SBOM report"
       shell: bash
       run: zip sbom-repository-report.json.zip sbom-repository-report.json

.github/workflows/stage-1-commit.yaml

@@ -75,7 +75,7 @@ jobs:
         uses: actions/checkout@v4
       - name: "Lint Terraform"
         uses: ./.github/actions/lint-terraform
-  cloc-repository:
+  count-lines-of-code:
     name: "Count lines of code"
     runs-on: ubuntu-latest
     permissions:
@@ -86,7 +86,7 @@ jobs:
       - name: "Checkout code"
         uses: actions/checkout@v4
       - name: "Count lines of code"
-        uses: ./.github/actions/cloc-repository
+        uses: ./.github/actions/create-lines-of-code-report
         with:
           build_datetime: "${{ inputs.build_datetime }}"
           build_timestamp: "${{ inputs.build_timestamp }}"

.tool-versions

@@ -7,9 +7,14 @@ pre-commit 3.4.0
 # The section below is reserved for Docker image versions.
 
 # TODO: Move this section - consider using a different file for the repository template dependencies.
+# docker/ghcr.io/anchore/grype v0.69.1@sha256:d41fcb371d0af59f311e72123dff46900ebd6d0482391b5a830853ee4f9d1a76 # SEE: https://github.com/anchore/grype/pkgs/container/grype
+# docker/ghcr.io/anchore/syft v0.92.0@sha256:63c60f0a21efb13e80aa1359ab243e49213b6cc2d7e0f8179da38e6913b997e0 # SEE: https://github.com/anchore/syft/pkgs/container/syft
 # docker/ghcr.io/gitleaks/gitleaks v8.18.0@sha256:fd2b5cab12b563d2cc538b14631764a1c25577780e3b7dba71657d58da45d9d9 # SEE: https://github.com/gitleaks/gitleaks/pkgs/container/gitleaks
+# docker/ghcr.io/igorshubovych/markdownlint-cli v0.37.0@sha256:fb3e79946fce78e1cde84d6798c6c2a55f2de11fc16606a40d49411e281d950d # SEE: https://github.com/igorshubovych/markdownlint-cli/pkgs/container/markdownlint-cli
+# docker/ghcr.io/make-ops-tools/gocloc latest@sha256:6888e62e9ae693c4ebcfed9f1d86c70fd083868acb8815fe44b561b9a73b5032 # SEE: https://github.com/make-ops-tools/gocloc/pkgs/container/gocloc
 # docker/ghcr.io/nhs-england-tools/github-runner-image 20230909-321fd1e-rt@sha256:ce4fd6035dc450a50d3cbafb4986d60e77cb49a71ab60a053bb1b9518139a646 # SEE: https://github.com/nhs-england-tools/github-runner-image/pkgs/container/github-runner-image
 # docker/hadolint/hadolint 2.12.0-alpine@sha256:7dba9a9f1a0350f6d021fb2f6f88900998a4fb0aaf8e4330aa8c38544f04db42 # SEE: https://hub.docker.com/r/hadolint/hadolint/tags
 # docker/hashicorp/terraform 1.5.6@sha256:180a7efa983386a27b43657ed610e9deed9e6c3848d54f9ea9b6cb8a5c8c25f5 # SEE: https://hub.docker.com/r/hashicorp/terraform/tags
 # docker/koalaman/shellcheck latest@sha256:e40388688bae0fcffdddb7e4dea49b900c18933b452add0930654b2dea3e7d5c # SEE: https://hub.docker.com/r/koalaman/shellcheck/tags
+# docker/mstruebing/editorconfig-checker 2.7.1@sha256:dd3ca9ea50ef4518efe9be018d669ef9cf937f6bb5cfe2ef84ff2a620b5ddc24 # SEE: https://hub.docker.com/r/mstruebing/editorconfig-checker/tags
 # docker/sonarsource/sonar-scanner-cli 5.0.1@sha256:494ecc3b5b1ee1625bd377b3905c4284e4f0cc155cff397805a244dee1c7d575 # SEE: https://hub.docker.com/r/sonarsource/sonar-scanner-cli/tags

docs/developer-guides/Bash_and_Make.md

@@ -140,7 +140,7 @@ VERBOSE=1 scripts/shellscript-linter.sh
 
 ### Scripts
 
-Most scripts provided with this repository template can utilise tools installed on your `PATH` if they are available or run them from within a Docker container. To force a script to use Docker, the `FORCE_USE_DOCKER` variable is provided. Here is an example of how to use it:
+Most scripts provided with this repository template can utilise tools installed on your `PATH` if they are available or run them from within a Docker container. To force a script to use Docker, the `FORCE_USE_DOCKER` variable is provided. This feature increases configurability of the development environment, allowing you to use custom tooling by default if present on the command-line path. Here is an example of how to use it:
 
 ```shell
 FORCE_USE_DOCKER=1 scripts/shellscript-linter.sh

docs/user-guides/Scan_dependencies.md

@@ -15,7 +15,7 @@ In modern software development, leveraging third-party dependencies is a common
 
 ## Key files
 
-- [generate-sbom.sh](../../scripts/reports/generate-sbom.sh): A shell script that generates SBOM (Software Bill of Materials)
+- [create-sbom-report.sh](../../scripts/reports/create-sbom-report.sh): A shell script that generates SBOM (Software Bill of Materials)
 - [syft.yaml](../../scripts/config/syft.yaml): A configuration file for the SBOM generator
 - [scan-vulnerabilities.sh](../../scripts/reports/scan-vulnerabilities.sh): A shell script that performs CVE analysis
 - [grype.yaml](../../scripts/config/grype.yaml): A configuration file for the CVE scanner
@@ -41,7 +41,7 @@ You can run and test the process locally on a developer's workstation using the
 SBOM generator
 
 ```shell
-./scripts/reports/generate-sbom.sh
+./scripts/reports/create-sbom-report.sh
 cat sbom-repository-report.json | jq
 ```
 

docs/user-guides/Test_GitHub_Actions_locally.md

@@ -28,7 +28,7 @@ The following command-line tools are expected to be installed:
 Here is an example on how to run a GitHub workflow job:
 
 ```shell
-$ make runner-act workflow="stage-1-commit" job="cloc-repository"
+$ make runner-act workflow="stage-1-commit" job="create-lines-of-code-report"
 
 [Commit stage/Count lines of code] 🚀  Start image=ghcr.io/nhs-england-tools/github-runner-image:20230101-abcdef0-rt
 [Commit stage/Count lines of code]   🐳  docker pull image=ghcr.io/nhs-england-tools/github-runner-image:20230101-abcdef0-rt platform=linux/amd64 username= forcePull=false
@@ -42,7 +42,7 @@ $ make runner-act workflow="stage-1-commit" job="cloc-repository"
 [Commit stage/Count lines of code]   ✅  Success - Main Create CLOC report
 [Commit stage/Count lines of code] ⭐ Run Main Compress CLOC report
 [Commit stage/Count lines of code]   🐳  docker exec cmd=[bash --noprofile --norc -e -o pipefail /var/run/act/workflow/1-composite-1.sh] user= workdir=
-| updating: cloc-report.json (deflated 68%)
+| updating: lines-of-code-report.json (deflated 68%)
 [Commit stage/Count lines of code]   ✅  Success - Main Compress CLOC report
 [Commit stage/Count lines of code]   ☁  git clone 'https://github.com/actions/upload-artifact' # ref=v3
 [Commit stage/Count lines of code] ⭐ Run Main Check prerequisites for sending the report

scripts/docker/docker.lib.sh

@@ -27,8 +27,8 @@ function docker-build() {
 
   version-create-effective-file
   _create-effective-dockerfile
-  # The current directory must be changed for the image build script to access
-  # assets that need to be copied
+  # The current directory must be changed for the image build script to access
+  # assets that need to be copied
   current_dir=$(pwd)
   cd "$dir"
   docker build \
@@ -164,7 +164,7 @@ function docker-get-image-version-and-pull() {
   #   digest="sha256:hash"
 
   # Get the image full version from the '.tool-versions' file,
-  # match it by name and version regex, if given.
+  # match it by name and version regex, if given.
   local versions_file="${TOOL_VERSIONS:=$(git rev-parse --show-toplevel)/.tool-versions}"
   local version="latest"
   if [ -f "$versions_file" ]; then

scripts/docker/dockerfile-linter.sh

@@ -8,12 +8,12 @@ set -euo pipefail
 # otherwise it will run it in a Docker container.
 #
 # Usage:
-#   $ ./dockerfile-linter.sh
+#   $ [options] ./dockerfile-linter.sh
 #
 # Arguments (provided as environment variables):
 #   file=Dockerfile         # Path to the Dockerfile to lint, relative to the project's top-level directory, default is './Dockerfile.effective'
-#   VERBOSE=true            # Show all the executed commands, default is 'false'
 #   FORCE_USE_DOCKER=true   # If set to true the command is run in a Docker container, default is 'false'
+#   VERBOSE=true            # Show all the executed commands, default is 'false'
 
 # ==============================================================================
 
@@ -23,16 +23,16 @@ function main() {
 
   local file=${file:-./Dockerfile.effective}
   if command -v hadolint > /dev/null 2>&1 && ! is-arg-true "${FORCE_USE_DOCKER:-false}"; then
-    file="$file" cli-run-hadolint
+    file="$file" run-hadolint-natively
   else
-    file="$file" docker-run-hadolint
+    file="$file" run-hadolint-in-docker
   fi
 }
 
 # Run hadolint natively.
 # Arguments (provided as environment variables):
 #   file=[path to the Dockerfile to lint, relative to the project's top-level directory]
-function cli-run-hadolint() {
+function run-hadolint-natively() {
 
   # shellcheck disable=SC2001
   hadolint "$(echo "$file" | sed "s#$PWD#.#")"
@@ -41,7 +41,7 @@ function cli-run-hadolint() {
 # Run hadolint in a Docker container.
 # Arguments (provided as environment variables):
 #   file=[path to the Dockerfile to lint, relative to the project's top-level directory]
-function docker-run-hadolint() {
+function run-hadolint-in-docker() {
 
   # shellcheck disable=SC1091
   source ./scripts/docker/docker.lib.sh

scripts/docker/tests/docker.test.sh

@@ -73,12 +73,12 @@ function test-docker-build() {
 
 function test-docker-image-from-signature() {
 
-  # Arrange
+  # Arrange
   TOOL_VERSIONS="$(git rev-parse --show-toplevel)/scripts/docker/tests/.tool-versions.test"
   cp Dockerfile Dockerfile.effective
-  # Act
+  # Act
   _replace-image-latest-by-specific-version
-  # Assert
+  # Assert
   grep -q "FROM python:.*-alpine.*@sha256:.*" Dockerfile.effective && return 0 || return 1
 }
 

scripts/githooks/check-file-format.sh

@@ -6,14 +6,19 @@ set -euo pipefail
 
 # Pre-commit git hook to check the EditorConfig rules compliance over changed
 # files. It ensures all non-binary files across the codebase are formatted
-# according to the style defined in the `.editorconfig` file.
+# according to the style defined in the `.editorconfig` file. This is a
+# editorconfig command wrapper. It will run editorconfig natively if it is
+# installed, otherwise it will run it in a Docker container.
 #
 # Usage:
-#   $ check={all,staged-changes,working-tree-changes,branch} [dry_run=true] ./check-file-format.sh
+#   $ [options] ./check-file-format.sh
 #
 # Options:
-#   BRANCH_NAME=other-branch-than-main  # Branch to compare with, default is `origin/main`
-#   VERBOSE=true                        # Show all the executed commands, default is `false`
+#   check={all,staged-changes,working-tree-changes,branch}  # Check mode, default is 'working-tree-changes'
+#   dry_run=true                                            # Do not check, run dry run only, default is 'false'
+#   BRANCH_NAME=other-branch-than-main                      # Branch to compare with, default is `origin/main`
+#   FORCE_USE_DOCKER=true                                   # If set to true the command is run in a Docker container, default is 'false'
+#   VERBOSE=true                                            # Show all the executed commands, default is `false`
 #
 # Exit codes:
 #   0 - All files are formatted correctly
@@ -28,22 +33,13 @@ set -euo pipefail
 #   check=working-tree-changes: check modified, unstaged files. This is the default.
 #   check=branch: check for all changes since branching from $BRANCH_NAME
 #
-# If the `dry_run` parameter is set to a truthy value, the list of
-# files that ec would check is output, with no check done.
-#
 # Notes:
 #   Please make sure to enable EditorConfig linting in your IDE. For the
 #   Visual Studio Code editor it is `editorconfig.editorconfig` that is already
 #   specified in the `./.vscode/extensions.json` file.
 
 # ==============================================================================
 
-# SEE: https://hub.docker.com/r/mstruebing/editorconfig-checker/tags, use the `linux/amd64` os/arch
-image_version=2.7.1@sha256:dd3ca9ea50ef4518efe9be018d669ef9cf937f6bb5cfe2ef84ff2a620b5ddc24
-
-# ==============================================================================
-
-
 function main() {
 
   cd "$(git rev-parse --show-toplevel)"
@@ -70,13 +66,42 @@ function main() {
       ;;
   esac
 
+  if command -v editorconfig > /dev/null 2>&1 && ! is-arg-true "${FORCE_USE_DOCKER:-false}"; then
+    filter="$filter" dry_run_opt="${dry_run_opt:-}" run-editorconfig-natively
+  else
+    filter="$filter" dry_run_opt="${dry_run_opt:-}" run-editorconfig-in-docker
+  fi
+}
+
+# Run editorconfig natively.
+# Arguments (provided as environment variables):
+#   dry_run_opt=[dry run option]
+#   filter=[filter for files to check]
+function run-editorconfig-natively() {
+
+  # shellcheck disable=SC2046,SC2086
+  editorconfig \
+    --exclude '.git/' $dry_run_opt $($filter)
+}
+
+# Run editorconfig in a Docker container.
+# Arguments (provided as environment variables):
+#   dry_run_opt=[dry run option]
+#   filter=[filter for files to check]
+function run-editorconfig-in-docker() {
+
+  # shellcheck disable=SC1091
+  source ./scripts/docker/docker.lib.sh
+
+  # shellcheck disable=SC2155
+  local image=$(name=mstruebing/editorconfig-checker docker-get-image-version-and-pull)
   # We use /dev/null here as a backstop in case there are no files in the state
   # we choose. If the filter comes back empty, adding `/dev/null` onto it has
   # the effect of preventing `ec` from treating "no files" as "all the files".
   docker run --rm --platform linux/amd64 \
     --volume "$PWD":/check \
-    mstruebing/editorconfig-checker:$image_version \
-      sh -c "ec --exclude '.git/' ${dry_run_opt:-} \$($filter) /dev/null"
+    "$image" \
+      sh -c "ec --exclude '.git/' $dry_run_opt \$($filter) /dev/null"
 }
 
 # ==============================================================================

scripts/githooks/check-markdown-format.sh

@@ -5,14 +5,18 @@
 set -euo pipefail
 
 # Pre-commit git hook to check the Markdown file formatting rules compliance
-# over changed files.
+# over changed files. This is a markdownlint command wrapper. It will run
+# markdownlint natively if it is installed, otherwise it will run it in a Docker
+# container.
 #
 # Usage:
-#   $ check={all,staged-changes,working-tree-changes,branch} ./check-markdown-format.sh
+#   $ [options] ./check-markdown-format.sh
 #
 # Options:
-#   BRANCH_NAME=other-branch-than-main  # Branch to compare with, default is `origin/main`
-#   VERBOSE=true                        # Show all the executed commands, default is `false`
+#   check={all,staged-changes,working-tree-changes,branch}  # Check mode, default is 'working-tree-changes'
+#   BRANCH_NAME=other-branch-than-main                      # Branch to compare with, default is `origin/main`
+#   FORCE_USE_DOCKER=true                                   # If set to true the command is run in a Docker container, default is 'false'
+#   VERBOSE=true                                            # Show all the executed commands, default is `false`
 #
 # Exit codes:
 #   0 - All files are formatted correctly
@@ -27,11 +31,6 @@ set -euo pipefail
 
 # ==============================================================================
 
-# SEE: https://github.com/igorshubovych/markdownlint-cli/pkgs/container/markdownlint-cli, use the `linux/amd64` os/arch
-image_version=v0.37.0@sha256:fb3e79946fce78e1cde84d6798c6c2a55f2de11fc16606a40d49411e281d950d
-
-# ==============================================================================
-
 function main() {
 
   cd "$(git rev-parse --show-toplevel)"
@@ -53,15 +52,43 @@ function main() {
   esac
 
   if [ -n "$files" ]; then
-    # shellcheck disable=SC2086
-    docker run --rm --platform linux/amd64 \
-      --volume "$PWD":/workdir \
-      ghcr.io/igorshubovych/markdownlint-cli:$image_version \
-        $files \
-        --config /workdir/scripts/config/markdownlint.yaml
+    if command -v markdownlint > /dev/null 2>&1 && ! is-arg-true "${FORCE_USE_DOCKER:-false}"; then
+      files="$files" run-markdownlint-natively
+    else
+      files="$files" run-markdownlint-in-docker
+    fi
   fi
 }
 
+# Run markdownlint natively.
+# Arguments (provided as environment variables):
+#   files=[files to check]
+function run-markdownlint-natively() {
+
+  # shellcheck disable=SC2086
+  markdownlint \
+    $files \
+    --config "$PWD/scripts/config/markdownlint.yaml"
+}
+
+# Run markdownlint in a Docker container.
+# Arguments (provided as environment variables):
+#   files=[files to check]
+function run-markdownlint-in-docker() {
+
+  # shellcheck disable=SC1091
+  source ./scripts/docker/docker.lib.sh
+
+  # shellcheck disable=SC2155
+  local image=$(name=ghcr.io/igorshubovych/markdownlint-cli docker-get-image-version-and-pull)
+  # shellcheck disable=SC2086
+  docker run --rm --platform linux/amd64 \
+    --volume "$PWD":/workdir \
+    "$image" \
+      $files \
+      --config /workdir/scripts/config/markdownlint.yaml
+}
+
 # ==============================================================================
 
 function is-arg-true() {

scripts/githooks/check-terraform-format.sh

@@ -7,12 +7,12 @@ set -euo pipefail
 # Pre-commit git hook to check format Terraform code.
 #
 # Usage:
-#   $ ./check-terraform-format.sh
+#   $ [options] ./check-terraform-format.sh
 #
 # Options:
 #   check_only=true         # Do not format, run check only, default is 'false'
-#   VERBOSE=true            # Show all the executed commands, default is 'false'
 #   FORCE_USE_DOCKER=true   # If set to true the command is run in a Docker container, default is 'false'
+#   VERBOSE=true            # Show all the executed commands, default is 'false'
 
 # ==============================================================================