Skip to content

Commit

Permalink
[CI] Add GH workflow to run dev build using submodules at head (#5561)
Browse files Browse the repository at this point in the history
  • Loading branch information
chalin authored Nov 8, 2024
1 parent 6a6ce10 commit 8a81b41
Show file tree
Hide file tree
Showing 3 changed files with 111 additions and 13 deletions.
88 changes: 88 additions & 0 deletions .github/workflows/build-dev.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
name: Dev build

on:
workflow_dispatch:
inputs:
submodule_path_regex:
description:
Regex of submodule paths to updated to HEAD before building.
default: content-modules
type: string

jobs:
build-and-test:
name: BUILD and CHECK LINKS
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4

- name: Create NPM cache-hash input file
run: |
mkdir -p tmp
jq '{devDependencies, dependencies, engines, gitHubActionCacheKey}' package.json > tmp/package-ci.json
- name: Create and use reduced-dependencies package.json
run: |
jq '.devDependencies |= with_entries(
select(.key | test("prefix|hugo|css"))
)
| del(.dependencies, .optionalDependencies)' \
package.json > tmp/package-min.json
cp tmp/package-min.json package.json
- uses: actions/setup-node@v4
with:
node-version-file: .nvmrc
cache: npm
cache-dependency-path: tmp/package-ci.json

- run: npm install --omit=optional

- name: Fetch all submodules at HEAD
run: npm run update:submodule

- name:
Revert to pinned versions of submodules except for
submodule_path_regex
run: npm run _sync -- ${{ inputs.submodule_path_regex }}

- run: npm run log:test-and-fix
continue-on-error: true
env:
GET: no

- uses: actions/upload-artifact@v4
with:
name: build-log-etc
path: |
tmp/build-log.txt
tmp/package*.json
static/refcache.json
check-refcache:
name: REFCACHE updates?
needs: build-and-test
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/download-artifact@v4
with: { name: build-log-etc }
- name: Fail when refcache contains entries with HTTP status 4XX
run: |
if grep -B 1 -e '"StatusCode": 4' static/refcache.json; then
echo "Run 'npx gulp prune' to remove 4xx entries from the refcache"
exit 1
fi
- name: Does the refcache need updating?
run: npm run diff:fail

check-build-log-for-issues:
name: WARNINGS in build log?
needs: build-and-test
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/download-artifact@v4
with: { name: build-log-etc }
- run: cat tmp/build-log.txt
- run: scripts/check-build-log.sh
23 changes: 10 additions & 13 deletions package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
{
"Notes": [
"The 'all' runs _all_ named scripts in sequence, even if one fails; and exits with failure in that case."
"The 'all' runs _all_ named scripts in sequence, even if one fails; and exits with failure in that case.",
"check:links actually has the side-effect of 'fixing' the refcache; this is why test-and-fix uses _fix:most"
],
"scripts": {
"__check:links": "make --keep-going check-links",
Expand All @@ -20,6 +21,7 @@
"_diff:fail": "git diff --name-only --exit-code || exit 1",
"_fail": "exit 1",
"_filename-error": "echo 'ERROR: the following files violate naming conventions; fix using: `npm run fix:filenames`'; echo; npm run -s _ls-bad-filenames; exit 1",
"_fix:most": "npm run all -- $(npm -s run _list:fix:* | grep -Ev 'refcache|submodule')",
"_get:no": "echo SKIPPING get operation",
"_get:submodule:non-lang": "npm run _get:submodule -- content-modules/opentelemetry-specification themes/docsy",
"_get:submodule": "set -x && git submodule update --init ${DEPTH:- --depth 999}",
Expand All @@ -30,13 +32,13 @@
"_list:dict": "grep '^\\s*\"@cspell' package.json | awk -F: '{print $1}' | tr -d '\"'",
"_list:fix:*": "npm run --loglevel=warn | grep -Ee '^\\s*fix:[^:]+$' | grep -v 'fix:all'",
"_ls-bad-filenames": "find assets content static -name '*_*' ! -name '[_.]*'",
"_pin": "scripts/pin-submodules.pl",
"_prebuild": "npm run seq -- get:submodule cp:spec",
"_prepare:docsy": "cd themes/docsy && npm install",
"_prettier:any": "npx prettier --ignore-path ''",
"_rename-to-kebab-case": "find assets content static -name '*_*' ! -name '[_.]*' -exec sh -c 'mv \"$1\" \"${1//_/-}\"' _ {} \\;",
"_serve:hugo": "hugo server --buildDrafts --minify",
"_serve:netlify": "netlify dev -c \"npm run _serve:hugo -- --renderToMemory\"",
"_sync": "scripts/sync-submodules.pl",
"all": "bash -c 'x=0; for c in \"$@\"; do npm run $c || x=$((x+1)); done; ((!x)) || (echo \"ERROR: some scripts failed!\" && exit 1)' -",
"build:preview": "set -x && npm run _build -- --minify",
"build:production": "npm run _hugo -- --minify",
Expand All @@ -59,7 +61,7 @@
"cp:spec": "scripts/content-modules/cp-pages.sh",
"diff:check": "npm run _diff:check || (echo; echo 'WARNING: the files above have not been committed'; echo)",
"diff:fail": "npm run _diff:check || (echo; echo 'ERROR: the files above have changed. Locally rerun `npm run test-and-fix` and commit changes'; echo; exit 1)",
"fix:all": "npm run seq -- $(npm -s run _list:fix:*)",
"fix:all": "npm run all -- $(npm -s run _list:fix:*)",
"fix:dict": "find content/en layouts -name \"*.md\" -print0 | xargs -0 scripts/normalize-cspell-front-matter.pl",
"fix:filenames": "npm run _rename-to-kebab-case",
"fix:format": "npm run format",
Expand All @@ -69,12 +71,13 @@
"fix:i18n": "npm run fix:i18n:new",
"fix:markdown": "npm run check:markdown -- --fix",
"fix:refcache": "npm run check:links",
"fix:submodules": "npm run _sync",
"fix:submodules": "npm run update:submodule && npm run _pin",
"fix:text": "npm run check:text -- --fix",
"fix": "npm run fix:all",
"format": "npm run _check:format -- --write && npm run _check:format:ja+zh -- --write",
"get:submodule": "npm run _get:${GET:-submodule} --",
"log:check:links": "npm run check:links | tee tmp/build-log.txt",
"log:test-and-fix": "npm run test-and-fix | tee tmp/build-log.txt",
"make:public": "make public ls-public",
"netlify-build:preview": "npm run seq -- build:preview diff:check",
"netlify-build:production": "npm run seq -- build:production diff:check",
Expand All @@ -93,17 +96,11 @@
"serve:hugo": "npm run _serve:hugo -- --renderToMemory",
"serve:netlify": "npm run _serve:netlify",
"serve": "npm run serve:hugo --",
"sync": "scripts/sync-submodules.pl",
"test-and-fix": "npm run seq -- check fix:dict fix:filenames",
"test-and-fix": "npm run seq -- fix:submodules _fix:most test",
"test": "npm run check",
"update:docsy-dep": "npm install --save-dev autoprefixer@latest postcss-cli@latest",
"update:htmltest-config": "scripts/htmltest-config.sh",
"update:hugo": "npm install --save-dev --save-exact hugo-extended@latest",
"update:hugo+": "npm run update:hugo && npm run update:docsy-dep",
"update:netlify": "npm install --save-optional netlify-cli@latest",
"update:other-pkg": "npm install --save-dev gulp@latest",
"update:pkg:hugo": "npm install --save-dev --save-exact hugo-extended@latest",
"update:pkgs": "npx npm-check-updates -u",
"update:submodule:lang": "npm run seq -- update:submodule _get:submodule:non-lang",
"update:submodule": "set -x && git submodule update --remote ${DEPTH:- --depth 999}"
},
"devDependencies": {
Expand Down Expand Up @@ -157,5 +154,5 @@
"proseWrap": "always",
"singleQuote": true
},
"spelling": "cSpell:ignore docsy elemetry htmltest hugo loglevel netlify nowrap postnetlify prebuild precheck preinstall postbuild postget refcache textlint -"
"spelling": "cSpell:ignore docsy elemetry htmltest hugo loglevel netlify nowrap pkgs postnetlify prebuild precheck preinstall postbuild postget refcache textlint -"
}
13 changes: 13 additions & 0 deletions scripts/sync-submodules.pl → scripts/pin-submodules.pl
Original file line number Diff line number Diff line change
@@ -1,9 +1,17 @@
#!/usr/bin/perl -w
#
# Switch submodules to their pinned versions, all submodules by default. Pass
# submodule-path regex to specify submodules to skip.

use strict;
use warnings;
use FileHandle;

my $submodule_skip_regex = shift @ARGV;

print "Using submodule-path skip regex: $submodule_skip_regex\n\n"
if $submodule_skip_regex;

my $file = '.gitmodules';
my $fh = FileHandle->new($file, 'r') or die "Error opening $file: $!";
my $content = do { local $/; <$fh> };
Expand All @@ -17,6 +25,11 @@
my $submodule_path = $submodules[$i];
my $commit = $submodules[$i + 1];

if ($submodule_skip_regex && $submodule_path =~ /$submodule_skip_regex/) {
printf "Skipping $submodule_path\n";
next;
}

my $command = "cd $submodule_path && git switch --detach $commit";
print "> $command\n";
system($command);
Expand Down

0 comments on commit 8a81b41

Please sign in to comment.