add template engine functionality to support generic templates

.github/workflows/check-3rd-party.sh

@@ -11,12 +11,13 @@ for line in `sed -ne 's/[[:space:]-]*uses:[[:space:]]*//p' $1 | sed -e 's/\s*#.*
   # - .: local workflows
   # "actions": workflows authored by GitHub
   # "protocol": workflows published in the protocol org
-  if [[ $author == "." || $author == "actions" || $author == "protocol" ]]; then continue; fi
+  # "ipfs": workflows published in the ipfs org
+  if [[ $author == "." || $author == "actions" || $author == "protocol" || $author == "ipfs" ]]; then continue; fi
   version=`echo $line | awk -F@ '{print $2}' | awk '{print $1}'`
   if ! [[ "$version" =~ ^[a-f0-9]{40}$ ]]; then
     status=1
     echo "$FILE includes $line and doesn't use commit hash"
   fi
-done 
+done
 
 exit $status

.github/workflows/copy-workflow.yml

@@ -30,7 +30,7 @@ jobs:
       FILES: ""
       GITHUB_USER: "web3-bot"
       GITHUB_EMAIL: "[email protected]"
-      DEFAULTBRANCH: ""
+      DEFAULT_BRANCH: ""
     name: ${{ matrix.cfg.target }}
     steps:
     - name: Checkout ${{ matrix.cfg.target }}
@@ -48,8 +48,8 @@ jobs:
     - name: determine GitHub default branch
       working-directory: ${{ env.TARGET_REPO_DIR }}
       run: |
-        defaultbranch=$(git remote show origin | awk '/HEAD branch/ {print $NF}')
-        echo "DEFAULTBRANCH=$defaultbranch" >> $GITHUB_ENV
+        default_branch=$(git remote show origin | awk '/HEAD branch/ {print $NF}')
+        echo "DEFAULT_BRANCH=$default_branch" >> $GITHUB_ENV
     - name: git config
       working-directory: ${{ env.TARGET_REPO_DIR }}
       run: |
@@ -60,7 +60,8 @@ jobs:
       # See https://github.com/protocol/.github/blob/38135c75e47839623bf9b2748275d8c6167a8fa8/.github/workflows/copy-workflow.yml#L163-L168 for an example, how we used to make use of this.
       run: |
         files=${{ toJson(toJson(matrix.cfg.files)) }}
-        files=$(echo -e "$files" | jq -c '.')
+        extra_files=${{ toJson(toJson(matrix.cfg.extra_files)) }}
+        files=$(echo -e "$files" "$extra_files" | jq -nc '[inputs] | add')
         echo "FILES=$files" >> $GITHUB_ENV
     - name: is initial workflow deployment
       # INITIAL_WORKFLOW_DEPLOYMENT=1 iff none of the files in the target repository exist yet
@@ -106,13 +107,25 @@ jobs:
       uses: ./template-repo/.github/actions/copy-workflow-versioning
     - name: Add files
       run: |
+        config=${{ toJson(toJson(matrix.cfg)) }}
+        config=$(echo -e "$config" | jq -c '.')
+        github="{\"default_branch\":\"${{ env.DEFAULT_BRANCH }}\"}"
+        context="{\"config\":$config,\"github\":$github}"
+        # matches trimmed string between template context delimiters
+        regexp='\$\{\{\{\s*(.*?)\s*\}\}\}'
+        # replacement is a perl script that:
+        # 1. prepends . to the matched string to create a filter
+        # 2. escapes single quotes by replacing ' with '"'"' in filter
+        # 3. prints context escaped in the same manner
+        # 4. applies jq with filter to the stdout
+        replacement="\$filter = \".\$1\"; \$filter =~ s/'/'\"'\"'/g; \`echo '${context//\'/\'\"\'\"\'}' | jq -jc '\$filter'\`"
         for f in $(jq -r '.[]' <<< ${{ toJson(env.FILES) }}); do
           echo -e "\nProcessing $f."
           # add DO NOT EDIT header
           tmp=$(mktemp)
           cat $TEMPLATE_REPO_DIR/$TEMPLATE_DIR/header.yml $TEMPLATE_REPO_DIR/$TEMPLATE_DIR/$f > $tmp
-          # replace $default-branch with this repo's GitHub default branch
-          sed -i "s:\$default-branch:${{ env.DEFAULTBRANCH }}:g" $tmp
+          # replace template contexts with values from the JSON context object
+          perl -pi -e "s#$regexp#$replacement#ge" $tmp
           mv $tmp $TEMPLATE_REPO_DIR/$TEMPLATE_DIR/$f
           # create commit, if necessary
           commit_msg=""

.github/workflows/sync-release-assets.yml

@@ -0,0 +1,139 @@
+name: Sync github release assets with dist.ipfs.io
+on:
+  workflow_call:
+    inputs:
+      dist:
+        required: true
+        type: string
+
+concurrency:
+  group: release-assets-dist-sync
+  cancel-in-progress: true
+
+jobs:
+  sync-github-and-dist-ipfs-io:
+    runs-on: "ubuntu-latest"
+    steps:
+      - uses: ipfs/download-ipfs-distribution-action@v1
+      - uses: ipfs/start-ipfs-daemon-action@v1
+        with:
+          args: --init --init-profile=flatfs,server --enable-gc=false
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 14
+      - name: Sync the latest 5 github releases
+        uses: actions/github-script@v4
+        with:
+          script: |
+            const fs = require('fs').promises
+            const max_synced = 5
+
+            // fetch github releases
+            resp = await github.repos.listReleases({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              page: 1,
+              per_page: max_synced
+            })
+            const release_assets = [];
+            num_synced = 0;
+            for (const release of resp.data) {
+              console.log("checking release tagged", release.tag_name)
+              if (release.draft || release.prerelease) {
+                console.log("skipping draft/prerelease tagged", release.tag_name)
+                continue
+              }
+              if (num_synced > max_synced) {
+                console.log("done: synced", max_synced, "latest releases")
+                break;
+              }
+              num_synced += 1
+
+              const github_assets = new Set()
+              for (const asset of release.assets) {
+                github_assets.add(asset.name)
+              }
+
+              // fetch asset info from dist.ipfs.io
+              p = '/ipns/dist.ipfs.io/${{ inputs.dist }}/' + release.tag_name
+              let stdout = ''
+              const options = {}
+              options.listeners = {
+                stdout: (data) => {
+                  stdout += data.toString();
+                }
+              }
+              await exec.exec('ipfs', ['ls', p], options)
+
+              const dist_assets = new Set()
+              const missing_files = []
+              for (const raw_line of stdout.split("\n")) {
+                line = raw_line.trim();
+                if (line.length != 0) {
+                  file = line.split(/(\s+)/).filter( function(e) { return e.trim().length > 0; } )[2]
+                  dist_assets.add(file)
+                  if (!github_assets.has(file)) {
+                    missing_files.push(file)
+                  }
+                }
+              }
+
+              // if dist.ipfs.io has files not found in github, copy them over
+              for (const file of missing_files) {
+                file_sha = file + ".sha512"
+                file_cid = file + ".cid"
+
+                // skip files that don't have .cid and .sha512 checksum files
+                if (!dist_assets.has(file_sha) || !dist_assets.has(file_cid)) {
+                  if (!file.endsWith('.cid') && !file.endsWith('.sha512')) { // silent skip of .sha512.sha512 :)
+                    console.log(`skipping "${file}" as dist.ipfs.io does not provide .cid and .sha512 checksum files for it`)
+                  }
+                  continue
+                }
+
+                console.log("fetching", file, "from dist.ipfs.io")
+                await exec.exec('ipfs', ['get', p + '/' + file])
+                await exec.exec('ipfs', ['get', p + '/' + file_sha])
+                await exec.exec('ipfs', ['get', p + '/' + file_cid])
+                console.log("verifying contents of", file)
+
+                // compute sha512 output for file
+                let sha_stdout = ''
+                const sha_options = {}
+                sha_options.listeners = {
+                  stdout: (data) => {
+                    sha_stdout += data.toString();
+                  }
+                }
+                await exec.exec('sha512sum', [file], sha_options)
+                // read expected sha512 output
+                const sha_data = await fs.readFile(file_sha, "utf8")
+                const digest = (s) => s.split(' ').shift()
+                if (digest(sha_data) != digest(sha_stdout)) {
+                  console.log(`${file}.sha512: ${sha_data}`)
+                  console.log(`sha512sum ${file}: ${sha_stdout}`)
+                  throw "checksum verification failed for " + file
+                }
+
+                console.log("uploading", file, "to github release", release.tag_name)
+                const uploadReleaseAsset = async (file) => github.repos.uploadReleaseAsset({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  release_id: release.id,
+                  headers: {
+                    "content-type": "application/octet-stream",
+                    "content-length": `${(await fs.stat(file)).size}`
+                  },
+                  name: file,
+                  data: await fs.readFile(file)
+                })
+                await uploadReleaseAsset(file)
+                await uploadReleaseAsset(file_sha)
+                await uploadReleaseAsset(file_cid)
+
+              }
+              // summary of assets on both sides
+              release_assets.push({ tag: release.tag_name, github_assets, dist_assets })
+            }
+            console.log(release_assets)
+            return release_assets

configs/README.md

@@ -6,6 +6,7 @@ This directory contains config files used for workflow dispatch.
 | --- | --- |
 | go | repositories containing Go code |
 | testing | repositories used for testing unified CI workflows |
+| custom | repositories not ready for full unified CI |
 
 ## Adding new repository to existing config file
 

configs/custom.json

@@ -0,0 +1,15 @@
+{
+  "defaults": {
+    "files": []
+  },
+  "repositories": [
+    {
+      "target": "ipfs/go-ipfs",
+      "files": [
+        ".github/workflows/automerge.yml",
+        ".github/workflows/sync-release-assets.yml"
+      ],
+      "dist": "go-ipfs"
+    }
+  ]
+}

configs/go.json

@@ -146,7 +146,13 @@
     { "target": "libp2p/go-libp2p-quic-transport" },
     { "target": "libp2p/go-libp2p-raft" },
     { "target": "libp2p/go-libp2p-record" },
-    { "target": "libp2p/go-libp2p-relay-daemon" },
+    {
+      "target": "libp2p/go-libp2p-relay-daemon",
+      "extra_files": [
+        ".github/workflows/sync-release-assets.yml"
+      ],
+      "dist": "libp2p-relay-daemon"
+    },
     { "target": "libp2p/go-libp2p-resource-manager" },
     { "target": "libp2p/go-libp2p-routing-helpers" },
     { "target": "libp2p/go-libp2p-swarm" },

templates/.github/workflows/sync-release-assets.yml

@@ -0,0 +1,11 @@
+name: Sync github release assets with dist.ipfs.io
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  sync-github-and-dist-ipfs-io:
+    uses: protocol/.github/.github/workflows/sync-release-assets.yml@master
+    with:
+      dist: ${{{ config.dist }}}

templates/README.md

@@ -0,0 +1,83 @@
+# Templates
+
+This directory contains template files that can be expanded and copied over to the configured repositories.
+
+## Header
+
+The [header](header.yml) is prepended to all the files before they are copied to the target repositories.
+
+## Contexts
+
+You can access context information during template expansion (before copy is performed).
+
+### About contexts
+
+Contexts are a way to access information about the repository configuration object and repository GitHub settings. Contexts use the following syntax:
+
+```
+${{{ <context> }}}
+```
+
+| Context name | Type | Description |
+| --- | --- | --- |
+| `config` | `object` | Configuration object because of which the file is being copied. For more information, see [config context](#config-context) |
+| `github` | `object` | Information about the target repository the file is being copied to. For more information, see [github context](#github-context) |
+
+#### `config` context
+
+The `config` context is the configuration object because of which the file is being copied.
+
+| Property name | Type | Always present | Description |
+| --- | --- | --- | --- |
+| `config` | `object` | `true` | The top-level context. | true |
+| `config.files` | `array` | `true` | The files that are being copied. |
+| `config.extra_files` | `array` | `false` | The additional files that are also being copied. |
+| `config.target` | `string` | `true` | The name of the target repository in `{owner}/{repo}` format. |
+| `config.deploy_go` | `boolean` | `false` | Flag controling if Go specific setup should be applied to the repository. |
+| `config.deploy_versioning` | `boolean` | `false` | Flag controling if `versions.json` file should be deployed to the repository. |
+| `config.dist` | `string` | `false` | The name of the distribution built from the repository. |
+
+#### `github` context
+
+The `github` context contains information about the target repository the file is being copied to.
+
+| Property name | Type | Always present | Description |
+| --- | --- | --- | --- |
+| `github` | `object` | `true` | The top-level context. |
+| `github.default_branch` | `string` | `true` | The name of the default branch of the target repository. |
+
+### Examples
+
+#### Context container
+
+```json
+{
+    "config": {
+        "example": {
+            "greeting": "Hello"
+        },
+        "files": [],
+        "extra_files": [".github/workflows/example.yml"],
+        "target": "protocol/.github-test-target"
+    },
+    "github": {
+        "default_branch": "master"
+    }
+}
+```
+
+#### Template expanding context
+
+```yaml
+name: Hello
+on:
+  push:
+    branches:
+      - ${{{ github.default_branch }}}
+jobs:
+  echo:
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo ${{{ config.example.greeting }}}
+        shell: bash
+```