ci: publish: Move to release tag triggered publishing (#611)

* Move to release tag triggered publishing

I still prefer my old GitHub UI triggered release. It feels much less
complicated to me. But this project is part of clj-commons, and here
I defer to the larger team's preference for release triggered publishing.

I brought over this work from other projects, mostly rewrite-clj and
pomegranate. These incude:

- using neil to track/bump/store version in deps.edn
- automatically including date in changelog release headers

One refinement for Etaoin is avoiding the duplicate test run on
Publish. The release tag triggered strategy includes a commit and tag
from the person doing the publish. This commit triggers the normal
Test workflow but the Publish workflow also triggers the Test workflow
to ensure all is good before releasing. To avoid the duplicate test run,
I've added a new `check-for-skip` job in the Test workflow to skip tests
when appropriate. I'd love to skip the independent Test workflow on Publish
entirely, but don't think the GitHub Actions YAML magic to do so exists.

Closes #529

* merge new check-for-skip job into setup job

Since the check-for-skip work needs pretty much the same setup as the
setup job, merge the two and avoid the extra job.

* yaml syntax

* fix yaml

* [publish workflow] should cause tests to be skipped

A wee sanity test

* [publish workflow] debug commit msg

* remove println debug

On a PR the commit message is the merge commit.
That's not important to me, I care about behaviour on master, the branch
from which we publish.

Author: lread

.github/workflows/publish.yml

@@ -0,0 +1,55 @@
+name: Publish
+
+on:
+  push:
+    tags:
+      - 'v\d+.*'
+
+jobs:
+  test:
+    uses: ./.github/workflows/test.yml
+
+  publish:
+    environment: publish
+    runs-on: ubuntu-latest
+    needs: [test]
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Restore Clojure deps from cache
+      uses: actions/cache/restore@v4
+      with:
+        path: |
+          ~/.m2/repository
+          ~/.deps.clj
+          ~/.gitlibs
+        enableCrossOsArchive: true
+        key: cljdeps-${{ hashFiles('deps.edn', 'bb.edn') }}
+        restore-keys: cljdeps-
+
+    - name: Setup Java
+      uses: actions/setup-java@v4
+      with:
+        distribution: 'temurin'
+        java-version: '21'
+
+    - name: Install Clojure Tools
+      uses: DeLaGuardo/[email protected]
+      with:
+        bb: 'latest'
+
+    - name: Deploy to clojars
+      env:
+        CLOJARS_USERNAME: ${{ secrets.CLOJARS_USERNAME }}
+        CLOJARS_PASSWORD: ${{ secrets.CLOJARS_PASSWORD }}
+      run: bb -ci-clojars-deploy
+
+    - name: Create GitHub Release
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: bb -ci-github-create-release
+
+    - name: Inform Cljdoc
+      run: bb -ci-cljdoc-request-build

.github/workflows/release.yml

@@ -1,6 +1,7 @@
 name: Test
 
 on:
+  workflow_call: # for Publish
   push:
     branches:
     - master
@@ -13,6 +14,7 @@ jobs:
     runs-on: ubuntu-latest
 
     outputs:
+      skip_tests: ${{ steps.check_for_skip.outputs.skip_tests }}
       tests: ${{ steps.set-tests.outputs.tests }}
 
     steps:
@@ -44,14 +46,20 @@ jobs:
     - name: Bring down deps
       run: bb download-deps
 
+    - name: Check if Tests Should Run
+      id: check_for_skip
+      run: bb -ci-set-skip-tests
+
     - id: set-tests
+      if: steps.check_for_skip.outputs.skip_tests == 'false'
       name: Set test var for matrix
       # run test.clj directly instead of via bb task to avoid generic task output
       run: echo "tests=$(bb script/test_matrix.clj --format json)" >> $GITHUB_OUTPUT
 
   build:
     needs: setup
     runs-on: ${{ matrix.os }}-latest
+    if: needs.setup.outputs.skip_tests == 'false'
     strategy:
       fail-fast: false
       matrix:
@@ -129,7 +137,6 @@ jobs:
         mv ../../../.gitlibs ${USERPROFILE}
       shell: bash
 
-
     - name: Setup Java
       uses: actions/setup-java@v4
       with:

.github/workflows/test.yml

@@ -11,6 +11,12 @@ A release with an intentional breaking changes is marked with:
 * [breaking] you probably need to change your code
 * [minor breaking] you likely don't need to change your code
 
+// DO NOT EDIT: the "Unreleased" section header is automatically updated by bb publish
+// bb publish will fail on any of:
+// - unreleased section not found,
+// - unreleased section empty
+// - optional attribute is not [breaking] or [minor breaking]
+//   (adjust these in publish.clj as you see fit)
 == Unreleased
 
 * {pr}552[#552]: Add support for wide characters to input fill functions

CHANGELOG.adoc

@@ -6,7 +6,8 @@
                            :sha "cf44c15f30ea3867227fa61ceb823e5e942c707f"}
         dev.nubank/docopt {:mvn/version "0.6.1-fix7"}
         org.babashka/http-server {:mvn/version "0.1.12"}
-        wevre/natural-compare {:mvn/version "0.0.10"}}
+        wevre/natural-compare {:mvn/version "0.0.10"}
+        io.github.babashka/neil {:git/tag "v0.3.67" :git/sha "054ca51"}}
  :tasks
  {;; setup
   :requires ([babashka.classpath :as cp]
@@ -57,7 +58,7 @@
                   :task (do
                           ;; timbre default logging level is debug, which generates a lot of http logging noise
                           (timbre/set-level! :info)
-                          (exec 'build-shared/test))
+                          (exec 'test-shared/test))
                   :org.babashka/cli {:coerce {:nses [:symbol]
                                               :patterns [:string]
                                               :vars [:symbol]}}}
@@ -110,5 +111,14 @@
   fake-driver {:doc "Fake driver to support testing"
                :task-decoration :none
                :task fake-driver/-main}
-  ci-release     {:doc "release tasks, use --help for args"
-                  :task ci-release/-main}}}
+
+  pubcheck          {:task publish/pubcheck        :doc "Run only publish checks (without publishing)"}
+  publish           {:task publish/-main           :doc "Publish a release (for maintainers)"}
+  ;; let's not rely on a random version of neil
+  neil              {:task babashka.neil/-main     :doc "Pinned version of babashka/neil (used in scripting)"}
+
+  ;; hidden tasks, no need for folks to be trying these ci invoked tasks
+  -ci-set-skip-tests {:task ci-publish/set-skip-tests :doc "used on ci to determine if tests need to be run"}
+  -ci-clojars-deploy {:task ci-publish/clojars-deploy :doc "triggered on ci by release tag"}
+  -ci-github-create-release {:task ci-publish/github-create-release :doc "triggered on ci by release tag"}
+  -ci-cljdoc-request-build {:task ci-publish/cljdoc-request-build :doc "ask cljdoc to build docs for new release"}}}

bb.edn

@@ -1,37 +1,24 @@
 (ns build
-  (:require [clojure.edn :as edn]
-            [clojure.java.io :as io]
+  (:require [build-shared]
+            [clojure.edn :as edn]
             [clojure.tools.build.api :as b]))
 
-(defn version-string []
-  (let [{:keys [major minor release qualifier]} (-> "version.edn"
-                                                    slurp
-                                                    edn/read-string)]
-    (format "%s.%s.%s%s"
-            major minor release (if qualifier
-                                  (str "-" qualifier)
-                                  ""))))
+(def version (build-shared/lib-version))
+(def lib (build-shared/lib-artifact-name))
 
-(def lib 'etaoin/etaoin)
-(def version (version-string)) ;; the expectations is some pre-processing has bumped the version when releasing
 (def class-dir "target/classes")
 (def basis (b/create-basis {:project "deps.edn"}))
 (def jar-file (format "target/%s.jar" (name lib)))
-(def built-jar-version-file "target/built-jar-version.txt")
-
-(defn clean [_]
-  (b/delete {:path "target"}))
 
 (defn jar
   "Build library jar file.
-  Also writes built version to target/built-jar-version.txt for easy peasy pickup by any interested downstream operation.
-
-  We use the optional :version-suffix to optionally distinguish local installs from productin releases.
-  For example, when installing for a cljdoc preview suffix is: cljdoc-preview."
-  [{:keys [version-suffix] :as opts}]
-  (let [version (if version-suffix
-                  (format "%s-%s" version version-suffix)
-                  version)]
+  Supports `:version-override` for local testing, otherwise official version is used.
+  For example, when previewing for cljdoc we use a the version with  -cljdoc-preview suffix."
+  [{:keys [version-override] :as opts}]
+  (b/delete {:path class-dir})
+  (b/delete {:path jar-file})
+  (let [version (or version-override version)]
+    (println "jarring version" version)
     (b/write-pom {:class-dir class-dir
                   :lib lib
                   :version version
@@ -53,37 +40,26 @@
                  :target-dir class-dir})
     (b/jar {:class-dir class-dir
             :jar-file jar-file})
-    (spit built-jar-version-file version)
-    (assoc opts :built-jar-version version)))
-
-(defn- built-version* []
-  (when (not (.exists (io/file built-jar-version-file)))
-    (throw (ex-info (str "Built jar version file not found: " built-jar-version-file) {})))
-  (slurp built-jar-version-file))
-
-(defn built-version
-  ;; NOTE: Used by release script and github workflow
-  "Spit out version of jar built (with no trailing newline).
-  A separate task because I don't know what build.tools might spit to stdout."
-  [_]
-  (print (built-version*))
-  (flush))
+    (assoc opts :built-version version)))
 
-(defn install [opts]
-  (clean opts)
-  (let [{:keys [built-jar-version]} (jar opts)]
+(defn install
+  "Install built jar to local maven repo, optionally specify `:version-override` for local testing."
+  [opts]
+  (let [{:keys [built-version]} (jar opts)]
+    (println "installing version" built-version)
     (b/install {:class-dir class-dir
                 :lib lib
-                :version built-jar-version
+                :version built-version
                 :basis basis
                 :jar-file jar-file})))
 
-(defn deploy [opts]
+(defn deploy [_]
+  (jar {})
+  (println "deploy")
   ((requiring-resolve 'deps-deploy.deps-deploy/deploy)
    {:installer :remote
     :artifact jar-file
-    :pom-file (b/pom-path {:lib lib :class-dir class-dir})})
-  opts)
+    :pom-file (b/pom-path {:lib lib :class-dir class-dir})}))
 
 (defn download-deps
   "Download all deps for all aliases"
@@ -92,8 +68,7 @@
                      slurp
                      edn/read-string
                      :aliases
-                     keys
-                     sort)]
+                     keys)]
     ;; one at a time because aliases with :replace-deps will... well... you know.
     (println "Bring down default deps")
     (b/create-basis {})

build.clj

@@ -1,26 +1,26 @@
 (ns build-shared
-  (:refer-clojure :exclude [test])
   (:require
-   [cognitect.test-runner :as tr]))
+   [clojure.edn :as edn]
+   [clojure.string :as string]))
 
-;; private fn pasted from original sources
-(defn- do-test
-  [{:keys [dirs nses patterns vars includes excludes]}]
-  (let [adapted {:dir (when (seq dirs) (set dirs))
-                 :namespace (when (seq nses) (set nses))
-                 :namespace-regex (when (seq patterns) (map re-pattern patterns))
-                 :var (when (seq vars) (set vars))
-                 :include (when (seq includes) (set includes))
-                 :exclude (when (seq excludes) (set excludes))}]
-    (tr/test adapted)))
+(defn- project-info []
+  (-> (edn/read-string (slurp "deps.edn"))
+      :aliases :neil :project))
 
+(def version-tag-prefix "v")
 
-(defn test
-  "Reimplement test to not throw but instead exit with 1 on error."
-  [opts]
-  (try
-   (let [{:keys [fail error]} (do-test opts)]
-     (System/exit (if (zero? (+ fail error)) 0 1)))
-   (finally
-     ;; Only called if `test` raises an exception
-     (shutdown-agents))))
+(defn lib-version []
+  (-> (project-info) :version))
+
+(defn lib-artifact-name []
+  (-> (project-info) :name))
+
+(defn lib-github-coords []
+  (-> (project-info) :github-coords))
+
+(defn version->tag [version]
+  (str version-tag-prefix version))
+
+(defn tag->version [ci-tag]
+  (and (string/starts-with? ci-tag version-tag-prefix)
+       (string/replace-first ci-tag version-tag-prefix "")))