Merge pull request #259 from Veetaha/feat/github-action

Add a github action

.github/workflows/ci.yml

@@ -164,3 +164,39 @@ jobs:
           --baseline-root './upstream'
           --default-features
           --release-type minor
+
+  # Check that the Github Action works
+  github-action-test:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        # Make sure we cover all operating systems supported by Github Actions
+        os:
+          - windows-2019
+          - windows-2022
+          - ubuntu-20.04
+          - ubuntu-22.04
+          - macos-13
+          - macos-12
+          - macos-11
+
+    steps:
+      - uses: actions/checkout@v4
+
+      # This action downloads the latest released version of `cargo-marker`,
+      # and installs it into `$PATH`.
+      #
+      # Because `marker_lints` in this repo depends on the next dev version of
+      # `marker_api` it won't be compatible with the latest released `marker_api`,
+      # so there is no sense in actually running `cargo marker check` here.
+      # Therefore we set `install-only` to skip running a command.
+      #
+      # At least this checks that our installation script works as expected.
+      - uses: ./
+        with:
+          install-only: true
+
+      # +stable is to force using the pre-installed `cargo` on the runner instead of
+      # what's specified in `rust-toolchain.toml`
+      - run: cargo +stable marker --version

.github/workflows/release-on-tag.yml

@@ -3,9 +3,12 @@
 # See the docs/internal/release.md for more details.
 
 name: release-on-tag
+run-name: release-on-tag (${{ github.ref_name }})
 on:
   push:
-    tags: ['v*']
+    # Match only on specific tags. We don't want this workflow to be invoked when
+    # we put sliding `v{major}` and `v{major}.{minor}` tags on the same commit.
+    tags: ['v[0-9]+.[0-9]+.[0-9]+*']
 
 defaults:
   run:

.github/workflows/release.yml

@@ -32,6 +32,9 @@ jobs:
         run: echo "release_version=$(scripts/release/get-version-from-changelog.sh)" >> $GITHUB_ENV
       - name: Resolve the next dev version
         run: echo "next_dev_version=$(scripts/release/get-next-dev-version.sh ${{ env.release_version }})" >> $GITHUB_ENV
+      - name: Resolve the git tags
+        run: echo "tags=$(scripts/release/get-git-tags.sh ${{ env.release_version }})" >> $GITHUB_ENV
+
 
       # To be able to create a commit we need some committer identity.
       - run: |
@@ -42,11 +45,21 @@ jobs:
       - run: scripts/release/set-version.sh ${{ env.release_version }}
           --commit "🚀 Release v${{ env.release_version }}"
 
-      - run: git tag v${{ env.release_version }}
+      - run: |
+          for tag in ${{ env.tags }}; do
+            git tag $tag
+          done
 
       # Create a next dev version commit
       - run: scripts/release/set-version.sh ${{ env.next_dev_version }}
           --commit "🚧 Development v${{ env.next_dev_version }}"
 
-      # Push the branch and the new tag to the remote
-      - run: git push --atomic origin ${{ github.ref_name }} v${{ env.release_version }}
+      # Push the changes to the remote
+      - run: git push origin ${{ github.ref_name }}
+
+      # We use `--force` because the tags that are pushed here will include the
+      # sliding `v{major}` and `v{major}.{minor}` tags, so we have to overwrite
+      # them with the new ones. To prevent force-pushing the master branch we
+      # do this push with --force in a separate step, so it's not fully atomic,
+      # but it's good enough.
+      - run: git push --force --atomic origin ${{ env.tags }}

CHANGELOG.md

@@ -36,13 +36,15 @@ See the [v0.3.0 milestone] for a full list of all changes.
 [#245]: https://github.com/rust-marker/marker/pull/245
 [#252]: https://github.com/rust-marker/marker/pull/252
 [#256]: https://github.com/rust-marker/marker/pull/256
+[#259]: https://github.com/rust-marker/marker/pull/259
 [#263]: https://github.com/rust-marker/marker/pull/263
 [#265]: https://github.com/rust-marker/marker/pull/265
 
 ### Added
 - [#232]: Add scope config for visitors and `for_each_expr` to `marker_utils`
 - [#239]: GitHub releases now provide precompiled binaries of `cargo-marker` and `marker_rustc_driver`.
 - [#252]: Marker now provides install scripts for linux, macos and windows
+- [#259]: Introduced a GitHub Action for installing and running Marker
 
 ### Breaking Changes
 - [#256]: Renamed `AstContext` -> `MarkerContext`

action.yml

@@ -0,0 +1,28 @@
+name: Rust Marker Linter
+description: GitHub Action to install and run the Marker linter for Rust 🦀
+branding:
+  icon: edit-3
+  color: white
+
+inputs:
+  install-only:
+    description: >
+      If set to `true` then the action will only install `cargo marker`,
+      and will skip running `cargo marker`. Use this if you want to run
+      something more complex than just `cargo marker`. If you think there
+      may be a frequent use case for running a different command then we will be
+      glad if you open a feature request issue for that to extend the action input
+      parameters.
+
+    default: 'false'
+    required: false
+
+runs:
+  using: composite
+  steps:
+    - run: ${GITHUB_ACTION_PATH:?}/scripts/release/install.${{ runner.os == 'Windows' && 'ps1' || 'sh' }}
+      shell: bash
+
+    - run: cargo marker
+      if: ${{ inputs.install-only == 'false' }}
+      shell: bash

docs/internal/release.md

@@ -34,7 +34,7 @@ The `sha256` sum is a small file that users may optionally download together wit
 This [`install.sh`](https://raw.githubusercontent.com/rust-marker/marker/v0.2.1/scripts/release/install.sh) script, can be used to automatically download and verify Marker's binaries.
 <!-- endregion replace-version stable -->
 
-## Operating system versions coverage
+### Operating system versions coverage
 
 We want to cover not only the mainstream operating systems, but also some reasonable range of their versions. For example, for `ubuntu` we want our binaries to work on the current LTS version of `ubuntu` and also on the version of `ubuntu` that precedes it. At the time of this writing the latest LTS version of `ubuntu` is `22.04`, and the previous one is `20.04`, while the LTS version of ubuntu before that one is `18.04` and it already reached the "end of standard support" date, and so far practically noone uses it.
 
@@ -62,6 +62,21 @@ The binaries with `musl` in their name don't depend on GLIBC. They are staticall
 
 This comes at a cost, though. The `musl` implementation of `libc` isn't complete, and it may have bugs, performance degradations compared to GLIBC, etc. However, it generally covers everything you may ever need if you aren't doing something unusual.
 
+## Github Action
+
+There is a Github Action template `action.yml` file maintained within the Marker repository. It installs the pre-compiled binaries using the installation scripts and runs `cargo marker`.
+
+It is common to reference the Github Action using sliding tags like this:
+```yml
+- uses: rust-marker/[email protected]
+```
+Or this when marker reaches a `1.0.0` version and minor versions are compatible:
+```yml
+- uses: rust-marker/marker@v1
+```
+
+We maintain these tags in our automated release flow described below.
+
 # Regular release
 
 The regular release means a planned event when the maintainer of `marker` publishes the new version of the artifacts to the users. It may happen at any time when such a decision is made, which usually means on some consistent schedule.
@@ -80,7 +95,17 @@ Once the `CHANGELOG.md` is ready in `master` the maintainer can trigger the CI `
 
 The `release` CI workflow then checks out the `master` branch (assuming "Use workflow from" input wasn't changed from its default), parses the current release version from the `CHANGELOG.md` file and updates `Cargo.toml`, `Cargo.lock` and various `.rs` and `.md` files with the new version. It uses simple regex patterns with `sed` to edit the files. This logic may break, and thus there is a test on regular CI that makes sure it stays stable.
 
-The release commit is then assigned a `v{semver}` git tag. After that the workflow sets the next version for the new development cycle with the incremented release version and the `-dev` suffix, and creates a new commit with that. No additional tag at this point is created.
+The release commit is then assigned a `v{semver}` git tag . After that the workflow sets the next version for the new development cycle with the incremented release version and the `-dev` suffix, and creates a new commit with that.
+
+### Sliding `v{major}` and `v{major}.{minor}` tags.
+
+The release CI flow will also create and move the sliding `v{major}` and `v{major}.{minor}` tags. For example, if we release a version `0.3.0`, then there will be three tags created `v0.3.0`, `v0` and `v0.3`.
+
+And if we release a patch `0.3.1`, then there will be a new tag `v0.3.1`, and the existing `v0` and `v0.3` will be moved to this new  commit for `0.3.1`.
+
+The sliding tags won't be updated for pre-releases (versions with `-` suffix in them).
+
+These sliding tags may be used to hardcode only the major or minor version of `marker`, to allow for automatic updates, to the patch version. These sliding tags can be used to download the installation script or run Marker's Github Action and automatically get fixes from the upstream.
 
 ## `release-on-tag` workflow
 

scripts/release/get-git-tags.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+version="$1"
+
+IFS='.' read -r major minor patch <<< "$version"
+
+# We always put the full version tag even if this is a pre-release
+echo -n "v$version"
+
+# For suffixless stable release versions we also want to set the
+# sliding `v{major}` and `v{major}.{minor}` tags so that uses could
+# depend on `rust-marker/[email protected]` or `rust-marker/marker@v1`
+# version of the Github Action.
+if [[ "$version" != *-* ]]; then
+    echo -n " v$major v$major.$minor"
+fi

scripts/release/install.ps1

@@ -1,3 +1,5 @@
+#!/usr/bin/env pwsh
+#
 # This script downloads the `cargo-marker` and the `marker_rustc_driver`
 # binaries from the GitHub release assets. It also sets up the required
 # Rust toolchain that the `marker_rustc_driver` depends on.
@@ -14,6 +16,30 @@
 # This script is specifically for windows, but has a similar structure to the
 # unix `install.sh` script. If you modify this script, please check if the modifications
 # should also apply to the unix one.
+#
+# General PowerShell notes:
+#
+# Since we have $ErrorActionPreference = "Stop" the script will stop at any `Write-Error`.
+# Yeah, writing an error log counts as an error. Who could expect that? Regardless of how
+# unintuitive it is we use `Write-Error` instead of `throw` because the error that is generated
+# this way is more readable, because it refers to the call site of the function where the error
+# was written and not to the `throw` statement itself in the code snippet that is output.
+#
+# Example error output with `throw` shows the `throw` statement itself, not the call site:
+# ```
+# Line |
+#   74 |          throw "Command $cmd failed with exit code $LASTEXITCODE"
+#      |          ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+#      | Command curl failed with exit code 22
+# ```
+# Example error output with `Write-Error` shows the call site of the function which is more useful:
+# ```
+# Line |
+#  174 |      Step curl$exe `
+#      |      ~~~~~~~~~~~~~~~
+#      | Command curl failed with exit code 22
+# ```
+
 
 $ErrorActionPreference = "Stop"
 
@@ -25,34 +51,68 @@ $version = "0.2.1"
 
 $toolchain = "nightly-2023-08-24"
 
-function With-Log {
-    Write-Output "> $($args -join ' ')"
-
+# Log the command, execute, and fail if its exit code is non-zero.
+# Surprisingly PowerShell can't do the exit code checks for us out of the box.
+function Step {
     $cmd = $args[0]
     $rest = $args[1..$args.Length]
 
+    # ASCII escape symbol
+    $e = [char]0x1b
+
+    # This is the unicode code of the symbol ❱.
+    # Yeah, this is how you do unicode in PowerShell, yay -_-
+    $run_symbol = [char]0x2771
+
+    Write-Host "$e[32;1m$run_symbol $e[1;33m$cmd$e[0m $($rest -join ' ')"
+
     & $cmd @rest
+
+    # Turns out `ErrorActionPreference` doesn't affect the behavior of external
+    # processes. So if any process returns a non-zero exit code PowerShell will
+    # still ignore this. Yeah.. Life is cruel, but at least PowerShell has
+    # an experimental feature to fix this: `PSNativeCommandErrorActionPreference`.
+    # Anyway, since we have to support pre-historic PowerShell, we'll do our own
+    # error handling.
+    if ($? -eq $false) {
+        Write-Error "Command $cmd failed with exit code $LASTEXITCODE"
+    }
 }
 
-function Extract-TarGz {
+function Unzip {
     param (
         [string]$bin,
         [string]$dest
     )
     $file_stem = "${bin}-${host_triple}"
 
-    With-Log cd $temp_dir
-
-    Check-Sha256Sum $file_stem "tar.gz"
+    # We have to enter and exit from the temp dir because the destination path may be
+    # relative, and we don't want that path to be relative to the temp dir.
+    Step Push-Location $temp_dir
+    Step Check-Sha256Sum $file_stem "zip"
+    Step Pop-Location
 
-    With-Log tar --extract --file (Join-Path $temp_dir "$file_stem.tar.gz") --directory $dest
+    # There is `tar` on Windows installed by default, but it looks like different
+    # environments have extremely different variations of tar installed.
+    #
+    # For example, my home laptop with Windows 11 has `bsdtar 3.5.2` installed.
+    # It works fine with Windows paths out of the box. However, Windows Github Actions
+    # runner has `tar (GNU tar) 1.34`, which somehow doesn't work with Windows paths.
+    # It just doesn't eat a path with a drive letter and says "No such file or directory".
+    #
+    # Anyway, there is so much inconsistency between home Windows and Github Actions, that
+    # it's just easier to use the PowerShell builtin utility that has always been there.
+    Step Expand-Archive `
+        -Force `
+        -LiteralPath (Join-Path $temp_dir "$file_stem.zip") `
+        -DestinationPath $dest
 }
 
 # There isn't mktemp on Windows, so we have to reinvent the wheel.
 function New-TemporaryDirectory {
     $parent = [System.IO.Path]::GetTempPath()
     [string] $name = [System.Guid]::NewGuid()
-    New-Item -ItemType Directory -Path (Join-Path $parent $name)
+    Step New-Item -ItemType Directory -Path (Join-Path $parent $name)
 }
 
 # There isn't sha256sum on Windows, so we have to reinvent the wheel.
@@ -68,7 +128,8 @@ function Check-Sha256Sum {
     $expected = Get-Content "$file_stem.sha256"
 
     foreach ($line in $expected) {
-        $line -match '(\S+)\s*\*?(.*)'
+        # Silence the output from the match, otherwise it prints `True` or `False`
+        $null = $line -match '(\S+)\s*\*?(.*)'
         $expected_hash = $Matches[1]
         $expected_file = $Matches[2]
 
@@ -81,12 +142,15 @@ function Check-Sha256Sum {
             return
         }
 
-        throw "Checksum verification failed for $file. Expected: $expected_hash, actual: $actual"
+
+        Write-Error "Checksum verification failed for $file. Expected: $expected_hash, actual: $actual"
     }
 
-    throw "No checksum found for $file"
+    Write-Error "No checksum found for $file"
 }
 
+Write-Output "PowerShell version: $($PSVersionTable.PSVersion)"
+
 # This script can run on unix too if you have PowerShell installed there.
 # The only difference is that on Windows's old PowerShell 5 there is an alias
 # `curl` for `Invoke-WebRequest`, and if you want to use the real curl, then
@@ -103,7 +167,9 @@ $exe = if ([System.Environment]::OSVersion.Platform -eq "Win32NT") {
     ""
 }
 
-With-Log rustup install --profile minimal --no-self-update $toolchain
+Step curl$exe --version
+
+Step rustup install --profile minimal --no-self-update $toolchain
 
 $host_triple = (
     rustc +$toolchain --version --verbose `
@@ -115,13 +181,13 @@ $current_dir = (Get-Location).Path
 $temp_dir = New-TemporaryDirectory
 
 try {
-    $files = "{cargo-marker,marker_rustc_driver}-$host_triple.{tar.gz,sha256}"
+    $files = "{cargo-marker,marker_rustc_driver}-$host_triple.{zip,sha256}"
 
     # Curl is available by default on windows, yay!
     # https://curl.se/windows/microsoft.html
     #
     # Download all files using a single TCP connection with HTTP2 multiplexing
-    With-Log curl$exe `
+    Step curl$exe `
         --location `
         --silent `
         --fail `
@@ -140,18 +206,20 @@ try {
         Join-Path $HOME ".cargo"
     }
 
-    Extract-TarGz "cargo-marker" (Join-Path $cargo_home "bin")
+    Unzip "cargo-marker" (Join-Path $cargo_home "bin")
 
     $sysroot = (rustc +$toolchain --print sysroot)
-    Extract-TarGz "marker_rustc_driver" (Join-Path $sysroot "bin")
-} finally {
-    Write-Output "Removing the temp directory $temp_dir"
+    Unzip "marker_rustc_driver" (Join-Path $sysroot "bin")
 
+    # We use `+$toolchain` to make sure we don't try to install the default toolchain
+    # in the workspace via the rustup proxy, but use the toolchain we just installed.
+    Step cargo +$toolchain marker --version
+} finally {
     # Go back to the original directory before removing the temp directory
     # otherwise it will fail because the temporary directory is in use.
     cd $current_dir
 
-    Remove-Item -Force -Recurse $temp_dir
+    Step Remove-Item -Force -Recurse $temp_dir
 }
 
 # You, my friend will be surprised. But the workability of this entire