Flypkgs builds (#2893)

* Allow `+build.meta` suffixes in version numbers
This is needed for snapshot builds in testing/preflight
* distribute and version tools used for releases in CI
* remove old dev builds from CI
* new goreleaser config to isolate changes from the existing build
* GHA workflows for new builds
* Fix next version command on stable
* Comment out tests for now
* Fix previous tag logic on detached head PRs with no previous tag
* Update ci-dev.yml

* Update ci-dev.yml

* return proper head ref when GHA runs on a pull request merge branch

* cleanup ci-dev workflow, prevent race runs

* lint fixes

* fix go lint warnings

* Fix tests for PR head refs

* Fix go lint errors

* Update comments from feedback

* Fix DeepSource error

* Fix issue where previous version tags weren't picked up on PR builds

* Use makefile for tests

* Update ci-dev.yml
* Split workflow jobs to reduce races
* Call workflow manually until merged in default branch
* Remove concurrency until workflow is triggered independently
* Update release.yml
* Update release.yml
* Update release.yml
* Update release.yml
* fix lint issue
* JSON encoding tests for version.Version
* Pickup correct branch name if building a PR
* Use correct GH context env var for head ref
* Update ci-dev.yml

.github/workflows/ci-dev.yml

@@ -0,0 +1,73 @@
+name: CI - Dev
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - ready_for_review
+  # push:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
+
+jobs:
+  test:
+    uses:
+      ./.github/workflows/test.yml
+
+  # debug:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - run: echo "$GITHUB_CONTEXT"
+  #       env:
+  #         GITHUB_CONTEXT: ${{ toJSON(github) }}
+
+  # create a dev tag for every branch except master
+  tag_version:
+    needs: test
+    if: ${{ github.ref_type == 'branch' && github.ref_name != 'master' }}
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.version.outputs.next }}
+      tag: v${{ steps.version.outputs.next }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-tags: true
+          fetch-depth: 0
+      - uses: actions/setup-go@v4
+        with:
+          go-version-file: "go.mod"
+      - name: Get version number
+        id: version
+        run: |
+          echo "next=$(go run ./tools/version next)" >> "$GITHUB_OUTPUT"
+      - name: Print version number
+        run: |
+          echo "### Release Info :rocket:" >> $GITHUB_STEP_SUMMARY
+          echo "Version: ${{ steps.version.outputs.next }}" >> $GITHUB_STEP_SUMMARY
+      - name: Create tag
+        uses: actions/github-script@v6
+        env:
+          TAG_NAME: v${{ steps.version.outputs.next }}
+        with:
+          script: |
+            const { TAG_NAME } = process.env
+            github.rest.git.createRef({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref: `refs/tags/${TAG_NAME}`,
+              sha: context.sha
+            })
+
+  # we can remove this workflow_call once the release.yml workflow file is
+  # merged into the default branch
+  build_and_release:
+    uses:
+      ./.github/workflows/release.yml
+    needs: tag_version
+    secrets: inherit

.github/workflows/ci.yml

@@ -106,40 +106,6 @@ jobs:
           GITHUB_TOKEN: ${{ steps.installationToken.outputs.token }}
         run: scripts/publish_docs.sh ${{ github.ref_name }}
 
-  dev_release:
-    if: startsWith(github.ref, 'refs/tags/v0.') && contains(github.ref, 'dev-')
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - uses: actions/setup-go@v4
-        with:
-          go-version-file: "go.mod"
-          check-latest: true
-      - name: Get go version
-        id: go-version
-        run: echo "name=version::$(go env GOVERSION)" >> $GITHUB_OUTPUT
-      - name: Place wintun.dll
-        run: cp -r deps/wintun/bin/amd64/wintun.dll ./
-      - name: generate release notes
-        run: |
-          mkdir -p ./tmp
-          ./scripts/changelog.sh > ./tmp/changelog.txt
-      - name: Run GoReleaser
-        uses: goreleaser/goreleaser-action@v5
-        with:
-          version: latest
-          args: release --config .goreleaser.dev.yml --rm-dist --release-notes=./tmp/changelog.txt
-        env:
-          GITHUB_TOKEN: ${{ secrets.GORELEASER_GITHUB_TOKEN }}
-          BUILD_ENV: "development"
-      - name: Upload checksums as artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: checksums
-          path: dist/checksums.txt
-
   aur-publish:
     name: Build & publish to AUR
     needs: release

.github/workflows/release.yml

@@ -0,0 +1,129 @@
+name: Build & Release
+
+on:
+  push:
+    tags:
+      - v**
+
+  # need this until this file is on the default branch so tag creates are picked up
+  workflow_call:
+
+# concurrency:
+#   group: ${{ github.workflow }}-${{ github.ref }}
+#   cancel-in-progress: true
+
+jobs:
+  meta:
+    runs-on: ubuntu-latest
+    outputs:
+      sha_short: ${{ steps.gen.outputs.sha_short }}
+      json: ${{ steps.gen.outputs.json }}
+      version: ${{ fromJSON(steps.gen.outputs.json).version }}
+      tag: ${{ fromJSON(steps.gen.outputs.json).tag }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+      - uses: actions/setup-go@v4
+        with:
+          go-version-file: "go.mod"
+      - name: Generate release meta
+        run: |
+          go run ./tools/version show > release.json
+      - name: Output release meta
+        id: gen
+        run: |
+          echo "sha_short=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT"
+          echo "json=$(cat release.json)" >> "$GITHUB_OUTPUT"
+      - uses: actions/upload-artifact@v3
+        with:
+          name: build-artifacts
+          path: |
+            release.json
+
+  build:
+    needs: meta
+    strategy:
+      matrix:
+        GOOS: [linux, windows, darwin]
+    runs-on: ubuntu-latest-m
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-go@v4
+        with:
+          go-version-file: "go.mod"
+      - name: Place wintun.dll
+        run: cp -r deps/wintun/bin/amd64/wintun.dll ./
+      - id: cache
+        uses: actions/cache@v3
+        with:
+          path: dist/${{ matrix.GOOS }}
+          key: ${{ matrix.GOOS }}-${{ needs.meta.outputs.sha_short }}
+      - name: Run GoReleaser
+        if: steps.cache.outputs.cache-hit != 'true' # do not run if cache hit
+        uses: goreleaser/goreleaser-action@v4
+        with:
+          distribution: goreleaser-pro
+          version: latest
+          args: release --clean -f .goreleaser.2.yml --fail-fast --split
+        env:
+          GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
+          GGOOS: ${{ matrix.GOOS }}
+
+  release:
+    runs-on: ubuntu-latest
+    needs: [meta, build]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-go@v4
+        with:
+          go-version-file: "go.mod"
+      # copy the caches from prepare
+      - uses: actions/cache@v3
+        with:
+          path: dist/linux
+          key: linux-${{ needs.meta.outputs.sha_short }}
+      - uses: actions/cache@v3
+        with:
+          path: dist/darwin
+          key: darwin-${{ needs.meta.outputs.sha_short }}
+      - uses: actions/cache@v3
+        with:
+          path: dist/windows
+          key: windows-${{ needs.meta.outputs.sha_short }}
+      - name: Write release meta
+        env:
+          RELEASE_JSON: ${{ needs.meta.outputs.json }}
+        run: echo "$RELEASE_JSON" > ./dist/release.json
+      - uses: actions/upload-artifact@v3
+        with:
+          name: build-artifacts
+          path: |
+            dist/**/*.json
+            dist/**/*.zip
+            dist/**/*.tar.gz
+          retention-days: 1
+      - name: Upload to flypkgs
+        env:
+          FLYPKGS_API_TOKEN: ${{ secrets.FLYPKGS_API_TOKEN }}
+        run: go run ./tools/distribute upload ./dist
+
+  # preflight here once old and new workflows are merged
+
+  publish:
+    needs: [meta, release]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v4
+        with:
+          go-version-file: "go.mod"
+      - name: Upload to flypkgs
+        env:
+          FLYPKGS_API_TOKEN: ${{ secrets.FLYPKGS_API_TOKEN }}
+        run: go run ./tools/distribute publish ${{ needs.meta.outputs.version }}

.github/workflows/test.yml

@@ -0,0 +1,26 @@
+name: Test
+
+on:
+  workflow_call:
+
+jobs:
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest-m, macos-latest-xl, windows-latest-l ]
+        test:
+          - workdir: "."
+            target: "test"
+          - workdir: "api"
+            target: "test-api"
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-go@v4
+        with:
+          go-version-file: "${{ matrix.test.workdir }}/go.mod"
+      - name: Run Tests (${{ matrix.test.target }})
+        run: make ${{ matrix.test.target }}

.goreleaser.2.yml

@@ -0,0 +1,84 @@
+# This config targets the new flypkgs distribution system. Once all releases are
+# being served by flypkgs, this file will replace .goreleaser.yml and .goreleaser.dev.yml
+
+env:
+  - BUILD_ENV={{if index .Env "BUILD_ENV"}}{{.Env.BUILD_ENV}}{{else}}production{{end}}
+
+before:
+  hooks:
+    - go mod download
+    - go generate ./...
+
+builds:
+  - id: linux
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+    goarch:
+      - arm64
+      - amd64
+    ldflags:
+      - -X github.com/superfly/flyctl/internal/buildinfo.buildDate={{.CommitDate}}
+      - -X github.com/superfly/flyctl/internal/buildinfo.buildVersion={{.Version}}
+      - -X github.com/superfly/flyctl/internal/buildinfo.commit={{.ShortCommit}}
+    tags:
+      - "{{.Env.BUILD_ENV}}"
+
+  - id: macos
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - darwin
+    goarch:
+      - arm64
+      - amd64
+    ldflags:
+      - -X github.com/superfly/flyctl/internal/buildinfo.buildDate={{.CommitDate}}
+      - -X github.com/superfly/flyctl/internal/buildinfo.buildVersion={{.Version}}
+      - -X github.com/superfly/flyctl/internal/buildinfo.commit={{.ShortCommit}}
+    tags:
+      - "{{.Env.BUILD_ENV}}"
+
+  - id: windows
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - windows
+    goarch:
+      - arm64
+      - amd64
+    ldflags:
+      - -X github.com/superfly/flyctl/internal/buildinfo.buildDate={{.CommitDate}}
+      - -X github.com/superfly/flyctl/internal/buildinfo.buildVersion={{.Version}}
+      - -X github.com/superfly/flyctl/internal/buildinfo.commit={{.ShortCommit}}
+    tags:
+      - "{{.Env.BUILD_ENV}}"
+
+archives:
+  - id: linux
+    name_template: >-
+      {{ .ProjectName }}_{{ .Version }}_Linux_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}
+    builds: [linux]
+    files: [only-the-binary*]
+    wrap_in_directory: false
+    format: tar.gz
+
+  - id: macos
+    name_template: >-
+      {{ .ProjectName }}_{{ .Version }}_macOS_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}
+    builds: [macos]
+    files: [only-the-binary*]
+    wrap_in_directory: false
+    format: tar.gz
+
+  - id: windows
+    name_template: >-
+      {{ .ProjectName }}_{{ .Version }}_Windows_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}
+    builds: [windows]
+    files: [wintun.dll]
+    wrap_in_directory: false
+    format: zip
+
+release:
+  disable: true