diff --git a/.gitignore b/.gitignore index 51067f506..f875e8e0f 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ .eggs/ .envrc /*.sublime-* +.devenv/ __pycache__/ bin/ channels diff --git a/README.md b/README.md index fe6c566cf..200a245af 100644 --- a/README.md +++ b/README.md @@ -6,23 +6,67 @@ Development Mode Run in the source tree: - eval $(./dev-setup) + nix develop --impure -This sets up the `channels` directory and NIX_PATH. +This enters the dev shell where NIX_PATH is set properly and various scripts are available. -Development on a Test VM ------------------------- +Look at `flake.nix` to see how the dev shell is defined. The comment at the +top shows which commands are available in the dev shell. -For development on a FCIO test VM, sync the fc-nixos source tree to the target -machine and set up the `channels` directory with: +Running a Test VM on a local dev checkout +----------------------------------------- - ./dev-setup +To use a local dev checkout on a FCIO test VM, sync the `fc-nixos` source tree to the target +machine: -This can be done as regular user. Run the command again when nixpkgs changes -in order to update the `channels` directory. + rsync -aP ~/git/fc-nixos example01: + +On the machine, enter the dev shell and set up the `channels` directory: + + cd fc-nixos + nix develop --impure + build_channels_dir + +This can be done as regular user. Exit the shell and run the commands again +when nixpkgs changes. The VM has to use a matching environment that points to the `channels` dir. -`fc-manage switch` (as root) then uses the local code to rebuild the system. +`sudo fc-manage switch` then uses the local code to rebuild the system. + + +Automatically enter the dev shell with direnv +--------------------------------------------- + +Use `direnv` to automatically enter the dev shell when you change to the fc-nixos directory. + +To set it up with `home-manager`, see: +https://github.com/nix-community/nix-direnv?tab=readme-ov-file#via-home-manager + +Without home-manager +-------------------- + +On a NixOS machine, enabling `programs.direnv.enable` should be enough. + +Add `/etc/local/nixos/dev_vm.nix`, for example: + + { ... }: + { + nix.extraOptions = '' + keep-outputs = true + ''; + programs.direnv.enable = true; + } + +Rebuild the system, close the shell/tmux session and log in again. + +In `fc-nixos`, add an `.envrc` file like: + + use flake . --impure --allow-dirty + build_channels_dir + +Then, run `direnv allow` to build and enter the dev shell. + +Run `direnv allow` again if the dev shell disappears or doesn't reload automatically. Build Single Packages @@ -34,7 +78,7 @@ Run in development mode: Or build package by directly calling a Nix expression: - nix-build -E 'with import {}; callPackage path/to/file.nix {}' + nix-build -E 'with import {}; callPackage path/to/file.nix {}' (Dry-)Build System @@ -42,9 +86,9 @@ Or build package by directly calling a Nix expression: Run in development mode: - nix-build '' -A system + sudo nix-build '' -A system -Must be executed as *root* on FCIO test VMs. +(Must be executed as *root* on FCIO test VMs). Execute Tests @@ -87,18 +131,21 @@ Run the whole test suite (may take a very long time): nix-build release -A tested -Update Pinned Nixpkgs ---------------------- +Update Pinned Dependencies +-------------------------- + +The nixpkgs and nixos-mailserver versions used by the platform are pinned in `flake.lock`. The versions and hashes are written to `release/versions.json` by our release tooling and read from there by platform code. + +We use our [nixpkgs fork](https://github.com/flyingcircusio/nixpkgs) and the nixos-mailserver fork from our Gitlab. -We pin the used nixpkgs version in `versions.json` to a commit id from our -[nixpkgs fork](https://github.com/flyingcircusio/nixpkgs). The typical workflow -for a nixpkgs update looks like this: +The typical workflow for a nixpkgs update looks like this (run in the dev shell): -1. Prefetch hash for new version: `nix-prefetch-github flyingcircusio nixpkgs --rev nixos-23.11` -2. Change rev and sha256 in `versions.json` according to the prefetch output. -3. Create a draft PR with the changed `versions.json` and wait until Hydra finishes building. -4. When Hydra is green, try it out on a test VM. Don't forget to run `./dev-setup` to update the `channels` directory! +1. Rebase local nixpkgs onto current upstream version: `update_nixpkgs --nixpkgs-path ~/worksets/nixpkgs/fc/nixos-23.11 nixpkgs` +2. Update `versions.json` and `package-versions.json` (must be able to talk to hydra01): `update_nixpkgs fc-nixos` +3. Create a draft PR with the changes and wait until Hydra finishes building. +4. When Hydra is green, try it out on a test VM. Don't forget to run `build_channels_dir` if you haven't set up direnv! +To learn more about our release tooling, look at the comment in `flake.nix` at the top. License ------- diff --git a/dev-setup b/dev-setup index ea44e27d6..6b699f959 100755 --- a/dev-setup +++ b/dev-setup @@ -1,52 +1,6 @@ #!/usr/bin/env bash -# Usage: eval `./dev-setup` -set -e - +echo "This will set up the channels dir and show the export command to set the NIX_PATH accordingly." >&2 +echo "You can also use 'nix develop --impure' to open the dev shell and run build_channels_dir from there." >&2 # ensure PWD is the directory this script resides in (allows calls like ../dev-setup or $HOME/fc-nixos/dev-setup) cd "$(dirname "$(readlink -f "$0")")" - -base=$PWD -# preserve nixos-config -config=$(nix-instantiate --find-file nixos-config 2>/dev/null) || true - -# bootstrapping with what we find on the system -for candidate in \ - $HOME/.nix-defexpr/channels/nixos \ - $HOME/.nix-defexpr/channels_root/nixos \ - $HOME/.nix-defexpr/channels \ - $HOME/.nix-defexpr/channels_root; do - if [[ -e ${candidate} ]]; then - NIX_PATH="${NIX_PATH}${NIX_PATH:+:}${candidate}" - fi -done -export NIX_PATH -channels=`nix-build -Q --quiet versions.nix -A allUpstreams --no-out-link` -if [[ -z $channels ]]; then - echo "$0: failed to build nixpkgs+overlay" >&2 - exit 1 -fi -mkdir -p channels -find $channels -maxdepth 1 -type l | while read channel; do - target="channels/${channel##*/}" - # don't toucn anything which is not a link into the nix store - if [[ ! -L $target && -d $target ]]; then - echo "$0: warning: $target appears to be a local dev checkout" >&2 - continue - elif [[ -L $target && $(readlink $target) != /nix/store/* ]]; then - echo "$0: warning: $target appears to be a local dev checkout" >&2 - continue - fi - ln -fs $channel channels -done -if ! [[ -e channels/fc ]]; then - ln -s .. channels/fc -fi - -NIX_PATH="$base/channels" -if [[ -n "$config" ]]; then - NIX_PATH="${NIX_PATH}:nixos-config=$config" -else - NIX_PATH="${NIX_PATH}:nixos-config=${base}/nixos" -fi - -echo "export NIX_PATH=$NIX_PATH" +nix develop --impure "$@" --command dev_setup diff --git a/fc-get-current-channel-url.sh b/fc-get-current-channel-url.sh deleted file mode 100755 index 5e34185c5..000000000 --- a/fc-get-current-channel-url.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -branch=$1 -nixos_version=$(< nixos-version) -jobset="fc-$nixos_version-$branch" - -curl --silent --head https://hydra.flyingcircus.io/channel/custom/flyingcircus/${jobset}/release/nixexprs.tar.xz | grep "[Ll]ocation" | cut -f 2 -d " " diff --git a/fc/ceph/__init__.py b/fc/ceph/__init__.py deleted file mode 100644 index c51def152..000000000 --- a/fc/ceph/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# pkg diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..92f7358a3 --- /dev/null +++ b/flake.lock @@ -0,0 +1,338 @@ +{ + "nodes": { + "blobs": { + "flake": false, + "locked": { + "lastModified": 1604995301, + "narHash": "sha256-wcLzgLec6SGJA8fx1OEN1yV/Py5b+U5iyYpksUY/yLw=", + "owner": "simple-nixos-mailserver", + "repo": "blobs", + "rev": "2cccdf1ca48316f2cfd1c9a0017e8de5a7156265", + "type": "gitlab" + }, + "original": { + "owner": "simple-nixos-mailserver", + "repo": "blobs", + "type": "gitlab" + } + }, + "devenv": { + "inputs": { + "flake-compat": "flake-compat", + "nix": "nix", + "nixpkgs": [ + "nixpkgs" + ], + "pre-commit-hooks": "pre-commit-hooks" + }, + "locked": { + "lastModified": 1707004164, + "narHash": "sha256-9Hr8onWtvLk5A8vCEkaE9kxA0D7PR62povFokM1oL5Q=", + "owner": "cachix", + "repo": "devenv", + "rev": "0e68853bb27981a4ffd7a7225b59ed84f7180fc7", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "devenv", + "type": "github" + } + }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_2": { + "flake": false, + "locked": { + "lastModified": 1668681692, + "narHash": "sha256-Ht91NGdewz8IQLtWZ9LCeNXMSXHUss+9COoqu6JLmXU=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "009399224d5e398d03b22badca40a37ac85412a1", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1706830856, + "narHash": "sha256-a0NYyp+h9hlb7ddVz4LUn1vT/PLwqfrWYcHMvFB1xYg=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "b253292d9c0a5ead9bc98c4e9a26c6312e27d69f", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1685518550, + "narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "gitignore": { + "inputs": { + "nixpkgs": [ + "devenv", + "pre-commit-hooks", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1660459072, + "narHash": "sha256-8DFJjXG8zqoONA1vXtgeKXy68KdJL5UaXR8NtVMUbx8=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "a20de23b925fd8264fd7fad6454652e142fd7f73", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, + "lowdown-src": { + "flake": false, + "locked": { + "lastModified": 1633514407, + "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=", + "owner": "kristapsdz", + "repo": "lowdown", + "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8", + "type": "github" + }, + "original": { + "owner": "kristapsdz", + "repo": "lowdown", + "type": "github" + } + }, + "nix": { + "inputs": { + "lowdown-src": "lowdown-src", + "nixpkgs": [ + "devenv", + "nixpkgs" + ], + "nixpkgs-regression": "nixpkgs-regression" + }, + "locked": { + "lastModified": 1676545802, + "narHash": "sha256-EK4rZ+Hd5hsvXnzSzk2ikhStJnD63odF7SzsQ8CuSPU=", + "owner": "domenkozar", + "repo": "nix", + "rev": "7c91803598ffbcfe4a55c44ac6d49b2cf07a527f", + "type": "github" + }, + "original": { + "owner": "domenkozar", + "ref": "relaxed-flakes", + "repo": "nix", + "type": "github" + } + }, + "nixos-mailserver": { + "inputs": { + "blobs": "blobs", + "flake-compat": "flake-compat_2", + "nixpkgs": [ + "nixpkgs" + ], + "nixpkgs-22_11": [ + "nixpkgs" + ], + "nixpkgs-23_05": [ + "nixpkgs" + ], + "utils": "utils" + }, + "locked": { + "host": "gitlab.flyingcircus.io", + "lastModified": 1701453570, + "narHash": "sha256-2ai0rYD6o8fQgweO3zKqPbbQBy/SQC8Hs+ZTf+GeXuU=", + "owner": "flyingcircus", + "repo": "nixos-mailserver", + "rev": "b3d6e950b71f7f9a29268beb9de03003705765e8", + "type": "gitlab" + }, + "original": { + "host": "gitlab.flyingcircus.io", + "owner": "flyingcircus", + "repo": "nixos-mailserver", + "type": "gitlab" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1707338246, + "narHash": "sha256-1hxtQlAMcdqM+HgNih3y41P4hYw8VJjCMIK5Y+Y+KHw=", + "owner": "flyingcircusio", + "repo": "nixpkgs", + "rev": "ba2e1304f4d32379d4d884abe679df5ad06fbfa0", + "type": "github" + }, + "original": { + "owner": "flyingcircusio", + "ref": "nixos-23.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-lib": { + "locked": { + "dir": "lib", + "lastModified": 1706550542, + "narHash": "sha256-UcsnCG6wx++23yeER4Hg18CXWbgNpqNXcHIo5/1Y+hc=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "97b17f32362e475016f942bbdfda4a4a72a8a652", + "type": "github" + }, + "original": { + "dir": "lib", + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-regression": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, + "nixpkgs-stable": { + "locked": { + "lastModified": 1685801374, + "narHash": "sha256-otaSUoFEMM+LjBI1XL/xGB5ao6IwnZOXc47qhIgJe8U=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "c37ca420157f4abc31e26f436c1145f8951ff373", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-23.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "pre-commit-hooks": { + "inputs": { + "flake-compat": [ + "devenv", + "flake-compat" + ], + "flake-utils": "flake-utils", + "gitignore": "gitignore", + "nixpkgs": [ + "devenv", + "nixpkgs" + ], + "nixpkgs-stable": "nixpkgs-stable" + }, + "locked": { + "lastModified": 1704725188, + "narHash": "sha256-qq8NbkhRZF1vVYQFt1s8Mbgo8knj+83+QlL5LBnYGpI=", + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "rev": "ea96f0c05924341c551a797aaba8126334c505d2", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "type": "github" + } + }, + "root": { + "inputs": { + "devenv": "devenv", + "flake-parts": "flake-parts", + "nixos-mailserver": "nixos-mailserver", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "utils": { + "locked": { + "lastModified": 1605370193, + "narHash": "sha256-YyMTf3URDL/otKdKgtoMChu4vfVL3vCMkRqpGifhUn0=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "5021eac20303a61fafe17224c087f5519baed54d", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..436e70e11 --- /dev/null +++ b/flake.nix @@ -0,0 +1,202 @@ +# This flake is meant to be used with `nix develop --impure` +# to provide a dev shell for platform developers and release managers. + +# Platform dependencies are still read from +# release/versions.json, but the file is updated from flake inputs +# when update_nixpkgs / build_versions_json is used. + +# All former stand-alone scripts are now integrated into this flake. +# They can be executed from the dev shell: + +# ## Dev VM +# build_channels_dir (was part of ./dev-setup) +# nixos_repl + +# ## Release +# update_nixpkgs (was: update-nixpkgs.py) +# update_phps (was: up-nix-phps.sh) +# show_release_branch_status (was: fc-branch-diff-release.sh) +# perform_release (was: fc-release.sh) +# get_current_channel_url (was: fc-get-current-channel-url.sh) + +{ + description = "Flying Circus NixOS platform (dev/release tooling)"; + + inputs = { + nixpkgs.url = "github:flyingcircusio/nixpkgs/nixos-23.11"; + nixos-mailserver = { + url = "gitlab:flyingcircus/nixos-mailserver?host=gitlab.flyingcircus.io"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.nixpkgs-22_11.follows = "nixpkgs"; + inputs.nixpkgs-23_05.follows = "nixpkgs"; + }; + devenv = { + url = "github:cachix/devenv"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + flake-parts.url = "github:hercules-ci/flake-parts"; + }; + + outputs = inputs@{ flake-parts, ... }: + flake-parts.lib.mkFlake { inherit inputs; } + { + imports = [ + inputs.devenv.flakeModule + ./release/flake-part-linux-only-packages.nix + ]; + systems = [ "x86_64-linux" "x86_64-darwin" "aarch64-darwin" ]; + perSystem = { config, self', inputs', pkgs, lib, system, ... }: { + # Per-system attributes can be defined here. The self' and inputs' + # module parameters provide easy access to attributes of the same + # system. + + # We need our overlay here to get the right package versions. + # Other than that, it has no effect. + _module.args.pkgs = + let + inherit (builtins) elem getName; + nixpkgsConfig = import ./nixpkgs-config.nix; + in + import inputs.nixpkgs { + inherit system; + overlays = [ (import ./pkgs/overlay.nix) ]; + config = { + inherit (nixpkgsConfig) permittedInsecurePackages; + + allowUnfreePredicate = pkg: + elem (getName pkg) nixpkgsConfig.allowedUnfreePackageNames; + }; + }; + + packages = { + # These are packages that work on all systems. + # Also see release/flake-part-linux-only-packages.nix + + fcRelease = pkgs.writeShellApplication { + name = "perform_release"; + runtimeInputs = with pkgs; [ git coreutils ]; + text = (lib.readFile release/fc-release.sh); + }; + + fcBranchDiffRelease = pkgs.writeShellApplication { + name = "show_release_branch_status"; + runtimeInputs = with pkgs; [ gh git coreutils ]; + text = (lib.readFile release/fc-branch-diff-release.sh); + }; + + fcGetCurrentChannelUrl = pkgs.writeShellApplication { + name = "get_current_channel_url"; + runtimeInputs = with pkgs; [ curl ]; + text = (lib.readFile release/fc-get-current-channel-url.sh); + }; + + upNixPhps = pkgs.writeShellApplication { + name = "update_phps"; + excludeShellChecks = [ "SC2086" "SC2164" "SC2064" "SC2002" ]; + runtimeInputs = with pkgs; [ git curl jq ]; + text = (lib.readFile release/up-nix-phps.sh); + }; + + updateNixpkgs = + pkgs.writers.writePython3Bin + "update_nixpkgs" + { libraries = with pkgs.python3Packages; [ GitPython rich typer ]; } + (lib.readFile release/update-nixpkgs.py); + + versionsJson = pkgs.writeText "versions.json" (lib.generators.toJSON {} + { + nixpkgs = with inputs.nixpkgs; { + inherit rev; + hash = narHash; + owner = "flyingcircusio"; + repo = "nixpkgs"; + }; + nixos-mailserver = with inputs.nixos-mailserver; { + inherit rev; + hash = narHash; + url = "https://gitlab.flyingcircus.io/flyingcircus/nixos-mailserver.git/"; + fetchSubmodules = false; + deepClone = false; + leaveDotGit = false; + }; + } + ); + }; + + devenv.shells.default = + let + inherit (builtins) getEnv; + upstreams = { inherit (inputs) nixpkgs nixos-mailserver; }; + nixPathUpstreams = + lib.concatStringsSep ":" + (lib.mapAttrsToList (name: flake: "${name}=${flake.outPath}") upstreams); + NIX_PATH = "fc=${getEnv "PWD"}:${nixPathUpstreams}:nixos-config=/etc/nixos/configuration.nix"; + in + { + name = "fc-nixos-dev"; + env = { + inherit NIX_PATH; + }; + + packages = with pkgs; [ + jq + ] ++ (with self'.packages; [ + fcBranchDiffRelease + fcGetCurrentChannelUrl + fcRelease + upNixPhps + updateNixpkgs + ]); + + scripts = { + # This only works on Linux but I couldn't find an easy way to + # only build this script on Linux. It just produces an error + # message on Non-Linux because packageVersions is missing. + build_package_versions_json.exec = '' + jq < $(nix build .#packageVersions --print-out-paths) > release/package-versions.json + ''; + + build_versions_json.exec = '' + jq < $(nix build .#versionsJson --print-out-paths) > release/versions.json + ''; + + build_channels_dir.exec = '' + set -e + mkdir -p channels + if ! [[ -e channels/fc ]]; then + ln -s .. channels/fc + fi + '' + (lib.concatStringsSep "\n" (lib.mapAttrsToList (name: flake: '' + ln -sfT ${flake.outPath} channels/${name} + '') upstreams )); + + cat_package_versions_json.exec = '' + jq < $(nix build .#packageVersions --print-out-paths) + ''; + + dev_setup.exec = '' + build_channels_dir + + # -s gives us the absolute path without resolving symlinks. + NIX_PATH=`realpath -s channels` + + # preserve nixos-config + config=$(nix-instantiate --find-file nixos-config 2>/dev/null) || true + + if [[ -n "$config" ]]; then + NIX_PATH="$NIX_PATH:nixos-config=$config" + else + NIX_PATH="$NIX_PATH:nixos-config=$base/nixos" + fi + + echo "export NIX_PATH=$NIX_PATH" + ''; + + nixos_repl.exec = '' + sudo -E nix repl -f nixos/lib/nixos-repl.nix + ''; + }; + }; + }; + }; # end mkFlake +} diff --git a/get-package-versions.nix b/get-package-versions.nix deleted file mode 100644 index 915d6d9da..000000000 --- a/get-package-versions.nix +++ /dev/null @@ -1,19 +0,0 @@ -with builtins; - -let - pkgs = import {}; - lib = pkgs.lib; - pkgNamesToCheck = fromJSON (readFile ./important_packages.json); - -in pkgs.writeText "versions" (lib.generators.toJSON {} - (lib.listToAttrs - (map - (name: - let p = lib.attrByPath (lib.splitString "." name) null pkgs; - in lib.nameValuePair - name - (if (p != null && (hasAttr "name" p || hasAttr "pname" p)) - then { pname = lib.getName p; name = p.name or ""; version = (lib.getVersion p); } - else {})) - pkgNamesToCheck)) -) diff --git a/nixos-repl.nix b/nixos/lib/nixos-repl.nix similarity index 94% rename from nixos-repl.nix rename to nixos/lib/nixos-repl.nix index ae64560af..551051b46 100644 --- a/nixos-repl.nix +++ b/nixos/lib/nixos-repl.nix @@ -23,7 +23,7 @@ let inherit (nixos) options; etc = printEtcFile options; - replHelpers = pkgs.callPackage nixos/lib/repl-helpers.nix {}; + replHelpers = pkgs.callPackage ./repl-helpers.nix {}; inherit (replHelpers) printEtcFile format print; in builtins // nixos.config // { diff --git a/pkgs/fc/ceph/fc/ceph/maintenance.py b/pkgs/fc/ceph/fc/ceph/maintenance.py index 44166a161..1f0ca770b 100644 --- a/pkgs/fc/ceph/fc/ceph/maintenance.py +++ b/pkgs/fc/ceph/fc/ceph/maintenance.py @@ -4,9 +4,8 @@ import subprocess import sys -import fc.util.directory - import fc.ceph.images +import fc.util.directory from fc.ceph.api import Cluster, Pools diff --git a/pkgs/fc/ceph/fc/ceph/tests/test_ceph.py b/pkgs/fc/ceph/fc/ceph/tests/test_ceph.py index cd5e5ceda..7dd22a354 100644 --- a/pkgs/fc/ceph/fc/ceph/tests/test_ceph.py +++ b/pkgs/fc/ceph/fc/ceph/tests/test_ceph.py @@ -1,12 +1,11 @@ import collections +import fc.ceph.api.cluster +import fc.ceph.maintenance import fc.util.configfile import fc.util.directory import mock import pytest - -import fc.ceph.api.cluster -import fc.ceph.maintenance from fc.ceph.api.rbdimage import RBDImage diff --git a/pkgs/fc/ceph/fc/ceph/tests/test_main.py b/pkgs/fc/ceph/fc/ceph/tests/test_main.py index 42d60d0b7..0e3d1a5fc 100644 --- a/pkgs/fc/ceph/fc/ceph/tests/test_main.py +++ b/pkgs/fc/ceph/fc/ceph/tests/test_main.py @@ -1,6 +1,5 @@ -import pytest - import fc.ceph.main +import pytest def test_main(): diff --git a/release/default.nix b/release/default.nix index 3b3a5fbe3..587e6b236 100644 --- a/release/default.nix +++ b/release/default.nix @@ -133,7 +133,7 @@ let overlayPkgNames = getDottedPackageNames overlay []; overlayPkgNamesToTest = lib.subtractLists excludedPkgNames overlayPkgNames; - importantPkgNames = fromJSON (readFile ../important_packages.json); + importantPkgNames = fromJSON (readFile ../release/important_packages.json); importantPkgNamesToTest = lib.subtractLists excludedPkgNames importantPkgNames; # Results looks like: [ { python3Packages.requests.x86_64-linux = ; } ] diff --git a/fc-branch-diff-release.sh b/release/fc-branch-diff-release.sh similarity index 77% rename from fc-branch-diff-release.sh rename to release/fc-branch-diff-release.sh index 92363ff0d..36c0fb44b 100755 --- a/fc-branch-diff-release.sh +++ b/release/fc-branch-diff-release.sh @@ -1,16 +1,14 @@ -#! /usr/bin/env nix-shell -#! nix-shell -i bash --pure -p git gh coreutils - -nixos_version=$(< nixos-version) +#!/usr/bin/env bash +nixos_version=$(< release/nixos-version) dev="fc-${nixos_version}-dev" stag="fc-${nixos_version}-staging" prod="fc-${nixos_version}-production" export PAGER= -dev_commit=$(git rev-parse $dev) -stag_commit=$(git rev-parse $stag) -prod_commit=$(git rev-parse $prod) +dev_commit=$(git rev-parse "$dev") +stag_commit=$(git rev-parse "$stag") +prod_commit=$(git rev-parse "$prod") num_dev_prod_commits=$(git cherry "$prod" "$dev" | wc -l) diff --git a/release/fc-get-current-channel-url.sh b/release/fc-get-current-channel-url.sh new file mode 100755 index 000000000..df3defdcb --- /dev/null +++ b/release/fc-get-current-channel-url.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +branch="${1:-production}" +nixos_version=$(< release/nixos-version) +jobset="fc-$nixos_version-$branch" + +echo "Checking $jobset" + +curl --silent --head "https://hydra.flyingcircus.io/channel/custom/flyingcircus/${jobset}/release/nixexprs.tar.xz" | grep "[Ll]ocation" | cut -f 2 -d " " diff --git a/fc-release.sh b/release/fc-release.sh similarity index 83% rename from fc-release.sh rename to release/fc-release.sh index 9b9f469f0..1e2458689 100755 --- a/fc-release.sh +++ b/release/fc-release.sh @@ -1,6 +1,4 @@ #!/usr/bin/env bash -set -e - releaseid="${1:?no release id given}" if ! echo "$releaseid" | grep -Eq '^[0-9]{4}_[0-9]{3}$'; then @@ -8,33 +6,33 @@ if ! echo "$releaseid" | grep -Eq '^[0-9]{4}_[0-9]{3}$'; then exit 64 fi -nixos_version=$(< nixos-version) +nixos_version=$(< release/nixos-version) dev="fc-${nixos_version}-dev" stag="fc-${nixos_version}-staging" prod="fc-${nixos_version}-production" echo "$0: performing release based on $stag" -if ! git remote -v | grep -Eq "^origin\s.*github.com.flyingcircusio/fc-nixos" -then +if ! git remote -v | grep -Eq "^origin\s.*github.com.flyingcircusio/fc-nixos"; then echo "$0: please perform release in a clean checkout with proper origin" >&2 exit 64 fi + git fetch origin --tags --prune -git checkout $dev +git checkout "$dev" git merge --ff-only # expected to fail on unclean/unpushed workdirs -git checkout $stag +git checkout "$stag" git merge --ff-only -git checkout $prod +git checkout "$prod" git merge --ff-only msg="Merge branch '$stag' into $prod for release $releaseid" -git merge -m "$msg" $stag +git merge -m "$msg" "$stag" -git checkout $dev +git checkout "$dev" msg="Backmerge branch '$prod' into $dev for release $releaseid" -git merge -m "$msg" $prod +git merge -m "$msg" "$prod" echo "$0: committed changes:" PAGER='' git log --graph --decorate --format=short -n3 diff --git a/release/flake-part-linux-only-packages.nix b/release/flake-part-linux-only-packages.nix new file mode 100644 index 000000000..44f0df5ac --- /dev/null +++ b/release/flake-part-linux-only-packages.nix @@ -0,0 +1,30 @@ +{ withSystem, ... }: + +with builtins; + +let + pkgNamesToCheck = fromJSON (readFile ./important_packages.json); +in +{ + flake = { + # Packages that only build on Linux. + packages.x86_64-linux = withSystem "x86_64-linux" + ({ pkgs, ... }: + let + inherit (pkgs) lib; + in { + # Evaluating the kernel expression (which we need for finding the version) is only possible on Linux. + packageVersions = pkgs.writeText "package-versions" (lib.generators.toJSON {} + (lib.listToAttrs + (map + (name: + let p = lib.attrByPath (lib.splitString "." name) null pkgs; + in lib.nameValuePair + name + (if (p != null && (hasAttr "name" p || hasAttr "pname" p)) + then { pname = lib.getName p; name = p.name or ""; version = (lib.getVersion p); } + else {})) + pkgNamesToCheck))); + }); + }; +} diff --git a/important_packages.json b/release/important_packages.json similarity index 100% rename from important_packages.json rename to release/important_packages.json diff --git a/nixos-version b/release/nixos-version similarity index 100% rename from nixos-version rename to release/nixos-version diff --git a/package-versions.json b/release/package-versions.json similarity index 100% rename from package-versions.json rename to release/package-versions.json diff --git a/up-nix-phps.sh b/release/up-nix-phps.sh similarity index 91% rename from up-nix-phps.sh rename to release/up-nix-phps.sh index 7fd4553dd..12d8c5a37 100755 --- a/up-nix-phps.sh +++ b/release/up-nix-phps.sh @@ -1,7 +1,4 @@ -#!/usr/bin/env nix-shell -#! nix-shell -i bash -p git curl jq - -set -euo pipefail +#!/usr/bin/env bash T=$(mktemp -d) diff --git a/update-nixpkgs.py b/release/update-nixpkgs.py similarity index 78% rename from update-nixpkgs.py rename to release/update-nixpkgs.py index a9f1de2df..00e164568 100755 --- a/update-nixpkgs.py +++ b/release/update-nixpkgs.py @@ -1,17 +1,14 @@ -#!/usr/bin/env nix-shell -#! nix-shell -i python -p "python310.withPackages (p: with p; [ GitPython rich typer ])" import json import re import subprocess -import sys from dataclasses import dataclass -from enum import Enum +from enum import StrEnum from pathlib import Path from typing import Optional from git import Repo from rich import print -from typer import Argument, Option, Typer, confirm, echo +from typer import Argument, Option, Typer, confirm PKG_UPDATE_RE = re.compile( r"(?P.+): " @@ -19,10 +16,14 @@ r"(?P.*)" ) +NIXOS_VERSION_PATH = "release/nixos-version" +PACKAGE_VERSIONS_PATH = "release/package-versions.json" +VERSIONS_PATH = "release/versions.json" + app = Typer() -class NixOSVersion(str, Enum): +class NixOSVersion(StrEnum): NIXOS_2211 = "nixos-22.11" NIXOS_2305 = "nixos-23.05" NIXOS_2311 = "nixos-23.11" @@ -116,22 +117,6 @@ def rebase_nixpkgs(nixpkgs_repo: Repo, nixos_version: NixOSVersion): nixpkgs_repo.git.push(force_with_lease=True) -def prefetch_nixpkgs(nixos_version: str) -> dict[str, str]: - prefetch_cmd = [ - "/run/current-system/sw/bin/nix-prefetch-github", - "flyingcircusio", - "nixpkgs", - "--rev", - nixos_version, - ] - - print("Prefetching nixpkgs, this takes some time...") - prefetch_proc = run_on_hydra(*prefetch_cmd) - prefetch_result = json.loads(prefetch_proc.stdout) - print(prefetch_result) - return prefetch_result - - def version_diff_lines(old_versions, new_versions): lines = [] for pkg_name in old_versions: @@ -152,9 +137,10 @@ def version_diff_lines(old_versions, new_versions): return lines -def update_package_versions_json(package_versions_path: Path): +def update_package_versions_json(workdir_path: Path): basedir = "$XDG_RUNTIME_DIR" - local_path = package_versions_path.parent.absolute() + package_versions_path = workdir_path / PACKAGE_VERSIONS_PATH + local_path = workdir_path.absolute() dest = f"{basedir}/{local_path.name}/" rsync_cmd = [ "rsync", @@ -167,7 +153,7 @@ def update_package_versions_json(package_versions_path: Path): print("rsync: ", " ".join(rsync_cmd)) subprocess.run(rsync_cmd, check=True) proc = run_on_hydra( - f"(cd {dest}; eval $(./dev-setup); set pipefail; nix-build ./get-package-versions.nix | xargs cat)" + f"(cd {dest};nix develop --impure --command cat_package_versions_json)" ) old_versions = json.loads(package_versions_path.read_text()) new_versions = json.loads(proc.stdout) @@ -178,8 +164,10 @@ def update_package_versions_json(package_versions_path: Path): package_versions_path.write_text(json.dumps(new_versions, indent=2) + "\n") -def get_interesting_commit_msgs(nixpkgs_repo, old_rev, new_rev): - with open("package-versions.json") as f: +def get_interesting_commit_msgs( + workdir_path: Path, nixpkgs_repo, old_rev, new_rev +): + with open(workdir_path / PACKAGE_VERSIONS_PATH) as f: package_versions = json.load(f) version_range = f"{old_rev}..{new_rev}" print(f"comparing {version_range}") @@ -239,18 +227,6 @@ def filter_and_merge_commit_msgs(msgs): return out_msgs -def update_versions_json(versions_json_path: Path, rev, hash): - with open(versions_json_path) as f: - versions_json = json.load(f) - - versions_json["nixpkgs"]["rev"] = rev - versions_json["nixpkgs"]["hash"] = hash - - with open(versions_json_path, "w") as wf: - json.dump(versions_json, wf, indent=2) - wf.write("\n") - - def format_fcio_commit_msg( msgs: list[str], ticket_number: Optional[str] ) -> str: @@ -273,29 +249,38 @@ def update_fc_nixos( nixpkgs_repo: Repo, fc_nixos_repo: Repo, ticket_number: str, - prefetch_json: dict[str, str], ): workdir_path = Path(fc_nixos_repo.working_dir) - versions_json_path = workdir_path / "versions.json" - package_versions_path = workdir_path / "package-versions.json" + flake_lock_path = workdir_path / "flake.lock" + versions_json_path = workdir_path / VERSIONS_PATH + package_versions_path = workdir_path / PACKAGE_VERSIONS_PATH with open(versions_json_path) as f: versions_json = json.load(f) - old_rev = versions_json["nixpkgs"]["rev"] - new_rev = str(nixpkgs_repo.head.commit) + print(f"Updating {flake_lock_path}...") + subprocess.run(["nix", "flake", "update"]) - update_versions_json(versions_json_path, new_rev, prefetch_json["hash"]) + print(f"Building {versions_json_path}") + subprocess.run(["build_versions_json"]) print() print("-" * 80) - update_package_versions_json(package_versions_path) + print(f"Updating {package_versions_path}...") + update_package_versions_json(workdir_path) fc_nixos_repo.index.add( - [str(versions_json_path), str(package_versions_path)] + [ + str(flake_lock_path), + str(versions_json_path), + str(package_versions_path), + ] ) + old_rev = versions_json["nixpkgs"]["rev"] + new_rev = str(nixpkgs_repo.head.commit) + interesting_msgs = get_interesting_commit_msgs( - nixpkgs_repo, old_rev, new_rev + workdir_path, nixpkgs_repo, old_rev, new_rev ) print("All matching new commit messages (before filter & merge):") @@ -313,12 +298,13 @@ def update_fc_nixos( if ticket_number: do_commit = confirm( - f"Create feature branch ${feature_branch_name} and commit fc-nixos now?", + f"Create feature branch {feature_branch_name} " + "and commit fc-nixos now?", default=True, ) else: do_commit = confirm( - f"Commit to current fc-nixos now?", + "Commit to current fc-nixos now?", default=True, ) if do_commit: @@ -336,7 +322,7 @@ class Context: nixpkgs_path: Path -context: Context +context: Context | None = None @app.callback(no_args_is_help=True) @@ -351,7 +337,7 @@ def update_nixpkgs( ): global context if not nixos_version: - version_str = (fc_nixos_path / "nixos-version").read_text().strip() + version_str = (fc_nixos_path / NIXOS_VERSION_PATH).read_text().strip() nixos_version = NixOSVersion("nixos-" + version_str) context = Context(nixos_version, fc_nixos_path, nixpkgs_path) @@ -364,14 +350,7 @@ def nixpkgs(): @app.command() def package_versions(): - update_package_versions_json( - context.fc_nixos_path / "package-versions.json" - ) - - -@app.command() -def prefetch(): - print(prefetch_nixpkgs(context.nixos_version)) + update_package_versions_json(context.fc_nixos_path) @app.command() @@ -379,11 +358,11 @@ def version_diff( old_fc_nixos_path: Path = Argument(..., dir_okay=True, file_okay=False) ): """ - Shows package changes between the current (new) fc-nixos work tree and another (old) - one based on package-versions.json. + Shows package changes between the current (new) fc-nixos work tree and + another (old) one based on package-versions.json. """ - package_versions_path = context.fc_nixos_path / "package-versions.json" - old_package_versions_path = old_fc_nixos_path / "package-versions.json" + package_versions_path = context.fc_nixos_path / PACKAGE_VERSIONS_PATH + old_package_versions_path = old_fc_nixos_path / PACKAGE_VERSIONS_PATH old_versions = json.loads(old_package_versions_path.read_text()) new_versions = json.loads(package_versions_path.read_text()) @@ -402,8 +381,7 @@ def fc_nixos( ticket_number = ticket_number.removeprefix("PL-") nixpkgs_repo = Repo(context.nixpkgs_path) fc_nixos_repo = Repo(context.fc_nixos_path) - prefetch_json = prefetch_nixpkgs(context.nixos_version) - update_fc_nixos(nixpkgs_repo, fc_nixos_repo, ticket_number, prefetch_json) + update_fc_nixos(nixpkgs_repo, fc_nixos_repo, ticket_number) if __name__ == "__main__": diff --git a/versions.json b/release/versions.json similarity index 100% rename from versions.json rename to release/versions.json diff --git a/release/versions.nix b/release/versions.nix new file mode 100644 index 000000000..d8a3b727d --- /dev/null +++ b/release/versions.nix @@ -0,0 +1,55 @@ +{ pkgs ? import {} }: + +with pkgs.lib; + +let + versions = importJSON ./versions.json; + + channels = + mapAttrs ( + name: repoInfo: + # Hydra expects fixed length rev ids + assert stringLength repoInfo.rev == 40; + + if repoInfo ? url then + pkgs.fetchgit repoInfo // { + name = "${name}-${substring 0 11 repoInfo.rev}"; + } + else + pkgs.fetchFromGitHub { + inherit (repoInfo) owner repo rev hash; + name = "${name}-${substring 0 11 repoInfo.rev}"; + }) + versions; + + arrangeChannels = + builtins.toFile "arrange-channels.sh" '' + mkdir $out + set -- ''${channels[@]} + # 1=name 2=path + while [[ -n "$1" && -n "$2" ]]; do + ln -s $2 $out/"$1" + shift 2 + done + ''; + +in +assert channels ? "nixpkgs"; +let + pkgs = import channels.nixpkgs {}; +in +# export "nixos-18_09" instead of "nixos-18.09" for example +(mapAttrs' (name: val: nameValuePair (replaceStrings [ "." ] [ "_" ] name) val) + channels) +// +{ + allUpstreams = builtins.derivation { + args = [ "-e" arrangeChannels ]; + builder = pkgs.stdenv.shell; + channels = mapAttrsToList (name: path: "${name} ${path}") channels; + name = "all-upstream-sources"; + PATH = with pkgs; makeBinPath [ coreutils ]; + preferLocalBuild = true; + system = builtins.currentSystem; + }; +} diff --git a/shell.nix b/shell.nix deleted file mode 100644 index ec70f2eee..000000000 --- a/shell.nix +++ /dev/null @@ -1,21 +0,0 @@ -{ pkgs ? import {} }: -with builtins; -let - lib = pkgs.lib; - channels = (import ./versions.nix { }); - nixPathUpstreams = - lib.concatStringsSep - ":" - (lib.mapAttrsToList (name: channel: "${name}=${channel}") channels); - - nixosRepl = pkgs.writeShellScriptBin "nixos-repl" '' - sudo -E nix repl nixos-repl.nix - ''; - -in pkgs.mkShell { - name = "fc-nixos"; - shellHook = '' - export NIX_PATH="fc=${toString ./.}:${nixPathUpstreams}:nixos-config=/etc/nixos/configuration.nix" - export PATH=$PATH:${nixosRepl}/bin - ''; -} diff --git a/versions.nix b/versions.nix deleted file mode 100644 index d8a3b727d..000000000 --- a/versions.nix +++ /dev/null @@ -1,55 +0,0 @@ -{ pkgs ? import {} }: - -with pkgs.lib; - -let - versions = importJSON ./versions.json; - - channels = - mapAttrs ( - name: repoInfo: - # Hydra expects fixed length rev ids - assert stringLength repoInfo.rev == 40; - - if repoInfo ? url then - pkgs.fetchgit repoInfo // { - name = "${name}-${substring 0 11 repoInfo.rev}"; - } - else - pkgs.fetchFromGitHub { - inherit (repoInfo) owner repo rev hash; - name = "${name}-${substring 0 11 repoInfo.rev}"; - }) - versions; - - arrangeChannels = - builtins.toFile "arrange-channels.sh" '' - mkdir $out - set -- ''${channels[@]} - # 1=name 2=path - while [[ -n "$1" && -n "$2" ]]; do - ln -s $2 $out/"$1" - shift 2 - done - ''; - -in -assert channels ? "nixpkgs"; -let - pkgs = import channels.nixpkgs {}; -in -# export "nixos-18_09" instead of "nixos-18.09" for example -(mapAttrs' (name: val: nameValuePair (replaceStrings [ "." ] [ "_" ] name) val) - channels) -// -{ - allUpstreams = builtins.derivation { - args = [ "-e" arrangeChannels ]; - builder = pkgs.stdenv.shell; - channels = mapAttrsToList (name: path: "${name} ${path}") channels; - name = "all-upstream-sources"; - PATH = with pkgs; makeBinPath [ coreutils ]; - preferLocalBuild = true; - system = builtins.currentSystem; - }; -} diff --git a/versions.nix b/versions.nix new file mode 120000 index 000000000..f0f87728c --- /dev/null +++ b/versions.nix @@ -0,0 +1 @@ +release/versions.nix \ No newline at end of file