added script to upload modules to binary cache; split out lkl

Author: airportyh

.gitignore

@@ -2,3 +2,5 @@ result
 build
 .envrc
 .direnv
+.netrc
+nix_build_cache_signing_key

.semaphore/build-nixmodules.yml

@@ -14,6 +14,7 @@ auto_cancel:
 global_job_config:
   secrets:
     - name: gcp-goval
+    - name: nix-build-cache-auth
   env_vars:
     - name: NIX_FLAGS
       value: "--extra-experimental-features nix-command --extra-experimental-features flakes --extra-experimental-features discard-references"
@@ -34,7 +35,8 @@ blocks:
       jobs:
         - name: build nixmodules disk image
           commands:
-            - ./scripts/build_disk_image.sh > nixmodules_build.log 2>&1
+            - scripts/setup_nix_binary_cache.py
+            - scripts/build_disk_image.sh 2>&1 | tee nixmodules_build.log
       epilogue:
         always:
           commands:

pkgs/bundle-image/default.nix

@@ -12,20 +12,13 @@
 , upgrade-maps
 , active-modules
 , fetchFromGitHub
+, pkgs
 }:
 
 let
   label = "nixmodules-${revstring}";
   registry = ../../modules.json;
-  # wating for upstream to include our patch: https://github.com/lkl/linux/pull/532 and https://github.com/lkl/linux/pull/534
-  lkl' = lkl.overrideAttrs (oldAttrs: {
-    src = fetchFromGitHub {
-      owner = "numtide";
-      repo = "linux-lkl";
-      rev = "2cbcbd26044f72e47740588cfa21bf0e7b698262";
-      sha256 = "sha256-i7uc69bF84kkS7MahpgJ2EnWZLNah+Ees2oRGMzIee0=";
-    };
-  });
+  lkl' = pkgs.callPackage ../lkl { };
 in
 
 derivation {

pkgs/default.nix

@@ -57,6 +57,8 @@ rec {
 
   rev_long = pkgs.writeText "rev_long" revstring_long;
 
+  lkl = pkgs.callPackage ./lkl { };
+
   active-modules = import ./active-modules {
     inherit pkgs;
     inherit self;
@@ -81,6 +83,15 @@ rec {
     };
   };
 
+  all-historical-modules = mapAttrs
+    (moduleId: module:
+      let
+        flake = builtins.getFlake "github:replit/nixmodules/${module.commit}";
+        shortModuleId = elemAt (strings.splitString ":" moduleId) 0;
+      in
+      flake.modules.${shortModuleId})
+    all-modules;
+
   bundle-squashfs = bundle-squashfs-fn {
     moduleIds = [ "python-3.10" "nodejs-18" ];
     inherit upgrade-maps;

pkgs/lkl/default.nix

@@ -0,0 +1,10 @@
+# waiting for upstream to include our patch: https://github.com/lkl/linux/pull/532 and https://github.com/lkl/linux/pull/534
+{ lkl, fetchFromGitHub }:
+lkl.overrideAttrs (oldAttrs: {
+  src = fetchFromGitHub {
+    owner = "numtide";
+    repo = "linux-lkl";
+    rev = "2cbcbd26044f72e47740588cfa21bf0e7b698262";
+    sha256 = "sha256-i7uc69bF84kkS7MahpgJ2EnWZLNah+Ees2oRGMzIee0=";
+  };
+})

scripts/cache_all_modules.py

@@ -0,0 +1,17 @@
+import json
+import subprocess
+
+def main():
+  file = open('modules.json')
+  modules = json.load(file)
+  file.close()
+  for module_id in modules.keys():
+    cmd = [
+      'scripts/cache_nix_build.sh', '.#all-historical-modules."%s"' % module_id
+    ]
+    print(" ".join(cmd))
+    subprocess.check_output(cmd, stderr=subprocess.PIPE)
+    print('%s done' % " ".join(cmd))
+
+if __name__ == '__main__':
+  main()

scripts/cache_nix_build.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Builds and uploads a nix package and its run-time and build-time dependencies to our Nix cahce.
+
+set -eo pipefail
+
+nix build $@
+
+nix-store -qR --include-outputs $(nix-store -qd $(nix path-info $@)) \
+	| grep -v '\.drv$' \
+	| xargs nix copy --to https://nix-build-cache.replit.com?secret-key=./nix_build_cache_signing_key
+
+

scripts/setup_nix_binary_cache.py

@@ -0,0 +1,62 @@
+#! /usr/bin/env python3
+import os
+import pathlib
+
+def main():
+  auth = get_auth()
+  net_rc_path = create_netrc(auth)
+  create_signing_key_file(auth)
+  create_nix_conf(auth, net_rc_path)
+
+def create_signing_key_file(auth):
+  file_contents = "replit-internal-nixcache:%s" % auth["signingKey"]
+  signing_key_file_path = os.path.abspath("nix_build_cache_signing_key")
+  f = open(signing_key_file_path, "w")
+  f.write(file_contents)
+  f.close()
+  print("Wrote %s" % signing_key_file_path)
+
+def create_nix_conf(auth, netrc_path):
+  homedir = os.getenv("HOME")
+  nix_conf = "\n".join([
+    "substituters = https://cache.nixos.org/ https://nix-build-cache.replit.com/",
+    "trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= replit-internal-nixcache:%s" % auth["publicKey"],
+    "netrc-file = %s" % netrc_path,
+    "experimental-features = nix-command flakes discard-references"
+  ])
+  nix_conf_dir_path = os.path.abspath("%s/.config/nix/" % homedir)
+  pathlib.Path(nix_conf_dir_path).mkdir(parents=True, exist_ok=True)
+  nix_conf_path = os.path.abspath("%s/nix.conf" % nix_conf_dir_path)
+  f = open(nix_conf_path, "w")
+  f.write(nix_conf)
+  f.close()
+  print("Wrote %s" % nix_conf_path)
+
+def create_netrc(auth):
+  netrc = "\n".join([
+    "machine nix-build-cache.replit.com",
+    "login %s" % auth["login"],
+    "password %s" % auth["password"],
+    ""
+  ])
+  netrc_path = os.path.abspath(".netrc")
+  f = open(netrc_path, "w")
+  f.write(netrc)
+  f.close()
+  print("Wrote %s" % netrc_path)
+  return netrc_path
+
+def get_auth():
+  login = os.getenv('NIX_BUILD_CACHE_LOGIN')
+  password = os.getenv('NIX_BUILD_CACHE_PASSWORD')
+  publicKey = os.getenv('NIX_BUILD_CACHE_PUBLIC_KEY')
+  signingKey = os.getenv('NIX_BUILD_CACHE_SIGNING_KEY')
+  return {
+    'login': login,
+    'password': password,
+    'publicKey': publicKey,
+    'signingKey': signingKey
+  }
+
+if __name__ == '__main__':
+  main()