Merge pull request flyingcircusio#785 from flyingcircusio/PL-131736-enc-release-metadata

Save release ENC metadata, use it in motd

Author: dpausp

nixos/platform/default.nix

@@ -292,6 +292,7 @@ in {
     '';
 
     environment.etc."fcio_environment_name".text = config.flyingcircus.enc.parameters.environment or "";
+    environment.etc."fcio_release".text = config.flyingcircus.platform.release.release_name or "pre";
 
     flyingcircus = {
       enc_services = enc_services;

nixos/platform/enc.nix

@@ -64,6 +64,12 @@ with lib;
       description = "Where to find the ENC service clients json file.";
     };
 
+    releasesPath = mkOption {
+      default = /etc/nixos/releases.json;
+      defaultText = "/etc/nixos/releases.json";
+      type = path;
+      description = "Where to find the releases json file.";
+    };
     systemStatePath = mkOption {
       default = /etc/nixos/system_state.json;
       type = path;
@@ -84,6 +90,20 @@ with lib;
       '';
     };
 
+    platform = {
+      release = mkOption {
+        readOnly = true;
+        default = (config.flyingcircus.platform.knownReleases.${config.system.nixos.label} or {});
+        defaultText = "Platform release metadata loaded from the file at `releasesPath`";
+      };
+
+      knownReleases = mkOption {
+        internal = true;
+        readOnly = true;
+        default = fclib.jsonFromFile cfg.releasesPath "{}";
+        defaultText = "Metadata for all known releases from the file at `releasesPath`";
+      };
+    };
   };
 
   config = {

nixos/platform/shell.nix

@@ -12,6 +12,9 @@ let
 in
 {
   config = {
+
+    environment.etc.motd.text = config.users.motd;
+
     environment.interactiveShellInit = ''
       export TMOUT=43200
     '';
@@ -64,15 +67,16 @@ in
     );
 
     users.motd = let
-      has_release_name = parameters ? release_name;
+      inherit (config.flyingcircus.platform) release;
+      isStableRelease = release != {};
     in
     ''
       Welcome to the Flying Circus!
 
       Status:     https://status.flyingcircus.io/
       Docs:       https://doc.flyingcircus.io/
-      ${ opt (parameters ? release_changelog ) ("ChangeLog:  " + parameters.release_changelog)}
-      Release:    ${ opt has_release_name "${parameters.release_name} (" + config.system.nixos.label + opt has_release_name ")"}
+      Release:    ${opt isStableRelease "${release.release_name} (" + config.system.nixos.label + opt isStableRelease ")"}
+      ${opt isStableRelease ("ChangeLog:  " + release.release_changelog)}
 
     '' +
     (opt (enc ? name && parameters ? location && parameters ? environment)

pkgs/fc/agent/fc/maintenance/activity/tests/test_update.py

@@ -20,6 +20,9 @@
 CURRENT_BUILD = 93111
 NEXT_BUILD = 93222
 NEXT_NEXT_BUILD = 93333
+CURRENT_RELEASE = "2021_002"
+NEXT_RELEASE = "2021_003"
+CHANGELOG_URL = "https://doc.flyingcircus.io/platform/changes/2021/r003.html"
 CURRENT_CHANNEL_URL = f"https://hydra.flyingcircus.io/build/{CURRENT_BUILD}/download/1/nixexprs.tar.xz"
 NEXT_CHANNEL_URL = f"https://hydra.flyingcircus.io/build/{NEXT_BUILD}/download/1/nixexprs.tar.xz"
 ENVIRONMENT = "fc-21.05-production"
@@ -37,7 +40,7 @@
     "stop": ["postgresql.service"],
 }
 
-CHANGELOG = textwrap.dedent(
+SUMMARY = textwrap.dedent(
     f"""\
     System update: {CURRENT_VERSION} -> {NEXT_VERSION}
 
@@ -47,21 +50,26 @@
     Restart: telegraf
     Reload: nginx
 
+    Release: {CURRENT_RELEASE} -> {NEXT_RELEASE}
+    ChangeLog: {CHANGELOG_URL}
     Environment: {ENVIRONMENT} (unchanged)
     Build number: {CURRENT_BUILD} -> {NEXT_BUILD}
     Channel URL: {NEXT_CHANNEL_URL}"""
 )
 
 SERIALIZED_ACTIVITY = f"""\
 !!python/object:fc.maintenance.activity.update.UpdateActivity
+changelog_url: {CHANGELOG_URL}
 current_channel_url: https://hydra.flyingcircus.io/build/93111/download/1/nixexprs.tar.xz
 current_environment: fc-21.05-production
 current_kernel: 5.10.45
+current_release: '{CURRENT_RELEASE}'
 current_system: {CURRENT_SYSTEM_PATH}
 current_version: 21.05.1233.a9cc58d
 next_channel_url: https://hydra.flyingcircus.io/build/93222/download/1/nixexprs.tar.xz
 next_environment: fc-21.05-production
 next_kernel: 5.10.50
+next_release: '{NEXT_RELEASE}'
 next_system: {NEXT_SYSTEM_PATH}
 next_version: 21.05.1235.bacc11d
 reboot_needed: !!python/object/apply:fc.maintenance.activity.RebootType
@@ -82,12 +90,15 @@
 def activity(logger, nixos_mock):
     activity = UpdateActivity(next_channel_url=NEXT_CHANNEL_URL, log=logger)
     activity.current_channel_url = CURRENT_CHANNEL_URL
+    activity.changelog_url = CHANGELOG_URL
     activity.current_environment = ENVIRONMENT
+    activity.current_release = CURRENT_RELEASE
     activity.current_version = CURRENT_VERSION
     activity.current_kernel = CURRENT_KERNEL_VERSION
     activity.next_channel_url = NEXT_CHANNEL_URL
     activity.next_environment = ENVIRONMENT
     activity.next_kernel = NEXT_KERNEL_VERSION
+    activity.next_release = NEXT_RELEASE
     activity.next_system = NEXT_SYSTEM_PATH
     activity.next_version = NEXT_VERSION
     activity.reboot_needed = RebootType.WARM
@@ -245,7 +256,7 @@ def test_update_activity_prepare(log, logger, tmp_path, activity, nixos_mock):
     assert (
         activity.reboot_needed == RebootType.WARM
     ), "expected warm reboot request"
-    assert activity.changelog == CHANGELOG
+    assert activity.summary == SUMMARY
 
     assert log.has(
         "update-prepare-start",

pkgs/fc/agent/fc/maintenance/activity/update.py

@@ -36,9 +36,12 @@ def __init__(
         super().__init__()
         self.next_environment = next_environment
         self.next_channel_url = next_channel_url
+        self.changelog_url = None
         self.current_system = None
         self.next_system = None
         self.current_channel_url = None
+        self.current_release = None
+        self.next_release = None
         self.current_version = None
         self.next_version = None
         self.current_environment = None
@@ -249,7 +252,7 @@ def run(self):
         self.returncode = 0
 
     @property
-    def changelog(self):
+    def summary(self):
         """
         A human-readable summary of what will be changed by this update.
         Includes possible reboots, significant unit state changes (start, stop,
@@ -271,6 +274,13 @@ def changelog(self):
             msg.extend(unit_change_lines)
             msg.append("")
 
+        if self.next_release:
+            msg.append(
+                f"Release: {self.current_release} -> {self.next_release}"
+            )
+        if self.changelog_url:
+            msg.append(f"ChangeLog: {self.changelog_url}")
+
         if self.current_environment != self.next_environment:
             msg.append(
                 f"Environment: {self.current_environment} -> {self.next_environment}"
@@ -293,7 +303,7 @@ def changelog(self):
 
     @property
     def comment(self):
-        return self.changelog
+        return self.summary
 
     def merge(self, other: Activity) -> ActivityMergeResult:
         if not isinstance(other, UpdateActivity):

pkgs/fc/agent/fc/maintenance/maintenance.py

@@ -187,7 +187,7 @@ def request_update(log, enc, current_requests) -> Optional[Request]:
                 "Update preparation was successful. This update will apply "
                 "changes to the system."
             ),
-            _output=activity.changelog,
+            _output=activity.summary,
             current_channel=activity.current_channel_url,
             next_channel=activity.next_channel_url,
         )

pkgs/fc/agent/fc/util/enc.py

@@ -9,7 +9,9 @@
 from pathlib import Path
 
 import structlog
+from fc.util import nixos
 from fc.util.directory import connect
+from fc.util.time_date import utcnow
 
 structlog = structlog.get_logger()
 
@@ -112,15 +114,15 @@ def update_enc_nixos_config(log, enc, enc_path):
             content=hashlib.sha256(config.encode("utf-8")).hexdigest(),
         )
         target = os.path.join(basedir, filename)
-        conditional_update(target, config, encode_json=False)
+        conditional_update(target, config, mode=0o640, encode_json=False)
         os.chown(target, -1, sudo_srv)
         previous_files -= {filename}
     for filename in previous_files:
         log.info("remove-stale-enc-nixos-config", filename=filename)
         os.unlink(os.path.join(basedir, filename))
 
 
-def conditional_update(filename, data, encode_json=True):
+def conditional_update(filename, data, mode=0o640, encode_json=True):
     """Updates JSON file on disk only if there is different content."""
     with tempfile.NamedTemporaryFile(
         mode="w",
@@ -134,7 +136,7 @@ def conditional_update(filename, data, encode_json=True):
         else:
             tf.write(data)
         tf.write("\n")
-        os.chmod(tf.fileno(), 0o640)
+        os.chmod(tf.fileno(), mode)
     if not (os.path.exists(filename)) or not (filecmp.cmp(filename, tf.name)):
         with open(tf.name, "a") as f:
             os.fsync(f.fileno())
@@ -158,15 +160,15 @@ def inplace_update(filename, data):
         os.fsync(f.fileno())
 
 
-def retrieve(log, directory_lookup, tgt):
+def retrieve(log, func, tgt, mode=0o640):
     log.info("retrieve-enc", _replace_msg="Getting: {tgt}", tgt=tgt)
     try:
-        data = directory_lookup()
+        data = func()
     except Exception:
         log.error("retrieve-enc-failed", exc_info=True)
         return
     try:
-        conditional_update("/etc/nixos/{}".format(tgt), data)
+        conditional_update("/etc/nixos/{}".format(tgt), data, mode)
     except (IOError, OSError):
         inplace_update("/etc/nixos/{}".format(tgt), data)
 
@@ -208,6 +210,32 @@ def load_system_state():
     )
 
 
+def get_release_info(log, enc):
+    release_info_path = Path("releases.json")
+    if release_info_path.exists():
+        with release_info_path.open() as f:
+            releases = json.load(f)
+    else:
+        releases = {}
+
+    params = enc["parameters"]
+    release_name = params.get("release_name")
+    known_releases = [r["release_name"] for r in releases.values()]
+
+    if release_name and release_name not in known_releases:
+        environment_url = params["environment_url"]
+        version = nixos.channel_version(environment_url)
+        releases[version] = {
+            "environment": params["environment"],
+            "environment_url": environment_url,
+            "first_seen_at": utcnow().isoformat(),
+            "release_name": release_name,
+            "release_changelog": params.get("release_changelog", ""),
+        }
+
+    return releases
+
+
 def update_inventory(log, enc):
     if (
         not enc
@@ -250,6 +278,7 @@ def update_inventory(log, enc):
             (lambda: directory.list_service_clients(), "service_clients.json"),
             (lambda: directory.list_services(), "services.json"),
             (lambda: directory.list_users(), "users.json"),
+            (lambda: get_release_info(log, enc), "releases.json", 0o644),
         ],
     )