feat: implement RFC 16 to allow emergency node access

docs/docs/workflows/troubleshooting.md

@@ -56,8 +56,6 @@
 
 If your setup requires a change in the ordering of credentials, please open an issue and explain your desired behavior.
 
-
-
 ### Nodes fail to join with error `untrusted measurement value`
 
 This error indicates that a node's [attestation statement](../architecture/attestation.md) contains measurements that don't match the trusted values expected by the [JoinService](../architecture/microservices.md#joinservice).
@@ -128,24 +126,56 @@
 
 1. Figure out which node to connect to:
 
-    ```bash
-    kubectl get nodes
-    # or to see more information, such as IPs:
-    kubectl get nodes -o wide
-    ```
+   ```bash
+   kubectl get nodes
+   # or to see more information, such as IPs:
+   kubectl get nodes -o wide
+   ```
 
 2. Connect to the node:
 
-    ```bash
-    kubectl debug node/constell-worker-xksa0-000000 -it --image=busybox
-    ```
+   ```bash
+   kubectl debug node/constell-worker-xksa0-000000 -it --image=busybox
+   ```
 
-    You will be presented with a prompt.
+   You will be presented with a prompt.
 
-    The nodes file system is mounted at `/host`.
+   The nodes file system is mounted at `/host`.
 
 3. Once finished, clean up the debug pod:
 
-    ```bash
-    kubectl delete pod node-debugger-constell-worker-xksa0-000000-bjthj
-    ```
+   ```bash
+   kubectl delete pod node-debugger-constell-worker-xksa0-000000-bjthj
+   ```
+
+### Emergency SSH access
+
+Emergency SSH access to nodes can be useful to diagnose issues or download important data even in the event that the kubernetes API is not accessible anymore.
+
+1. Enter the `constellation-terraform` directory in your constellation workspace and allow emergency SSH access to the cluster:
+
+   ```bash
+    cd constellation-terraform
+    echo "emergency_ssh = true" >> ./terraform.tfvars
+    terraform apply
+   ```
+
+2. Sign an existing SSH keypair with your master secret:
+
+   ```bash
+   cd ../ # go back to your constellation workspace
+   constellation ssh --key your_public_key.pub
+   ```
+
+   A certificate will be written into the `constellation-terraform` directory.
+
+   The certificate is valid for 24 hours and allows you to access your constellation nodes using
+   [certificate based authentication](https://en.wikibooks.org/wiki/OpenSSH/Cookbook/Certificate-based_Authentication).
+
+3. Finally, you can connect to any constellation node:
+
+   ```bash
+   ssh -F ./constellation-terraform/ssh_config -i your_private_key <PRIVATE_NODE_IP>
+   ```
+
+   You can obtain the private IP via your CSPs web UI.

flake.nix

@@ -11,10 +11,10 @@
   };
 
   outputs =
-    {
-      self,
-      nixpkgsUnstable,
-      flake-utils,
+    { self
+    , nixpkgsUnstable
+    , flake-utils
+    ,
     }:
     flake-utils.lib.eachDefaultSystem (
       system:

image/base/mkosi.conf

@@ -41,6 +41,7 @@ Packages=containerd
 # Network
 Packages=iproute
          dbus
+         openssh-server
          systemd-networkd
          systemd-resolved
 

image/base/mkosi.skeleton/usr/lib/systemd/system-preset/30-constellation.preset

@@ -10,3 +10,4 @@ enable measurements.service
 enable export_constellation_debug.service
 enable systemd-timesyncd
 enable udev-trigger.service
+enable create-host-ssh-key.service

image/base/mkosi.skeleton/usr/lib/systemd/system/create-host-ssh-key.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Create a host SSH key
+Before=sshd.service
+
+[Service]
+Type=oneshot
+ExecStart=/bin/bash -c "mkdir -p /run/ssh; ssh-keygen -t ecdsa -q -N '' -f /run/ssh/ssh_host_ecdsa_key"
+
+[Install]
+WantedBy=multi-user.target

image/mirror/SHA256SUMS

@@ -1,4 +1,3 @@
-37abef83e8927b4b48f69fcbdcc249d349c6029cc669401676d01f0ea326999e  WALinuxAgent-udev-2.10.0.8-2.fc40.noarch.rpm
 8f9c8c8be1df166f4285824580c9f6588864c167c8a2d51a6c4621d1ea3f8fde  aardvark-dns-1.13.1-1.fc40.x86_64.rpm
 ac860c52abbc65af5835d1bd97400c531a5635d39bc1d68e36a1fe54863385ea  alternatives-1.27-1.fc40.x86_64.rpm
 6d0cfcd0e97421b42af58a824c7e99a6cbcdd0e81980b4ea9e0d4051ef723db3  audit-libs-4.0.2-1.fc40.i686.rpm
@@ -15,11 +14,11 @@ db18a583ebde21d8b0b67f0306e25908b273bef9c532469ac0b7ab92578438f4  authselect-lib
 5935816e8d377d0385e5287ca12e4d3b43e3c3cdc9cc4deafa653a6dba78611a  composefs-libs-1.0.6-1.fc40.x86_64.rpm
 db246f6445469b5a71e965a081685471768393cf04181e7250ce0ddcb8a9c3d4  conmon-2.1.12-2.fc40.x86_64.rpm
 adf4b75cdd9fae9d2d37fb71d9f0bf625a6705c0f0a7784569ab21463fe22152  conntrack-tools-1.4.7-7.fc40.x86_64.rpm
-621302b0ea9cdd73d5eea4d30935cb415143df1649cd8e92424e967ea98fc34d  container-selinux-2.234.2-1.fc40.noarch.rpm
 bbe29e0c7b4ca076d50b4ac3954eb383459230d96b13f353ee71ebd5de33b6d1  containerd-1.6.23-5.fc40.x86_64.rpm
 0705251ea64b1558098016b2120f202c5aba77470093cb8f89ce6adb2a0b46b6  containernetworking-plugins-1.5.1-1.fc40.x86_64.rpm
 3e35525e9224d3427f10343c98036b251fac34bf67c9007335561d846736d0d5  containers-common-0.61.0-1.fc40.noarch.rpm
 b0740195d12d356e5637b83ece8650fc3f764f37e734678a07cb637fb14faf7d  containers-common-extra-0.61.0-1.fc40.noarch.rpm
+621302b0ea9cdd73d5eea4d30935cb415143df1649cd8e92424e967ea98fc34d  container-selinux-2.234.2-1.fc40.noarch.rpm
 299d3e7e1cbc110d9ae8a47f6ca95142c3e3783cb1464bfbd6bc550c414b97ec  coreutils-single-9.4-9.fc40.x86_64.rpm
 d941a78ffb6e2e0b4c24d0097d0351ced8796edde90208b4bddee459bce0a949  cpio-2.15-1.fc40.x86_64.rpm
 faa23cb6a7a612c0a6e874c788c5add967c5e193bd38c2e6093b82b38a162f81  cracklib-2.9.11-5.fc40.i686.rpm
@@ -68,11 +67,11 @@ a6f2098fc2ed16df92c9325bd7459cc41479e17306a4f9cddfd5df8a1b80d0f8  file-5.45-4.fc
 f76684ee78408660db83ab9932978a1346b280f4210cd744524b00b2e5891fe1  file-libs-5.45-4.fc40.x86_64.rpm
 063af3db3808bea0d5c07dbb2d8369b275e1d05ad0850c80a8fec0413f47cd64  filesystem-3.18-8.fc40.x86_64.rpm
 21725de2a93e1ea19f8d298e32a2428a3a08b9c98f22561cc778a807ed43639f  findutils-4.9.0-9.fc40.x86_64.rpm
+2d6631d65e3b5c91afdb100a51ee8e50294f0e074a944c1662008d878d47456e  fuse3-3.16.2-3.fc40.x86_64.rpm
+a9c6502a5b190aaf169e93afd337c009e0b2e235e31f3da23d29c7d063ad2ff9  fuse3-libs-3.16.2-3.fc40.x86_64.rpm
 f4c2d51c7b4577f7b7ef498f8e2afb1b007da2de00cca28e220f50129c40a48c  fuse-common-3.16.2-3.fc40.x86_64.rpm
 f94315e447afb7442033b7b82e43a4ed62754f603afda53930280300855e46c7  fuse-libs-2.9.9-21.fc40.x86_64.rpm
 8fe84b7e0319afcc9c9eb28130b74e0cd7c675667a6ce075eb7ee2ec1b0014c2  fuse-overlayfs-1.13-1.fc40.x86_64.rpm
-2d6631d65e3b5c91afdb100a51ee8e50294f0e074a944c1662008d878d47456e  fuse3-3.16.2-3.fc40.x86_64.rpm
-a9c6502a5b190aaf169e93afd337c009e0b2e235e31f3da23d29c7d063ad2ff9  fuse3-libs-3.16.2-3.fc40.x86_64.rpm
 6c80dfdaf7b27ea92c1276856b8b2ae5fde1ae5c391b773805be725515fdc1ac  gawk-5.3.0-3.fc40.x86_64.rpm
 c4cc69bf3a2655b9ee9ac23492d377bac57811c5b4f81fbf43537520ee33c7af  gawk-all-langpacks-5.3.0-3.fc40.x86_64.rpm
 21470eb4ec55006c9efeee84c97772462008fceda1ab332e58d2caddfdaa0d1e  gdbm-1.23-6.fc40.x86_64.rpm
@@ -245,6 +244,8 @@ c425cdd1d0889edb688809ccc2a35a96e67a7dedc119ad540ddd05f8a8997b5e  netavark-1.13.
 784e0fbc9ccb7087c10f4c41edbed13904f94244ff658f308614abe48cdf0d42  npth-1.7-1.fc40.x86_64.rpm
 f814bc09b50daaab468715088ec056373dbc209a5075306e4ce76f5c55eb2b42  nvme-cli-2.8-1.fc40.x86_64.rpm
 deed5caa94b7590e42976c73944e882ac6be7ac94b87ea8d476a7dfe4e56c427  openldap-2.6.8-1.fc40.x86_64.rpm
+7cc7617d495bdb6b5c06bef538068a53e7cec8209c674918fd30ac82fba95b11  openssh-9.6p1-1.fc40.4.x86_64.rpm
+a1142e22df88c6200a7378f20f6d92ec62908ac67aa3fbc223dba874bdf162ba  openssh-server-9.6p1-1.fc40.4.x86_64.rpm
 5df04d37e492e5f107cc21e547240f9f98b0b7613320467bc0b08f6aa1b0fb88  openssl-libs-3.2.2-3.fc40.i686.rpm
 e9fca52d76eb6277b9fec3238226faafc0938806318fad1143a527fdd28a16cf  openssl-libs-3.2.2-3.fc40.x86_64.rpm
 9f0336deb6f1b1524ec48d837622e7e2291995369b0356d7ad1e1d427f3b659a  os-prober-1.81-6.fc40.x86_64.rpm
@@ -275,10 +276,10 @@ c03ba1c46e0e2dda36e654941f307aaa0d6574ee5143d6fec6e9af2bdf3252a2  popt-1.19-6.fc
 af85755cda79959a19161ebc26a45e507003298bd97b472b9ab0d512afa5e46a  protobuf-c-1.5.0-3.fc40.x86_64.rpm
 45ff2e9814aa059f323b23710c73309d41d36306667a3004f5fbb86b0cab4484  psmisc-23.6-6.fc40.x86_64.rpm
 cca50802d4f75306bc37126feb92db79fed44dcdabf76c1556853334995b9d3b  publicsuffix-list-dafsa-20240107-3.fc40.noarch.rpm
-7c703b431508f44c5184b5c1df052ed0f49b7439d68aa3597a9a57a5b26bd648  python-pip-wheel-23.3.2-2.fc40.noarch.rpm
-86e17167996c17798e116974f42e63dc2e0ac6bce1c10a47416d421c785a5ea4  python-unversioned-command-3.12.8-2.fc40.noarch.rpm
 5526220160d59c64689dd2c017a03a26a909c5c50f7973c8bf3750f8f39ca114  python3-3.12.8-2.fc40.x86_64.rpm
 0905050a05fce20538191ad45e61bca86d61877f58da47df1b59465d034a4ae6  python3-libs-3.12.8-2.fc40.x86_64.rpm
+7c703b431508f44c5184b5c1df052ed0f49b7439d68aa3597a9a57a5b26bd648  python-pip-wheel-23.3.2-2.fc40.noarch.rpm
+86e17167996c17798e116974f42e63dc2e0ac6bce1c10a47416d421c785a5ea4  python-unversioned-command-3.12.8-2.fc40.noarch.rpm
 d50b24d1a217e5201b4f8350945b7a3bc3fa01a61a8dd8d28e1b9512295238e1  qemu-user-static-8.2.8-2.fc40.x86_64.rpm
 11f752c50493eca8f6dddf3140c694d3db4bc808771eaba25978ea2c309b2196  qemu-user-static-aarch64-8.2.8-2.fc40.x86_64.rpm
 8598fde32ac72cafcc57f30edbfed1f920c58001dbeecb6932f4de8ce76091ba  qemu-user-static-alpha-8.2.8-2.fc40.x86_64.rpm
@@ -343,6 +344,7 @@ df5ab0573bfe904aaab3b735ea1855202c16df9308a73082b77771cf8b21da07  vim-common-9.1
 dc1d384fc3554f7185a5b5d59044b3f723bf131dd6e15e7c082b62d990dec950  vim-data-9.1.1000-1.fc40.noarch.rpm
 150be77f965f76c47f4f3f6e13ce794d89fae5fc35593bffae849998da707bd3  vim-enhanced-9.1.1000-1.fc40.x86_64.rpm
 90e56222622b0be483ab536789611e3a4272ee55a6208e4c2ad32d185e2254bc  vim-filesystem-9.1.1000-1.fc40.noarch.rpm
+37abef83e8927b4b48f69fcbdcc249d349c6029cc669401676d01f0ea326999e  WALinuxAgent-udev-2.10.0.8-2.fc40.noarch.rpm
 5f4aef6a6f19712c142b3e592ff05bba03dee877a0a098df294d876063918805  wget2-2.2.0-1.fc40.x86_64.rpm
 a4119091a85b4aa4262a26f6ed2d6653de9b7c4def3636a2b0ad066436f29acd  wget2-libs-2.2.0-1.fc40.x86_64.rpm
 4948040a53814b1b4b76f6ec9d64ec21f3f2d1196a0a1c5b117f91fa58a267b1  wget2-wget-2.2.0-1.fc40.x86_64.rpm

image/mirror/packages.txt

@@ -19,6 +19,7 @@ mokutil
 nano
 nano-default-editor
 nvme-cli
+openssh-server
 passt-selinux
 passwd
 podman

image/sysroot-tree/etc/ssh/sshd_config

@@ -0,0 +1,5 @@
+HostKey /run/ssh/ssh_host_ecdsa_key
+TrustedUserCAKeys /run/ssh/ssh_ca.pub
+PasswordAuthentication no
+ChallengeResponseAuthentication no
+AuthorizedKeysFile /dev/null

terraform/infrastructure/aws/main.tf

@@ -29,6 +29,7 @@ locals {
     { name = "recovery", port = "9999", health_check = "TCP" },
     { name = "join", port = "30090", health_check = "TCP" },
     var.debug ? [{ name = "debugd", port = "4000", health_check = "TCP" }] : [],
+    var.emergency_ssh ? [{ name = "ssh", port = "22", health_check = "TCP" }] : [],
   ])
   target_group_arns = {
     control-plane : [

terraform/infrastructure/aws/variables.tf

@@ -85,3 +85,9 @@ variable "additional_tags" {
   default     = {}
   description = "Additional tags that should be applied to created resources."
 }
+
+variable "emergency_ssh" {
+  type        = bool
+  default     = false
+  description = "Wether to expose the SSH port through the public load balancer."
+}

terraform/infrastructure/azure/main.tf

@@ -40,6 +40,7 @@ locals {
     { name = "recovery", port = "9999", health_check_protocol = "Tcp", path = null, priority = 104 },
     { name = "join", port = "30090", health_check_protocol = "Tcp", path = null, priority = 105 },
     var.debug ? [{ name = "debugd", port = "4000", health_check_protocol = "Tcp", path = null, priority = 106 }] : [],
+    var.emergency_ssh ? [{ name = "ssh", port = "22", health_check_protocol = "Tcp", path = null, priority = 107 }] : [],
   ])
   // wildcard_lb_dns_name is the DNS name of the load balancer with a wildcard for the name.
   // example: given "name-1234567890.location.cloudapp.azure.com" it will return "*.location.cloudapp.azure.com"
@@ -295,3 +296,19 @@ data "azurerm_user_assigned_identity" "uaid" {
   name                = local.uai_name
   resource_group_name = local.uai_resource_group
 }
+
+# emergency ssh configuration files
+resource "local_file" "ssh_config" {
+  filename        = "./ssh_config"
+  file_permission = "0600"
+  content         = <<EOF
+Host ${azurerm_public_ip.loadbalancer_ip[0].fqdn}
+  ProxyJump none
+
+Host *
+  PreferredAuthentications publickey
+  IdentityFile ./emergency_ssh_key
+  User root
+  ProxyJump ${azurerm_public_ip.loadbalancer_ip[0].fqdn}
+EOF
+}

terraform/infrastructure/azure/variables.tf

@@ -101,3 +101,9 @@ variable "additional_tags" {
   default     = {}
   description = "Additional tags that should be applied to created resources."
 }
+
+variable "emergency_ssh" {
+  type        = bool
+  default     = false
+  description = "Wether to expose the SSH port through the public load balancer."
+}

terraform/infrastructure/gcp/main.tf

@@ -40,6 +40,7 @@ locals {
     { name = "recovery", port = "9999", health_check = "TCP" },
     { name = "join", port = "30090", health_check = "TCP" },
     var.debug ? [{ name = "debugd", port = "4000", health_check = "TCP" }] : [],
+    var.emergency_ssh ? [{ name = "ssh", port = "22", health_check = "TCP" }] : [],
   ])
   node_groups_by_role = {
     for name, node_group in var.node_groups : node_group.role => name...

terraform/infrastructure/gcp/variables.tf

@@ -75,3 +75,9 @@ variable "additional_labels" {
   default     = {}
   description = "Additional labels that should be given to created recources."
 }
+
+variable "emergency_ssh" {
+  type        = bool
+  default     = false
+  description = "Wether to expose the SSH port through the public load balancer."
+}

terraform/infrastructure/openstack/main.tf

@@ -43,6 +43,7 @@ locals {
     { name = "recovery", port = "9999", health_check = "TCP" },
     { name = "join", port = "30090", health_check = "TCP" },
     var.debug ? [{ name = "debugd", port = "4000", health_check = "TCP" }] : [],
+    var.emergency_ssh ? [{ name = "ssh", port = "22", health_check = "TCP" }] : [],
   ])
   cidr_vpc_subnet_nodes = "192.168.178.0/24"
   cidr_vpc_subnet_lbs   = "192.168.177.0/24"

terraform/infrastructure/openstack/variables.tf

@@ -71,3 +71,9 @@ variable "stackit_project_id" {
   type        = string
   description = "STACKIT project ID."
 }
+
+variable "emergency_ssh" {
+  type        = bool
+  default     = false
+  description = "Wether to expose the SSH port through the public load balancer."
+}