feat(sysbox): onboard sysbox as a secure container runtime (#788)

rauerhans · michaeljguarino · web-flow · commit 39c2aaa1a0cc · 2023-08-16T15:53:41.000+02:00
* sysbox artifact init

* add ng

* ignore local tf files during debugging

* change path

* pass on launch_template var

* test

* add icon

* rm ingress

* fix vars

* seems to have troubles with subnets, rm for now

* not yet published, hardcode

* set some required variables

* add sysbox reserved taint

* add block device mappings

* add sysbox-install label

* desired capacity one

* clean up

* sysbox resources for install

* fix pluralfile

* rm gitignore, messes with tf publish

* format

* make recipes private

* rename ng, lt

* string valued

* output ngs, debug

* use subnet ids from other nodegroups

* paremetrize subnets correctly

* shorten names

* disk size in lt

* mark recipes as non-private

* add outputs to artifact

* pass key pari create flag

* add eks label

* create key pair

* eks.amazonaws.com/ is reserved label prefix

* fix typo

* bubble up key s

* fix ami

* clean up

* revert to tpl

* prepare usable helm chart

* prep helm chart

* mark not private

* get subnets from bootstrap artifact

* add toleration

* rm node role arn, inferred from eks data soruce

* uninstall with hook

* fix tpl

* add helm labels and annotations

* fix labels

* change filter

* set max pods to 16

* pass owners

* restructure temporarily

* labels and taints

* move into its onw

* complete yaml

* fix ami filter

* add plural ami ng

* fix owners

* update plural sysbox ami

* new ami

* test new ami

* rm outputs

* test new ami, + tf

* rearrange, rm unnecessary templates

* new ami

* use multi az

* correct labels

* update ami

* rm old lt

* rm explicit labels

* rm k8s install

* update ami

* new ami

* clean up

* fix performancetype

* new ami

* test max pods

* update final ami

* update ami owner

* pass lt as var to ng

* clean up

* point to module-library release ref

* set app version to sysbox version

* rm sysbox ns

* rm azure and gcp artifacts

* rm gcp recipe

* rm deps from azure/gcp

* bump

---------

Co-authored-by: michaeljguarino &lt;mguarino46@gmail.com&gt;
diff --git a/ray/terraform/aws/variables.tf b/ray/terraform/aws/variables.tf
@@ -1,5 +1,5 @@
 variable "namespace" {
-  type = string
+  type    = string
   default = "ray"
 }
 
@@ -8,7 +8,7 @@ variable "cluster_name" {
 }
 
 variable "create_single_az_node_groups" {
-  type = bool
+  type    = bool
   default = false
 }
 
@@ -27,34 +27,35 @@ variable "node_groups_defaults" {
   type        = any
   default = {
     desired_capacity = 0
-    min_capacity = 0
-    max_capacity = 3
+    min_capacity     = 0
+    max_capacity     = 3
 
-    instance_types = ["t3.large", "t3a.large"]
-    disk_size = 50
-    ami_release_version = "1.22.15-20221222"
+    instance_types       = ["t3.large", "t3a.large"]
+    disk_size            = 50
+    ami_release_version  = "1.22.15-20221222"
     force_update_version = true
-    ami_type = "AL2_x86_64"
-    k8s_labels = {}
-    k8s_taints = []
+    ami_type             = "AL2_x86_64"
+    k8s_labels           = {}
+    k8s_taints           = []
   }
 }
 
 variable "single_az_node_groups" {
   type = any
   default = {
     ray_small_burst_spot = {
-      name = "ray-small-burst-spot"
-      capacity_type = "SPOT"
+      name           = "ray-small-burst-spot"
+      capacity_type  = "SPOT"
       instance_types = ["t3.large", "t3a.large"]
       k8s_labels = {
-        "plural.sh/capacityType" = "SPOT"
+        "plural.sh/capacityType"    = "SPOT"
         "plural.sh/performanceType" = "BURST"
-        "plural.sh/scalingGroup" = "ray-small-burst-spot"
+        "plural.sh/scalingGroup"    = "ray-small-burst-spot"
+        "sysbox-install"            = "yes"
       }
       k8s_taints = [{
-        key = "plural.sh/capacityType"
-        value = "SPOT"
+        key    = "plural.sh/capacityType"
+        value  = "SPOT"
         effect = "NO_SCHEDULE"
       }]
     }
diff --git a/sysbox/Pluralfile b/sysbox/Pluralfile
@@ -0,0 +1,6 @@
+REPO sysbox
+ATTRIBUTES Plural repository.yaml
+
+TF terraform/*
+HELM helm/*
+RECIPE plural/recipes/*
diff --git a/sysbox/helm/sysbox/.helmignore b/sysbox/helm/sysbox/.helmignore
@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/sysbox/helm/sysbox/Chart.yaml b/sysbox/helm/sysbox/Chart.yaml
@@ -0,0 +1,6 @@
+apiVersion: v2
+name: sysbox
+description: helm chart for sysbox
+type: application
+version: 0.1.2
+appVersion: "v0.6.2"
diff --git a/sysbox/helm/sysbox/README.md b/sysbox/helm/sysbox/README.md
@@ -0,0 +1 @@
+A helm chart for sysbox
diff --git a/sysbox/helm/sysbox/deps.yaml b/sysbox/helm/sysbox/deps.yaml
@@ -0,0 +1,17 @@
+apiVersion: plural.sh/v1alpha1
+kind: Dependencies
+metadata:
+  application: true
+  description: Deploys sysbox crafted for the target cloud
+spec:
+  dependencies:
+  - type: helm
+    name: bootstrap
+    repo: bootstrap
+    version: '>= 0.5.1'
+  - type: terraform
+    name: aws
+    repo: sysbox
+    version: '>= 0.1.0'
+    optional: true
+    
diff --git a/sysbox/helm/sysbox/templates/_helpers.tpl b/sysbox/helm/sysbox/templates/_helpers.tpl
@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "sysbox.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "sysbox.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "sysbox.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "sysbox.labels" -}}
+helm.sh/chart: {{ include "sysbox.chart" . }}
+{{ include "sysbox.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "sysbox.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "sysbox.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "sysbox.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "sysbox.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}
diff --git a/sysbox/helm/sysbox/templates/runtimeclass.yaml b/sysbox/helm/sysbox/templates/runtimeclass.yaml
@@ -0,0 +1,8 @@
+apiVersion: node.k8s.io/v1
+kind: RuntimeClass
+metadata:
+  name: sysbox-runc
+handler: sysbox-runc
+scheduling:
+  nodeSelector:
+    sysbox-runtime: running
diff --git a/sysbox/helm/sysbox/values.yaml b/sysbox/helm/sysbox/values.yaml
@@ -0,0 +1 @@
+{}
diff --git a/sysbox/helm/sysbox/values.yaml.tpl b/sysbox/helm/sysbox/values.yaml.tpl
@@ -0,0 +1 @@
+{}
diff --git a/sysbox/plural/icons/sysbox.png b/sysbox/plural/icons/sysbox.png
diff --git a/sysbox/plural/notes.tpl b/sysbox/plural/notes.tpl
@@ -0,0 +1 @@
+Use `plural watch sysbox` to track the status of your application
diff --git a/sysbox/plural/recipes/sysbox-aws.yaml b/sysbox/plural/recipes/sysbox-aws.yaml
@@ -0,0 +1,16 @@
+name: sysbox-aws
+description: Installs sysbox on an aws EKS cluster
+provider: AWS
+primary: true
+private: false
+dependencies:
+- repo: bootstrap
+  name: aws-k8s
+sections:
+- name: sysbox
+  configuration: []
+  items:
+  - type: TERRAFORM
+    name: aws
+  - type: HELM
+    name: sysbox
diff --git a/sysbox/repository.yaml b/sysbox/repository.yaml
@@ -0,0 +1,8 @@
+name: sysbox
+description: sysbox deployed on plural
+category: SECURITY
+private: false
+icon: plural/icons/sysbox.png
+notes: plural/notes.tpl
+gitUrl: https://github.com/nestybox/sysbox
+homepage: https://www.nestybox.com/sysbox
diff --git a/sysbox/terraform/aws/deps.yaml b/sysbox/terraform/aws/deps.yaml
@@ -0,0 +1,13 @@
+apiVersion: plural.sh/v1alpha1
+kind: Dependencies
+metadata:
+  description: sysbox aws setup
+  version: 0.1.2
+spec:
+  dependencies:
+  - name: aws-bootstrap
+    repo: bootstrap
+    type: terraform
+    version: '>= 0.1.1'
+  providers:
+  - aws
diff --git a/sysbox/terraform/aws/main.tf b/sysbox/terraform/aws/main.tf
@@ -0,0 +1,93 @@
+
+data "aws_eks_cluster" "cluster" {
+  name = var.cluster_name
+}
+
+data "aws_eks_node_groups" "cluster" {
+  cluster_name = var.cluster_name
+}
+
+data "aws_eks_node_group" "main" {
+  cluster_name    = var.cluster_name
+  node_group_name = tolist(data.aws_eks_node_groups.cluster.names)[0]
+}
+
+module "launch_templates" {
+  source   = "github.com/pluralsh/module-library//terraform/eks-node-groups/launch-template?ref=eks-node-groups-v1.0.0"
+  for_each = var.launch_templates
+
+  tags = try(each.value.tags, {})
+  # required
+  launch_template_name            = try(each.value.launch_template_name, join("-", [var.cluster_name, each.key]))
+  launch_template_use_name_prefix = try(each.value.launch_template_use_name_prefix, false)
+  launch_template_description     = try(each.value.launch_template_description, null)
+  ebs_optimized                   = try(each.value.ebs_optimized, false)
+  # one of the following must be specified, either ami_id or a combination of ami_filter_name & ami_owners 
+  ami_id          = try(each.value.ami_id, null)
+  ami_filter_name = try(each.value.ami_filter_name, null)
+  ami_owners      = try(each.value.ami_owners, [])
+  # optional
+  create_key_pair                        = try(each.value.create_key_pair, false)
+  key_name                               = try(each.value.key_name, null)
+  vpc_security_group_ids                 = try(each.value.vpc_security_group_ids, [])
+  cluster_primary_security_group_id      = try(each.value.cluster_primary_security_group_id, null)
+  launch_template_default_version        = try(each.value.launch_template_default_version, null)
+  update_launch_template_default_version = try(each.value.update_launch_template_default_version, true)
+  disable_api_termination                = try(each.value.disable_api_termination, null)
+  kernel_id                              = try(each.value.kernel_id, null)
+  ram_disk_id                            = try(each.value.ram_disk_id, null)
+  block_device_mappings                  = try(each.value.block_device_mappings, {})
+  capacity_reservation_specification     = try(each.value.capacity_reservation_specification, {})
+  cpu_options                            = try(each.value.cpu_options, {})
+  credit_specification                   = try(each.value.credit_specification, {})
+  elastic_gpu_specifications             = try(each.value.elastic_gpu_specifications, {})
+  elastic_inference_accelerator          = try(each.value.elastic_inference_accelerator, {})
+  enclave_options                        = try(each.value.enclave_options, {})
+  instance_market_options                = try(each.value.instance_market_options, {})
+  maintenance_options                    = try(each.value.maintenance_options, {})
+  license_specifications                 = try(each.value.license_specifications, {})
+  metadata_options                       = try(each.value.metadata_options, {})
+  enable_monitoring                      = try(each.value.enable_monitoring, false)
+  network_interfaces                     = try(each.value.network_interfaces, [])
+  placement                              = try(each.value.placement, {})
+  private_dns_name_options               = try(each.value.private_dns_name_options, {})
+  launch_template_tags                   = try(each.value.launch_template_tags, {})
+  tag_specifications                     = try(each.value.tag_specifications, [])
+  # the following are required if you need custom user data in you launch template, e.g. because you're using custom AMI 
+  enable_bootstrap_user_data = try(each.value.enable_bootstrap_user_data, false)
+  cluster_name               = try(each.value.cluster_name, var.cluster_name)
+  cluster_endpoint           = try(each.value.cluster_endpoint, "")
+  cluster_auth_base64        = try(each.value.cluster_auth_base64, "")
+  # this is optional if you're using a custom 
+  cluster_service_ipv4_cidr = try(each.value.cluster_service_ipv4_cidr, null)
+  pre_bootstrap_user_data   = try(each.value.pre_bootstrap_user_data, "")
+  post_bootstrap_user_data  = try(each.value.post_bootstrap_user_data, "")
+  bootstrap_extra_args      = try(each.value.bootstrap_extra_args, "")
+  kubelet_extra_args        = try(each.value.kubelet_extra_args, {})
+  k8s_labels                = merge(var.multi_az_node_groups[each.key]["k8s_labels"], try(each.value.k8s_labels, {}))
+  k8s_taints                = concat(var.multi_az_node_groups[each.key]["k8s_taints"], try(each.value.k8s_taints, []))
+  max_pods_per_node         = try(each.value.max_pods_per_node, null)
+}
+
+
+module "node_groups" {
+  source               = "github.com/pluralsh/module-library//terraform/eks-node-groups/multi-az-node-groups?ref=eks-node-groups-v1.0.0"
+  cluster_name         = var.cluster_name
+  default_iam_role_arn = data.aws_eks_node_group.main.node_role_arn
+  tags                 = var.tags
+  node_groups_defaults = var.node_groups_defaults
+  node_groups = {
+    for k, v in var.multi_az_node_groups : k => merge(
+      v,
+      try(var.launch_templates[k].launch_template_name, null) != null ?
+      {
+        launch_template_id      = module.launch_templates[k].launch_template_id
+        launch_template_version = module.launch_templates[k].launch_template_latest_version
+      } :
+      {}
+    )
+  }
+  set_desired_size   = false
+  private_subnet_ids = var.private_subnets[*].id
+  depends_on         = [module.launch_templates]
+}
diff --git a/sysbox/terraform/aws/outputs.tf b/sysbox/terraform/aws/outputs.tf
@@ -0,0 +1 @@
+
diff --git a/sysbox/terraform/aws/terraform.tfvars b/sysbox/terraform/aws/terraform.tfvars
@@ -0,0 +1,8 @@
+{{- $bootstrap := .Applications.TerraformValues "bootstrap" -}}
+
+namespace = {{ .Namespace | quote }}
+cluster_name = {{ .Cluster | quote }}
+private_subnets = yamldecode(<<EOT
+{{ $bootstrap.cluster_worker_private_subnets | toYaml }}
+EOT
+)
diff --git a/sysbox/terraform/aws/variables.tf b/sysbox/terraform/aws/variables.tf
