add k0s support

.gitignore

@@ -12,3 +12,6 @@ kubeconfig
 .venv*
 # Taskfile
 .tasks
+# intellij
+.idea
+

.taskfiles/FluxTasks.yaml

@@ -26,9 +26,15 @@ tasks:
       - kubectl apply --server-side --filename {{.KUBERNETES_DIR}}/flux/vars/cluster-settings-user.yaml
       - kubectl apply --server-side --kustomize {{.KUBERNETES_DIR}}/flux/config
     preconditions:
-      - { msg: "Flux already appears installed", sh: "exit $(( ! $(kubectl get namespace flux-system) ))" }
+      - { msg: "Flux already appears installed", sh: "kubectl get namespace flux-system &>/dev/null && exit 1 || exit 0" }
       - { msg: "Age private key not found",      sh: "test -f {{.ROOT_DIR}}/age.key" }
 
+  github-deploy-key:
+    cmds:
+      - sops --decrypt {{.KUBERNETES_DIR}}/bootstrap/github-deploy-key.sops.yaml | kubectl apply -f -
+    preconditions:
+      - { msg: "Flux is not installed", sh: "kubectl get namespace flux-system &>/dev/null && exit 0 || exit 1" }
+
   apply:
     desc: Apply a Flux Kustomization resource for a cluster
     summary: |

.taskfiles/K0sTasks.yaml

@@ -0,0 +1,25 @@
+---
+version: "3"
+
+tasks:
+
+  kubeconfig:
+    desc: Gets k0s cluster kubeconfig
+    cmds:
+      - k0sctl kubeconfig -c k0s-config.yaml > kubeconfig
+    preconditions:
+      - { msg: "k0s-config.yaml not found", sh: "test -f {{.ROOT_DIR}}/k0s-config.yaml" }
+
+  apply:
+    desc: Apply k0s cluster k0s-config.yaml
+    cmds:
+      - k0sctl apply --config k0s-config.yaml
+      - task: kubeconfig
+    preconditions:
+      - { msg: "k0s-config.yaml not found", sh: "test -f {{.ROOT_DIR}}/k0s-config.yaml" }
+
+  reset:
+    desc: Resets the k0s cluster
+    cmd: k0sctl reset --config k0s-config.yaml
+    preconditions:
+      - { msg: "k0s-config.yaml not found", sh: "test -f {{.ROOT_DIR}}/k0s-config.yaml" }

Taskfile.yaml

@@ -17,6 +17,7 @@ includes:
     aliases: ["k8s"]
     taskfile: .taskfiles/KubernetesTasks.yaml
   flux: .taskfiles/FluxTasks.yaml
+  k0s: .taskfiles/K0sTasks.yaml
 
 tasks:
 
@@ -45,7 +46,6 @@ tasks:
   configure:
     desc: Configure repository from Ansible vars
     prompt: Any conflicting config in the root kubernetes and ansible directories will be overwritten... continue?
-    dir: "{{.BOOTSTRAP_DIR}}"
-    cmd: ansible-playbook configure.yaml
+    cmd: ./.venv/bin/ansible-playbook {{.BOOTSTRAP_DIR}}/configure.yaml
     env:
       ANSIBLE_DISPLAY_SKIPPED_HOSTS: "false"

bootstrap/configure.yaml

@@ -17,23 +17,30 @@
       ansible.builtin.set_fact:
         repository_path: "{{ repository.stdout }}"
 
-    - name: Override kube-vip address when there is a single master node and no address is defined
-      when: bootstrap_nodes.master | length == 1 and not bootstrap_kube_vip_addr
+    - name: Override kubeapi address when there is a single master node and no address is defined
+      when: bootstrap_nodes.master | length == 1 and not bootstrap_kubeapi_addr
       ansible.builtin.set_fact:
         bootstrap_kube_vip_enabled: false
-        bootstrap_kube_vip_addr: "{{ bootstrap_nodes.master[0].address }}"
+        bootstrap_kubeapi_addr: "{{ bootstrap_nodes.master[0].address }}"
 
     - name: Verify configuration
       ansible.builtin.include_tasks: tasks/validation/main.yaml
 
     - name: Template Sops configuration
       ansible.builtin.include_tasks: tasks/sops/main.yaml
 
+    - name: Template k0s configuration
+      when: bootstrap_distribution == "k0s"
+      ansible.builtin.include_tasks: tasks/k0s/main.yaml
+
     - name: Template Ansible configuration
       ansible.builtin.include_tasks: tasks/ansible/main.yaml
 
     - name: Template Kubernetes configuration
       ansible.builtin.include_tasks: tasks/kubernetes/main.yaml
 
+    - name: Template Kubernetes optional configuration
+      ansible.builtin.include_tasks: tasks/optional/main.yaml
+
     - name: Template Kubernetes addon configuration
       ansible.builtin.include_tasks: tasks/addons/main.yaml

bootstrap/tasks/addons/main.yaml

@@ -21,7 +21,7 @@
   ansible.builtin.include_tasks: kube_prometheus_stack.yaml
 
 - name: Process addon system-upgrade-controller
-  when: system_upgrade_controller.enabled | default(false)
+  when: (bootstrap_distribution == "k3s") and (system_upgrade_controller.enabled | default(false))
   ansible.builtin.include_tasks: system_upgrade_controller.yaml
 
 - name: Process addon weave-gitops

bootstrap/tasks/ansible/main.yaml

@@ -27,13 +27,4 @@
         mode: "0644"
         force: true
       with_community.general.filetree: ["../templates/ansible/"]
-    - name: Template encrypted node secrets
-      community.sops.sops_encrypt:
-        path: "{{ repository_path }}/ansible/inventory/host_vars/{{ item.name }}.sops.yaml"
-        age: ["{{ bootstrap_age_public_key }}"]
-        content_yaml: "{{ lookup('ansible.builtin.template', 'templates/node.sops.yaml.j2', template_vars=dict(password=item.password)) | from_yaml }}"
-        mode: "0644"
-        force: true
-      loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}"
-      loop_control:
-        label: "{{ item.address }}"
+

bootstrap/tasks/k0s/main.yaml

@@ -0,0 +1,6 @@
+---
+- name: Template k0s configuration file
+  ansible.builtin.template:
+    src: "templates/k0s-config.yaml.j2"
+    dest: "{{ repository_path }}/k0s-config.yaml"
+    mode: "0644"

bootstrap/tasks/optional/coredns.yaml

@@ -0,0 +1,34 @@
+---
+- name: Set addon facts
+  ansible.builtin.set_fact:
+    optional_name: coredns
+    optional_namespace: kube-system
+
+- name: Ensure directories exist for {{ optional_namespace }}/{{ optional_name }}
+  when: item.state == 'directory'
+  ansible.builtin.file:
+    path: "{{ repository_path }}/kubernetes/apps/{{ optional_namespace }}/{{ optional_name }}/{{ item.path }}"
+    state: directory
+    mode: "0755"
+  with_community.general.filetree: ["../templates/optional/{{ optional_name }}/"]
+
+- name: Template unencrypted files for {{ optional_namespace }}/{{ optional_name }}
+  when: item.state == 'file' and 'sops' not in item.path
+  ansible.builtin.template:
+    src: "{{ item.src }}"
+    dest: "{{ repository_path }}/kubernetes/apps/{{ optional_namespace }}/{{ optional_name }}/{{ item.path | replace('.j2', '') }}"
+    mode: "0644"
+  with_community.general.filetree: ["../templates/optional/{{ optional_name }}/"]
+
+- name: Template encrypted files for {{ optional_namespace }}/{{ optional_name }}
+  block:
+    - name: Template encrypted files
+      when: item.state == 'file' and 'sops' in item.path
+      community.sops.sops_encrypt:
+        path: "{{ repository_path }}/kubernetes/apps/{{ optional_namespace }}/{{ optional_name }}/{{ item.path | replace('.j2', '') }}"
+        encrypted_regex: ^(data|stringData)$
+        age: ["{{ bootstrap_age_public_key }}"]
+        content_yaml: "{{ lookup('ansible.builtin.template', item.src) | from_yaml }}"
+        mode: "0644"
+        force: true
+      with_community.general.filetree: ["../templates/optional/{{ optional_name }}/"]

bootstrap/tasks/optional/main.yaml

@@ -0,0 +1,7 @@
+---
+
+- name: Process optional opendns
+  when: bootstrap_distribution == "k3s"
+  ansible.builtin.include_tasks: coredns.yaml
+
+

bootstrap/tasks/validation/github.yaml

@@ -14,6 +14,7 @@
     fail_msg: Github user {{ bootstrap_github_username }} does not exist
 
 - name: Query Github repo
+  when: not bootstrap_private_github_repo | default(false)
   ansible.builtin.uri:
     url: https://api.github.com/repos/{{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }}
     timeout: 5
@@ -22,12 +23,14 @@
   register: result
 
 - name: Check if repo exists
+  when: not bootstrap_private_github_repo | default(false)
   ansible.builtin.assert:
     that: result.json.full_name == bootstrap_github_username + '/' + bootstrap_github_repository_name
     success_msg: Github repo {{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }} exists
     fail_msg: Github repo {{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }} does not exist
 
 - name: Query Github repo branch
+  when: not bootstrap_private_github_repo | default(false)
   ansible.builtin.uri:
     url: https://api.github.com/repos/{{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }}/branches/{{ bootstrap_github_repository_branch | default('main', true) }}
     timeout: 5
@@ -36,6 +39,7 @@
   register: result
 
 - name: Check if repo branch exists
+  when: not bootstrap_private_github_repo | default(false)
   ansible.builtin.assert:
     that: result.json.name == bootstrap_github_repository_branch | default('main', true)
     success_msg: Github repo branch {{ bootstrap_github_repository_branch | default('main', true) }} exists

bootstrap/tasks/validation/net.yaml

@@ -98,17 +98,17 @@
     success_msg: external ingress address {{ bootstrap_external_ingress_addr }} is within {{ bootstrap_node_cidr }}.
     fail_msg: external ingress address {{ bootstrap_external_ingress_addr }} is not within {{ bootstrap_node_cidr }}.
 
-- name: Verify kube-vip
+- name: Verify kubeapi address
   ansible.builtin.assert:
-    that: bootstrap_kube_vip_addr is ansible.utils.ipv4
-    success_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is valid.
-    fail_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is invalid.
+    that: bootstrap_kubeapi_addr is ansible.utils.ipv4
+    success_msg: kubeapi address {{ bootstrap_kubeapi_addr }} is valid.
+    fail_msg: kubeapi address {{ bootstrap_kubeapi_addr }} is invalid.
 
-- name: Verify kube-vip in node CIDR
+- name: Verify kubeapi address in node CIDR
   ansible.builtin.assert:
-    that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_kube_vip_addr)
-    success_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is within {{ bootstrap_node_cidr }}.
-    fail_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is not within {{ bootstrap_node_cidr }}.
+    that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_kubeapi_addr)
+    success_msg: kubeapi address {{ bootstrap_kubeapi_addr }} is within {{ bootstrap_node_cidr }}.
+    fail_msg: kubeapi address {{ bootstrap_kubeapi_addr }} is not within {{ bootstrap_node_cidr }}.
 
 - name: Verify all IP addresses are unique
   ansible.builtin.assert:
@@ -117,7 +117,7 @@
         bootstrap_k8s_gateway_addr,
         bootstrap_external_ingress_addr,
         bootstrap_internal_ingress_addr,
-        bootstrap_kube_vip_addr
+        bootstrap_kubeapi_addr
       ] | unique | length == 4
     success_msg: All IP addresses are unique.
     fail_msg: All IP addresses are not unique.
@@ -133,12 +133,12 @@
   loop_control:
     label: "{{ item.address }}"
 
-- name: Verify nodes are not the same IPs as k8s_gateway, ingress external/internal or kube-vip
-  when: bootstrap_kube_vip_enabled | default(true)
+- name: Verify nodes are not the same IPs as k8s_gateway, ingress external/internal or kubeapi address
+  when: (bootstrap_distribution == "k3s") and (bootstrap_kube_vip_enabled | default(true))
   ansible.builtin.assert:
-    that: item.address not in (bootstrap_k8s_gateway_addr, bootstrap_external_ingress_addr, bootstrap_internal_ingress_addr, bootstrap_kube_vip_addr)
-    success_msg: Node address {{ item.address }} is different than k8s_gateway, ingress-nginx or kube-vip.
-    fail_msg: Node address {{ item.address }} is not different than k8s_gateway, ingress-nginx or kube-vip.
+    that: item.address not in (bootstrap_k8s_gateway_addr, bootstrap_external_ingress_addr, bootstrap_internal_ingress_addr, bootstrap_kubeapi_addr)
+    success_msg: Node address {{ item.address }} is different than k8s_gateway, ingress-nginx or kubeapi.
+    fail_msg: Node address {{ item.address }} is not different than k8s_gateway, ingress-nginx or kubeapi.
     quiet: true
   loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}"
   loop_control:

bootstrap/tasks/validation/vars.yaml

@@ -16,23 +16,24 @@
     - bootstrap_cloudflare_tunnel_id
     - bootstrap_cloudflare_tunnel_secret
     - bootstrap_cluster_cidr
+    - bootstrap_distribution
+    - bootstrap_external_ingress_addr
     - bootstrap_flux_github_webhook_token
-    - bootstrap_github_repository_name
     - bootstrap_github_repository_branch
+    - bootstrap_github_repository_name
     - bootstrap_github_username
-    - bootstrap_external_ingress_addr
     - bootstrap_internal_ingress_addr
     - bootstrap_ipv6_enabled
     - bootstrap_k8s_gateway_addr
-    - bootstrap_kube_vip_addr
-    - bootstrap_local_path_provisioner_path
+    - bootstrap_kubeapi_addr
+    - bootstrap_local_storage_path
     - bootstrap_node_cidr
     - bootstrap_service_cidr
     - bootstrap_timezone
 
 - name: Verify bootstrap node names are valid
   ansible.builtin.assert:
-    that: item.name is match('^[a-z0-9-]+$')
+    that: item.name is match('^[a-z0-9-\.]+$')
     success_msg: Node name {{ item.name }} is valid
     fail_msg: Node name {{ item.name }} is not valid
   loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}"

bootstrap/templates/addons/grafana/app/helmrelease.yaml.j2

@@ -24,7 +24,7 @@ spec:
   uninstall:
     keepHistory: false
   dependsOn:
-    - name: local-path-provisioner
+    - name: openebs
       namespace: storage
   values:
     deploymentStrategy:
@@ -168,6 +168,6 @@ spec:
             - *host
     persistence:
       enabled: true
-      storageClassName: local-hostpath
+      storageClassName: openebs-hostpath
     testFramework:
       enabled: false

bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2

@@ -27,7 +27,7 @@ spec:
   uninstall:
     keepHistory: false
   dependsOn:
-    - name: local-path-provisioner
+    - name: openebs
       namespace: storage
   valuesFrom:
     - name: kube-prometheus-stack-values

bootstrap/templates/addons/kube-prometheus-stack/app/helmvalues.yaml.j2

@@ -114,7 +114,7 @@ data:
         storageSpec:
           volumeClaimTemplate:
             spec:
-              storageClassName: local-hostpath
+              storageClassName: openebs-hostpath
               resources:
                 requests:
                   storage: 10Gi

bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2

@@ -14,7 +14,7 @@ k3s_etcd_datastore: true
 {% else %}
 k3s_etcd_datastore: false
 {% endif %}
-k3s_registration_address: "{% raw %}{{ kube_vip_addr }}{% endraw %}"
+k3s_registration_address: "{% raw %}{{ kubeapi_addr }}{% endraw %}"
 # /var/lib/rancher/k3s/server/manifests
 k3s_server_manifests_templates:
   - custom-cilium-helmchart.yaml.j2

bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2

@@ -3,7 +3,7 @@
 timezone: "{{ bootstrap_timezone }}"
 github_username: "{{ bootstrap_github_username }}"
 coredns_addr: "{{ bootstrap_service_cidr.split(',')[0] | ansible.utils.nthhost(10) }}"
-kube_vip_addr: "{{ bootstrap_kube_vip_addr }}"
+kubeapi_addr: "{{ bootstrap_kubeapi_addr }}"
 cluster_cidr: "{{ bootstrap_cluster_cidr.split(',')[0] }}"
 service_cidr: "{{ bootstrap_service_cidr.split(',')[0] }}"
 node_cidr: "{{ bootstrap_node_cidr }}"

bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2

@@ -11,7 +11,7 @@ k3s_server:
   node-ip: "{% raw %}{{ ansible_host }}{% endraw %}"
 {% endif %}
   tls-san:
-    - "{% raw %}{{ kube_vip_addr }}{% endraw %}"
+    - "{% raw %}{{ kubeapi_addr }}{% endraw %}"
   docker: false
   flannel-backend: "none"             # This needs to be in quotes
   disable:

bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2

@@ -37,7 +37,7 @@ spec:
     ipv6:
       enabled: true
     {% endif %}
-    k8sServiceHost: "{% raw %}{{ kube_vip_addr }}{% endraw %}"
+    k8sServiceHost: "{% raw %}{{ kubeapi_addr }}{% endraw %}"
     k8sServicePort: 6443
     kubeProxyReplacement: true
     kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256

bootstrap/templates/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2.j2

@@ -15,7 +15,7 @@ spec:
       args: ["manager"]
       env:
         - name: address
-          value: "{% raw %}{{ kube_vip_addr }}{% endraw %}"
+          value: "{% raw %}{{ kubeapi_addr }}{% endraw %}"
         - name: vip_arp
           value: "true"
         - name: lb_enable