From 238a26bbcb7a724df3df272a5c76bcff624b152f Mon Sep 17 00:00:00 2001 From: Ewan Date: Wed, 30 Nov 2022 14:52:52 +0100 Subject: [PATCH 1/6] Implement Azure CSI disk creation and registration --- playbooks/roles/nomad/tasks/csi_plugins.yml | 25 ++++++ playbooks/roles/nomad/tasks/csi_volumes.yml | 81 +++++++++++++++++++ .../templates/azure_csi_controller.hcl.j2 | 67 +++++++++++++++ .../nomad/templates/azure_csi_node.hcl.j2 | 58 +++++++++++++ .../roles/nomad/templates/csi-volumes.json.j2 | 27 +++++++ 5 files changed, 258 insertions(+) create mode 100644 playbooks/roles/nomad/tasks/csi_plugins.yml create mode 100644 playbooks/roles/nomad/tasks/csi_volumes.yml create mode 100644 playbooks/roles/nomad/templates/azure_csi_controller.hcl.j2 create mode 100644 playbooks/roles/nomad/templates/azure_csi_node.hcl.j2 create mode 100644 playbooks/roles/nomad/templates/csi-volumes.json.j2 diff --git a/playbooks/roles/nomad/tasks/csi_plugins.yml b/playbooks/roles/nomad/tasks/csi_plugins.yml new file mode 100644 index 0000000..1b2bc4e --- /dev/null +++ b/playbooks/roles/nomad/tasks/csi_plugins.yml @@ -0,0 +1,25 @@ + + - name: Deploy CSI Controller plugin + community.general.nomad_job: + host: "nomad.service.consul" + state: present + content: "{{ lookup('ansible.builtin.template', item + '.hcl.j2') }}" + use_ssl: false + register: nomad_job_spawn + retries: 10 + delay: 5 + until: nomad_job_spawn.failed == false + + - name: Check the job state is healthy + ansible.builtin.uri: + url: "http://nomad.service.consul:4646/v1/job/{{ job_name }}" + method: GET + # headers: + # X-Nomad-Token: "{{ nomad_token }}" + remote_src: yes + register: job_status + until: job_status | json_query('json.Status') == 'running' + + - name: debug + debug: + msg: "{{ nomad_job_spawn }}" diff --git a/playbooks/roles/nomad/tasks/csi_volumes.yml b/playbooks/roles/nomad/tasks/csi_volumes.yml new file mode 100644 index 0000000..4c655b5 --- /dev/null +++ b/playbooks/roles/nomad/tasks/csi_volumes.yml @@ -0,0 +1,81 @@ +--- + +- name: Check if volume has already been created and registered + ansible.builtin.uri: + url: "http://nomad.service.consul:4646/v1/volume/csi/{{ item.id }}" + method: GET + # headers: + # X-Nomad-Token: "{{ nomad_token }}" + remote_src: yes + register: volume_exists + ignore_errors: true + +- name: Check variables + ansible.builtin.debug: + msg: | + Name: {{ item.name }} + Id: {{ item.id }} + PluginID: {{ nomad_csi_plugin_id }} + + + +- name: Check for state of CSI Plugins to prevent race + block: + - name: Check controller plugin + ansible.builtin.uri: + url: "http://nomad.service.consul:4646/v1/job/plugin-azure_csi_controller" + method: GET + # headers: + # X-Nomad-Token: "{{ nomad_token }}" + remote_src: yes + register: controller_status + until: controller_status | json_query('json.Status') == 'running' + + - name: Check node plugin + ansible.builtin.uri: + url: "http://nomad.service.consul:4646/v1/job/plugin-azure_csi_node" + method: GET + # headers: + # X-Nomad-Token: "{{ nomad_token }}" + remote_src: yes + register: node_status + until: node_status | json_query('json.Status') == 'running' + + +- name: Create CSI volume with Nomad API + ansible.builtin.uri: + url: http://nomad.service.consul:4646/v1/volume/csi/{{ item.id }}/create + method: PUT + # headers: + # X-Nomad-Token: "{{ nomad_token }}" + # body: "{{ lookup('template', 'csi-volumes.json.j2') }}" + body_format: json + body: > + { + "Volumes": [ + { + "ID": "{{ item.id }}", + "Name": "{{ item.name }}", + "Namespace": "default", + "PluginID": "{{ nomad_csi_plugin_id }}", + "MountOptions": { + "FsType": "ext4", + "MountFlags": ["ro", "noatime"], + }, + "RequestedCapacityMin": 53687091200, + "RequestedCapacityMax": 53687091200, + "RequestedCapabilities": [ + { + "AccessMode": "single-node-writer", + "AttachmentMode": "file-system" + }, + { + "AccessMode": "single-node-writer", + "AttachmentMode": "block-device" + } + ], + } + ] + } + remote_src: yes + when: volume_exists.status == 404 diff --git a/playbooks/roles/nomad/templates/azure_csi_controller.hcl.j2 b/playbooks/roles/nomad/templates/azure_csi_controller.hcl.j2 new file mode 100644 index 0000000..8fc264f --- /dev/null +++ b/playbooks/roles/nomad/templates/azure_csi_controller.hcl.j2 @@ -0,0 +1,67 @@ +job "{{ nomad_csi_plugin_job_name }}" { + datacenters = ["{{ nomad_datacenter }}"] + type = "service" + + group "controller" { + count = 2 + + constraint { + distinct_hosts = true + } + + # disable deployments + update { + max_parallel = 0 + } + task "controller" { + driver = "docker" + + template { + change_mode = "noop" + destination = "local/azure.json" + data = < Date: Mon, 5 Dec 2022 16:37:49 +0100 Subject: [PATCH 2/6] Use template lookup instead of putting request body in task --- playbooks/roles/nomad/tasks/csi_volumes.yml | 61 +++------------------ 1 file changed, 7 insertions(+), 54 deletions(-) diff --git a/playbooks/roles/nomad/tasks/csi_volumes.yml b/playbooks/roles/nomad/tasks/csi_volumes.yml index 4c655b5..36d7667 100644 --- a/playbooks/roles/nomad/tasks/csi_volumes.yml +++ b/playbooks/roles/nomad/tasks/csi_volumes.yml @@ -4,78 +4,31 @@ ansible.builtin.uri: url: "http://nomad.service.consul:4646/v1/volume/csi/{{ item.id }}" method: GET - # headers: - # X-Nomad-Token: "{{ nomad_token }}" remote_src: yes register: volume_exists ignore_errors: true -- name: Check variables - ansible.builtin.debug: - msg: | - Name: {{ item.name }} - Id: {{ item.id }} - PluginID: {{ nomad_csi_plugin_id }} - - - - name: Check for state of CSI Plugins to prevent race block: - name: Check controller plugin ansible.builtin.uri: url: "http://nomad.service.consul:4646/v1/job/plugin-azure_csi_controller" method: GET - # headers: - # X-Nomad-Token: "{{ nomad_token }}" remote_src: yes register: controller_status until: controller_status | json_query('json.Status') == 'running' - - name: Check node plugin ansible.builtin.uri: url: "http://nomad.service.consul:4646/v1/job/plugin-azure_csi_node" method: GET - # headers: - # X-Nomad-Token: "{{ nomad_token }}" remote_src: yes register: node_status until: node_status | json_query('json.Status') == 'running' - - -- name: Create CSI volume with Nomad API - ansible.builtin.uri: - url: http://nomad.service.consul:4646/v1/volume/csi/{{ item.id }}/create - method: PUT - # headers: - # X-Nomad-Token: "{{ nomad_token }}" - # body: "{{ lookup('template', 'csi-volumes.json.j2') }}" - body_format: json - body: > - { - "Volumes": [ - { - "ID": "{{ item.id }}", - "Name": "{{ item.name }}", - "Namespace": "default", - "PluginID": "{{ nomad_csi_plugin_id }}", - "MountOptions": { - "FsType": "ext4", - "MountFlags": ["ro", "noatime"], - }, - "RequestedCapacityMin": 53687091200, - "RequestedCapacityMax": 53687091200, - "RequestedCapabilities": [ - { - "AccessMode": "single-node-writer", - "AttachmentMode": "file-system" - }, - { - "AccessMode": "single-node-writer", - "AttachmentMode": "block-device" - } - ], - } - ] - } - remote_src: yes + - name: Create CSI volume with Nomad API + ansible.builtin.uri: + url: http://nomad.service.consul:4646/v1/volume/csi/{{ item.id }}/create + method: PUT + body_format: json + body: "{{ lookup('template', 'csi-volumes.json.j2') }}" + remote_src: yes when: volume_exists.status == 404 From 7aca91516b95f28ef0c51a51b11c074d46df7f3f Mon Sep 17 00:00:00 2001 From: Ewan Date: Mon, 5 Dec 2022 16:39:45 +0100 Subject: [PATCH 3/6] Add templating to some variables --- playbooks/roles/nomad/templates/csi-volumes.json.j2 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/playbooks/roles/nomad/templates/csi-volumes.json.j2 b/playbooks/roles/nomad/templates/csi-volumes.json.j2 index e97c784..87f6fd7 100644 --- a/playbooks/roles/nomad/templates/csi-volumes.json.j2 +++ b/playbooks/roles/nomad/templates/csi-volumes.json.j2 @@ -3,14 +3,14 @@ { "ID": "{{ item.id }}", "Name": "{{ item.name }}", - "Namespace": "default", + "Namespace": "{{ item.namespace }}", "PluginID": "{{ nomad_csi_plugin_id }}", "MountOptions": { - "FsType": "ext4", - "MountFlags": ["ro", "noatime"] + "FsType": "{{ item.filesystem }}", + "MountFlags": ["noatime"] }, - "RequestedCapacityMin": 53687091200, - "RequestedCapacityMax": 53687091200, + "RequestedCapacityMin": {{ item.size }}, + "RequestedCapacityMax": {{ item.size }}, "RequestedCapabilities": [ { "AccessMode": "single-node-writer", From aac7156b9535415661613d3acd54f059d91601e7 Mon Sep 17 00:00:00 2001 From: Ewan Date: Mon, 5 Dec 2022 16:40:22 +0100 Subject: [PATCH 4/6] Add templating to region variable --- playbooks/roles/nomad/templates/azure_csi_controller.hcl.j2 | 4 ++-- playbooks/roles/nomad/templates/azure_csi_node.hcl.j2 | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/playbooks/roles/nomad/templates/azure_csi_controller.hcl.j2 b/playbooks/roles/nomad/templates/azure_csi_controller.hcl.j2 index 8fc264f..71a7df6 100644 --- a/playbooks/roles/nomad/templates/azure_csi_controller.hcl.j2 +++ b/playbooks/roles/nomad/templates/azure_csi_controller.hcl.j2 @@ -27,7 +27,7 @@ job "{{ nomad_csi_plugin_job_name }}" { "aadClientId": "{{ client_id }}", "aadClientSecret": "{{ client_secret }}", "resourceGroup": "{{ rg }}", -"location": "westeurope" +"location": "{{ azure_region }}" } EOH } @@ -38,7 +38,7 @@ EOH config { image = "{{ nomad_csi_plugin_image }}" - + network_mode = "host" volumes = [ "local/azure.json:/etc/kubernetes/azure.json" ] diff --git a/playbooks/roles/nomad/templates/azure_csi_node.hcl.j2 b/playbooks/roles/nomad/templates/azure_csi_node.hcl.j2 index 3510cc8..c60b8f7 100644 --- a/playbooks/roles/nomad/templates/azure_csi_node.hcl.j2 +++ b/playbooks/roles/nomad/templates/azure_csi_node.hcl.j2 @@ -17,7 +17,7 @@ job "{{ nomad_csi_plugin_job_name }}" { "aadClientId": "{{ client_id }}", "aadClientSecret": "{{ client_secret }}", "resourceGroup": "{{ rg }}", -"location": "westeurope" +"location": "{{ azure_region }}" } EOH } @@ -28,6 +28,7 @@ EOH config { image = "{{ nomad_csi_plugin_image }}" + network_mode = "host" volumes = [ "local/azure.json:/etc/kubernetes/azure.json" ] From 5300d7147c058b56bcb3e8300e0631d45cc12be0 Mon Sep 17 00:00:00 2001 From: Ewan Date: Tue, 6 Dec 2022 15:47:32 +0100 Subject: [PATCH 5/6] Fixes to CSI implementation --- playbooks/roles/nomad/tasks/csi_plugins.yml | 6 ++---- playbooks/roles/nomad/tasks/csi_volumes.yml | 1 + playbooks/roles/nomad/tasks/main.yml | 14 ++++++++++++++ .../roles/nomad/templates/azure_csi_node.hcl.j2 | 2 +- .../roles/nomad/templates/csi-volumes.json.j2 | 4 ++-- 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/playbooks/roles/nomad/tasks/csi_plugins.yml b/playbooks/roles/nomad/tasks/csi_plugins.yml index 1b2bc4e..2f1a192 100644 --- a/playbooks/roles/nomad/tasks/csi_plugins.yml +++ b/playbooks/roles/nomad/tasks/csi_plugins.yml @@ -1,5 +1,5 @@ - - name: Deploy CSI Controller plugin + - name: Deploy CSI plugin community.general.nomad_job: host: "nomad.service.consul" state: present @@ -12,10 +12,8 @@ - name: Check the job state is healthy ansible.builtin.uri: - url: "http://nomad.service.consul:4646/v1/job/{{ job_name }}" + url: "http://nomad.service.consul:4646/v1/job/plugin-{{ item }}" method: GET - # headers: - # X-Nomad-Token: "{{ nomad_token }}" remote_src: yes register: job_status until: job_status | json_query('json.Status') == 'running' diff --git a/playbooks/roles/nomad/tasks/csi_volumes.yml b/playbooks/roles/nomad/tasks/csi_volumes.yml index 36d7667..66f55ac 100644 --- a/playbooks/roles/nomad/tasks/csi_volumes.yml +++ b/playbooks/roles/nomad/tasks/csi_volumes.yml @@ -24,6 +24,7 @@ remote_src: yes register: node_status until: node_status | json_query('json.Status') == 'running' + - name: Create CSI volume with Nomad API ansible.builtin.uri: url: http://nomad.service.consul:4646/v1/volume/csi/{{ item.id }}/create diff --git a/playbooks/roles/nomad/tasks/main.yml b/playbooks/roles/nomad/tasks/main.yml index 2f1b5fc..f2572e8 100644 --- a/playbooks/roles/nomad/tasks/main.yml +++ b/playbooks/roles/nomad/tasks/main.yml @@ -30,3 +30,17 @@ failed_when: false loop: "{{ nomad_namespaces }}" run_once: true + +- name: Setup csi plugins + ansible.builtin.include_tasks: "csi_plugins.yml" + loop: + - azure_csi_controller + - azure_csi_node + when: nomad_csi_volumes is defined + run_once: true + +- name: Create csi volumes + ansible.builtin.include_tasks: "csi_volumes.yml" + loop: "{{ nomad_csi_volumes }}" + when: nomad_csi_volumes is defined + run_once: true diff --git a/playbooks/roles/nomad/templates/azure_csi_node.hcl.j2 b/playbooks/roles/nomad/templates/azure_csi_node.hcl.j2 index c60b8f7..54149ea 100644 --- a/playbooks/roles/nomad/templates/azure_csi_node.hcl.j2 +++ b/playbooks/roles/nomad/templates/azure_csi_node.hcl.j2 @@ -33,7 +33,7 @@ EOH "local/azure.json:/etc/kubernetes/azure.json" ] args = [ - "--nodeid={{ nomad_csi_plugin_node_id }}", + "--nodeid=${attr.unique.hostname}", "--endpoint=unix://csi/csi.sock", "--logtostderr", "--v=5", diff --git a/playbooks/roles/nomad/templates/csi-volumes.json.j2 b/playbooks/roles/nomad/templates/csi-volumes.json.j2 index 87f6fd7..4e63bd9 100644 --- a/playbooks/roles/nomad/templates/csi-volumes.json.j2 +++ b/playbooks/roles/nomad/templates/csi-volumes.json.j2 @@ -9,8 +9,8 @@ "FsType": "{{ item.filesystem }}", "MountFlags": ["noatime"] }, - "RequestedCapacityMin": {{ item.size }}, - "RequestedCapacityMax": {{ item.size }}, + "RequestedCapacityMin": {{ item.minsize }}, + "RequestedCapacityMax": {{ item.maxsize }}, "RequestedCapabilities": [ { "AccessMode": "single-node-writer", From c161c70eb188dd233b63f8296c22ba1bc2d6c117 Mon Sep 17 00:00:00 2001 From: Ewan Date: Tue, 6 Dec 2022 16:30:52 +0100 Subject: [PATCH 6/6] Add tags across deploy playbook --- playbooks/deploy.yml | 15 ++++++++++ playbooks/roles/nomad/tasks/main.yml | 43 +++++++++++++++++++--------- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/playbooks/deploy.yml b/playbooks/deploy.yml index 9d54faa..9da27a4 100644 --- a/playbooks/deploy.yml +++ b/playbooks/deploy.yml @@ -3,11 +3,26 @@ hosts: all:!bastion roles: - role: docker + tags: + - setup-all + - setup-docker - role: consul + tags: + - setup-all + - setup-consul - role: dns + tags: + - setup-all + - setup-dns - name: setup nomad import_playbook: sub_plays/nomad.yml + tags: + - setup-all + - setup-nomad - name: setup system jobs on nomad import_playbook: runtime.yml + tags: + - setup-all + - setup-runtime diff --git a/playbooks/roles/nomad/tasks/main.yml b/playbooks/roles/nomad/tasks/main.yml index f2572e8..211f56b 100644 --- a/playbooks/roles/nomad/tasks/main.yml +++ b/playbooks/roles/nomad/tasks/main.yml @@ -31,16 +31,33 @@ loop: "{{ nomad_namespaces }}" run_once: true -- name: Setup csi plugins - ansible.builtin.include_tasks: "csi_plugins.yml" - loop: - - azure_csi_controller - - azure_csi_node - when: nomad_csi_volumes is defined - run_once: true - -- name: Create csi volumes - ansible.builtin.include_tasks: "csi_volumes.yml" - loop: "{{ nomad_csi_volumes }}" - when: nomad_csi_volumes is defined - run_once: true +- block: + - name: Setup csi plugins + ansible.builtin.include_tasks: + file: "csi_plugins.yml" + apply: + tags: + - setup-all + - setup-csi + loop: + - azure_csi_controller + - azure_csi_node + tags: + - setup-all + - setup-csi + when: nomad_csi_volumes is defined + run_once: true + + - name: Create csi volumes + ansible.builtin.include_tasks: + file: "csi_volumes.yml" + apply: + tags: + - setup-all + - setup-csi + loop: "{{ nomad_csi_volumes }}" + tags: + - setup-all + - setup-csi + when: nomad_csi_volumes is defined + run_once: true