Merge remote-tracking branch 'origin/expert-mode' into expert-mode

Author: Rainbrand

.config/make/docker.mak

@@ -11,7 +11,7 @@ docker-lint: docker-lint-automation docker-lint-console-ui docker-lint-console-a
 docker-lint-automation: ## Lint automation Dockerfile
 	@echo "Lint automation container Dockerfile"
 	docker run --rm -i -v $(PWD)/automation/Dockerfile:/Dockerfile \
-	hadolint/hadolint hadolint --ignore DL3002 --ignore DL3008 --ignore DL3059 /Dockerfile
+	hadolint/hadolint hadolint --ignore DL3002 --ignore DL3008 --ignore DL3013 --ignore DL3059 /Dockerfile
 
 docker-lint-console-ui: ## Lint console ui Dockerfile
 	@echo "Lint console ui container Dockerfile"

automation/Dockerfile

@@ -13,18 +13,21 @@ COPY automation /autobase/automation
 RUN apt-get clean && rm -rf /var/lib/apt/lists/partial \
   && apt-get update -o Acquire::CompressionTypes::Order::=gz \
   && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
-  ca-certificates gnupg keychain debian-archive-keyring apt-transport-https \
-  git python3 python3-dev python3-pip ssh-client sshpass gcc g++ cmake make libssl-dev curl lsb-release \
+     ca-certificates gnupg keychain debian-archive-keyring apt-transport-https \
+     git python3 python3-dev python3-pip ssh-client sshpass gcc g++ cmake make libssl-dev curl lsb-release \
+  # fresh pip/setuptools/wheel (fewer builds from source)
+  && python3 -m pip install --break-system-packages --no-cache-dir --upgrade pip setuptools wheel \
   # repo and key for Azure CLI
   && curl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/microsoft.asc.gpg > /dev/null \
   && echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" > /etc/apt/sources.list.d/azure-cli.list \
   && apt-get update \
   # requirements
-  && pip3 install --break-system-packages --no-cache-dir -r /autobase/automation/requirements.txt \
+  && pip3 install --break-system-packages --no-cache-dir --retries 3 --timeout 60 \
+    -r /autobase/automation/requirements.txt \
   && ansible-galaxy install --force -r /autobase/automation/requirements.yml \
   && ansible-galaxy collection list \
-  && pip3 install --break-system-packages --no-cache-dir -r \
-  /root/.ansible/collections/ansible_collections/azure/azcollection/requirements.txt \
+  && pip3 install --break-system-packages --no-cache-dir --retries 3 --timeout 60 \
+    -r /root/.ansible/collections/ansible_collections/azure/azcollection/requirements.txt \
   # azure-cli
   && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y azure-cli \
   # cleanup
@@ -34,8 +37,8 @@ RUN apt-get clean && rm -rf /var/lib/apt/lists/partial \
   && chmod +x /autobase/automation/entrypoint.sh
 
 # Link collection source for Ansible runtime
-RUN mkdir -p /root/.ansible/collections/ansible_collections/vitabaks && \
-  ln -sfn /autobase/automation /root/.ansible/collections/ansible_collections/vitabaks/autobase
+RUN mkdir -p /root/.ansible/collections/ansible_collections/vitabaks \
+  && ln -sfn /autobase/automation /root/.ansible/collections/ansible_collections/vitabaks/autobase
 
 # Set environment variables
 ENV ANSIBLE_COLLECTIONS_PATH=/root/.ansible/collections/ansible_collections:/usr/local/lib/python3.11/dist-packages/ansible_collections

automation/roles/authorized_keys/tasks/main.yml

@@ -9,26 +9,17 @@
     - name: "Add public keys to ~{{ system_user.stdout | default('') }}/.ssh/authorized_keys"
       ansible.posix.authorized_key:
         user: "{{ system_user.stdout }}"
-        key: "{{ item }}"
+        key: "{{ item | replace(\"'\", '') | replace('\"', '') | trim }}"
         state: present
-      loop: '{{ ssh_public_keys_list }}'
-      vars:
-        ssh_public_keys_list: >-
-          {{
-            (
-              ssh_public_keys
-              | replace('[', '')
-              | replace(']', '')
-              | replace("'", '')
-              | replace('"', '')
-              | split(',')
-              | map('trim')
-              | reject('equalto', '')
-              | list
-              if ssh_public_keys is string
-              else ssh_public_keys
-            )
-          }}
+      loop: >-
+        {{
+          (ssh_public_keys
+          | replace('\n', ',')
+          | split(',')
+          | reject('equalto', '')
+          | list)
+          if ssh_public_keys is string else ssh_public_keys
+        }}
   when:
     - ssh_public_keys is defined
     - ssh_public_keys | length > 0

automation/roles/pgbouncer/templates/pgbouncer.ini.j2

@@ -1,6 +1,6 @@
 [databases]
 {% for pool in pgbouncer_pools %}
-{{ pool.name }} = host={{ postgresql_unix_socket_dir }} port={{ postgresql_port }} dbname={{ pool.dbname }} {{ pool.pool_parameters }}
+{{ pool.name }} = host={{ postgresql_unix_socket_dir }} port={{ postgresql_port }} dbname={{ pool.dbname }} {% if pool.pool_parameters is string %}{{ pool.pool_parameters }}{% elif pool.pool_parameters is mapping %}{% for k, v in pool.pool_parameters.items() %}{{ k }}={{ v }} {% endfor %}{% endif %}
 {% endfor %}
 
 * = host={{ postgresql_unix_socket_dir }} port={{ postgresql_port }}

automation/roles/pre_checks/tasks/pgbouncer.yml

@@ -16,15 +16,23 @@
 # The calculated pool size is then added to the total 'pgbouncer_pool_size'.
 - name: PgBouncer | Calculate pool_size
   ansible.builtin.set_fact:
-    pgbouncer_pool_size: "{{
-      (pgbouncer_pool_size | default(0) | int)
-      +
-      (pool_item.pool_parameters
-      | regex_search('pool_size=(\\d+)', multiline=False)
-      | regex_replace('[^0-9]', '')
-      | default(pgbouncer_default_pool_size | default(0), true)
-      | int)
-      }}"
+    pgbouncer_pool_size: >-
+      {{
+        (pgbouncer_pool_size | default(0) | int)
+        +
+        (
+          (
+            pool_item.pool_parameters.pool_size
+            if pool_item.pool_parameters is mapping
+            else (
+              pool_item.pool_parameters
+              | regex_search('pool_size\s*=?\s*(\d+)', multiline=False)
+            )
+          )
+          | default(pgbouncer_default_pool_size | default(0), true)
+          | int
+        )
+      }}
   loop: "{{ pgbouncer_pools | default([]) }}"
   loop_control:
     loop_var: pool_item

console/service/api/swagger.yaml

@@ -1145,7 +1145,9 @@ definitions:
         items:
           type: string
       extra_vars:
-        type: string
+        type: object
+        description: "Ansible extra vars (arbitrary JSON object)"
+        additionalProperties: true
       existing_cluster:
         type: boolean
         default: false

console/service/internal/controllers/cluster/post_cluster.go

@@ -77,11 +77,13 @@ func (h *postClusterHandler) Handle(param cluster.PostClustersParams) middleware
 	ansibleLogEnv := h.getAnsibleLogEnv(param.Body.Name)
 	localLog.Trace().Strs("file_log", ansibleLogEnv).Msg("got file log name")
 
-	// Parse ExtraVars JSON string from request
 	extraVars := map[string]interface{}{}
-	if param.Body.ExtraVars != "" {
-		if err := json.Unmarshal([]byte(param.Body.ExtraVars), &extraVars); err != nil {
-			localLog.Warn().Str("extra_vars_raw", param.Body.ExtraVars).Err(err).Msg("failed to parse extra_vars JSON; using empty object")
+
+	if param.Body.ExtraVars != nil {
+		if m, ok := param.Body.ExtraVars.(map[string]interface{}); ok {
+			extraVars = m
+		} else {
+			localLog.Warn().Interface("extra_vars_raw", param.Body.ExtraVars).Msg("unexpected type for extra_vars, expected map[string]interface{}")
 		}
 	}
 
@@ -150,21 +152,20 @@ func (h *postClusterHandler) Handle(param cluster.PostClustersParams) middleware
 		status = "ready"
 	}
 
-	// Marshal updated extraVars to JSON string for DB / Docker
+	// extraVars
 	extraVarsBytes, mErr := json.Marshal(extraVars)
 	if mErr != nil {
 		localLog.Error().Err(mErr).Msg("failed to marshal extra_vars; falling back to {}")
 		extraVarsBytes = []byte("{}")
 	}
-	extraVarsJSON := string(extraVarsBytes)
 
 	createdCluster, err := h.db.CreateCluster(param.HTTPRequest.Context(), &storage.CreateClusterReq{
 		ProjectID:         param.Body.ProjectID,
 		EnvironmentID:     param.Body.EnvironmentID,
 		Name:              param.Body.Name,
 		Description:       param.Body.Description,
 		SecretID:          secretID,
-		ExtraVars:         extraVarsJSON,
+		ExtraVars:         extraVarsBytes,
 		Location:          getValFromExtraVars(extraVars, LocationExtraVar),
 		ServerCount:       serverCount,
 		PostgreSqlVersion: getIntValFromExtraVars(extraVars, PostgreSqlVersionExtraVar),
@@ -259,7 +260,7 @@ func (h *postClusterHandler) Handle(param cluster.PostClustersParams) middleware
 	var dockerId xdocker.InstanceID
 	dockerId, err = h.dockerManager.ManageCluster(param.HTTPRequest.Context(), &xdocker.ManageClusterConfig{
 		Envs:      param.Body.Envs,
-		ExtraVars: extraVarsJSON,
+		ExtraVars: string(extraVarsBytes),
 		Mounts: []xdocker.Mount{
 			{
 				DockerPath: ansibleLogDir,

console/service/internal/storage/models.go

@@ -207,7 +207,7 @@ type CreateClusterReq struct {
 	Name              string
 	Description       string
 	SecretID          *int64
-	ExtraVars         string // JSON string
+	ExtraVars         []byte
 	Location          string
 	ServerCount       int
 	PostgreSqlVersion int