Enhance ApplicationSet template in appSets.yaml.gotmpl to support upstream chart sources and improve base application handling. Update test.py to filter out None values and handle lists in application generation, validation, and export processes.

Author: max06

appSets.yaml.gotmpl

@@ -162,6 +162,7 @@ applicationsets:
         name: "{{`{{ .values.clusterName }}-{{ .values.appInstance }}`}}"
         labels:
           debug: "{{`{{ . | toYaml }}`}}"
+          baseApp: "{{`{{ .baseApp }}`}}"
       spec:
         project: default # {{"{{ or .project \"default\" }}"}}
         destination:
@@ -173,126 +174,31 @@ applicationsets:
     templatePatch: |
       spec:
         sources:
-          {{`{{- if .baseApp }}`}}
+          # If the app is using a helmfile.yaml containing upstream charts, we need to add the upstream source
+          {{`{{- if and (eq .appSettings.path.filename "helmfile.yaml") .releases }}`}}
+          {{`{{- range $release := .releases }}`}}
+          {{`{{- if eq $release.chart "./" }}`}}
+          # Local chart source
           - repoURL: {{ .Values | get "repoUrl" "repoUrl missing"}}
             targetRevision: HEAD
-            path: {{ `apps/{{ .baseApp }}` }}
+            path: "{{ `apps/{{ $.baseApp }}` }}"
+          {{`{{- else }}`}}
+          # Upstream chart source
+          {{`{{- $repoName := (index (splitList "/" $release.chart) 0) }}`}}
+          {{`{{- $repoURL := "" }}`}}
+          {{`{{- range $repo := $.repositories }}`}}
+          {{`{{- if eq $repo.name $repoName }}`}}
+          {{`{{- $repoURL = $repo.url }}`}}
+          {{`{{- end }}`}}
+          {{`{{- end }}`}}
+          - repoURL: {{`{{ $repoURL }}`}}
+            targetRevision: {{`{{ $release.version }}`}}
+            chart: {{`{{ $release.chart }}`}}
+            helm:
+              valueFiles:
+                - {{`{{ $.values.appInstancePath }}/values.yaml`}}
+          {{`{{- end }}`}}
           {{`{{- end }}`}}
-          - repoURL: {{ .Values | get "repoUrl" "repoUrl missing"}}
-            targetRevision: HEAD
-            path: {{`{{ .values.appInstancePath }}`}}
-
-
-  root:
-    namespace: argocd
-    finalizers:
-      - resources-finalizer.argocd.argoproj.io
-    goTemplate: true
-    goTemplateOptions: ["missingkey=invalid"]
-
-    generators:
-      - matrix:
-          generators:
-            # Original app deployment generators
-            - matrix:
-                generators:
-                  - clusters:
-                      selectors:
-                      values:
-                        clusterGroup: "{{`{{ or .metadata.labels.clusterGroup nil }}`}}"
-                        clusterName: "{{`{{ or .metadata.labels.clusterId .name }}`}}"
-                        clusterSlug: "{{`{{ if .metadata.labels.clusterGroup }}{{ .metadata.labels.clusterGroup }}-{{ end }}{{ or .metadata.labels.clusterId .name }}`}}"
-
-                  - git:
-                      repoURL: {{ .Values | get "repoUrl" "repoUrl missing"}}
-                      revision: HEAD
-                      directories:
-                        # global - on every cluster
-                        - path: "{{ .Values | get "baseDir" "baseDir missing" }}/apps/*"
-                        # per clusterGroup - on every cluster in a group
-                        - path: "{{ .Values | get "baseDir" "baseDir missing" }}{{`/{{ .values.clusterGroup }}/apps/*`}}"
-                        # on a specific single cluster, either grouped or ungrouped
-                        - path: "{{ .Values | get "baseDir" "baseDir missing" }}{{`/{{ .values.clusterGroup }}/{{ .values.clusterName }}/apps/*`}}"
-                        - path: "{{ .Values | get "baseDir" "baseDir missing" }}{{`/{{ .values.clusterName }}/apps/*`}}"
-                      values:
-                        appInstance: '{{ printf "%s" "{{- printf `{{ .path.basename }}` -}}" }}'
-                        appInstancePath: '{{ printf "%s" "{{- printf `{{ .path.path }}` -}}" }}'
-
-            - merge:
-                mergeKeys:
-                  - values
-                generators:
-                  - git:
-                      repoURL: {{ .Values | get "repoUrl" "repoUrl missing"}}
-                      revision: HEAD
-                      files:
-                        - path: "{{ .Values | get "baseDir" "baseDir missing" }}/config.yaml" # 2. Universe globals
-                      values:
-                        selector: config
-
-                  - git:
-                      repoURL: {{ .Values | get "repoUrl" "repoUrl missing"}}
-                      revision: HEAD
-                      files:
-                        - path: "{{ .Values | get "baseDir" "baseDir missing" }}{{`/{{ .values.clusterGroup }}/config.yaml`}}" # 3. Constellation globals
-                      values:
-                        selector: config
-
-                  - git:
-                      repoURL: {{ .Values | get "repoUrl" "repoUrl missing"}}
-                      revision: HEAD
-                      files:
-                        - path: "{{ .Values | get "baseDir" "baseDir missing" }}{{`/{{ .values.clusterName }}/config.yaml`}}" # 4. Cluster globals
-                        - path: "{{ .Values | get "baseDir" "baseDir missing" }}{{`/{{ .values.clusterGroup }}/{{ .values.clusterName }}/config.yaml`}}" # 4. Cluster globals
-                      values:
-                        selector: config
-
-                  - git:
-                      repoURL: {{ .Values | get "repoUrl" "repoUrl missing"}}
-                      revision: HEAD
-                      files:
-                        - path: "{{ .Values | get "baseDir" "baseDir missing" }}{{`/apps/{{ .values.appInstance }}/config.yaml`}}" # 5. Universe-wide app settings
-                      values:
-                        selector: config
-
-                  - git:
-                      repoURL: {{ .Values | get "repoUrl" "repoUrl missing"}}
-                      revision: HEAD
-                      files:
-                        - path: "{{ .Values | get "baseDir" "baseDir missing" }}{{`/{{ .values.clusterGroup }}/apps/{{ .values.appInstance }}/config.yaml`}}" # 6. Constellation-wide app settings
-                      values:
-                        selector: config
-
-                  - git:
-                      repoURL: {{ .Values | get "repoUrl" "repoUrl missing"}}
-                      revision: HEAD
-                      files:
-                        - path: "{{ .Values | get "baseDir" "baseDir missing" }}{{`/{{ .values.clusterName }}/apps/{{ .values.appInstance }}/config.yaml`}}" # 7. Cluster-specific app settings
-                        - path: "{{ .Values | get "baseDir" "baseDir missing" }}{{`/{{ .values.clusterGroup }}/{{ .values.clusterName }}/apps/{{ .values.appInstance }}/config.yaml`}}" # 7. Cluster-specific app settings
-                      values:
-                        selector: config
-        selector:
-          matchExpressions:
-            - key: baseApp
-              operator: Exists
-    template:
-      metadata:
-        name: "{{`{{ .values.clusterName }}-{{ .values.appInstance }}`}}"
-      spec:
-        project: default # {{"{{ or .project \"default\" }}"}}
-        destination:
-          name: "{{`{{ .values.clusterSlug }}`}}"
-          namespace: "{{`{{ or .namespace .values.appInstance }}`}}"
-        syncPolicy:
-          syncOptions:
-            - CreateNamespace=true
-    templatePatch: |
-      spec:
-        sources:
-          {{`{{- if .baseApp }}`}}
-          - repoURL: {{ .Values | get "repoUrl" "repoUrl missing"}}
-            targetRevision: HEAD
-            path: {{ `apps/{{ .baseApp }}` }}
           {{`{{- end }}`}}
           - repoURL: {{ .Values | get "repoUrl" "repoUrl missing"}}
             targetRevision: HEAD

bin/test.py

@@ -55,14 +55,33 @@ def generate_applications(appset_file: str) -> List[Dict[str, Any]]:
             print(f"Error generating applications: {result.stderr}")
             return []
 
-        return list(yaml.safe_load_all(result.stdout))
+        # Parse all YAML documents from the output
+        yaml_docs = list(yaml.safe_load_all(result.stdout))
+
+        # Filter out None values and ensure all items are dictionaries
+        valid_apps = []
+        for doc in yaml_docs:
+            if doc is None:
+                continue
+
+            # If the document is a list, extract each item
+            if isinstance(doc, list):
+                for item in doc:
+                    if item is not None and isinstance(item, dict):
+                        valid_apps.append(item)
+            # If the document is a dictionary, add it directly
+            elif isinstance(doc, dict):
+                valid_apps.append(doc)
+
+        return valid_apps
 
 class ApplicationValidator:
     """Handles validation of Applications and ApplicationSets."""
 
     @staticmethod
     def validate_applications(apps: List[Dict[str, Any]], validation_rules: Optional[Dict[str, Callable]] = None) -> bool:
         """Validate the generated applications against predefined rules."""
+        # Filter out None values and ensure all items are dictionaries
         apps = [app for app in apps if app is not None]
         if not apps:
             print("No valid applications to validate")
@@ -72,6 +91,11 @@ def validate_applications(apps: List[Dict[str, Any]], validation_rules: Optional
         valid = True
 
         for i, app in enumerate(apps):
+            # Check if app is a list and handle it appropriately
+            if isinstance(app, list):
+                print(f"Application {i+1}: WARNING - Found a list instead of a dictionary, skipping validation")
+                continue
+
             print(f"Application {i+1}: {app.get('metadata', {}).get('name', 'unknown')}")
             valid = valid and ApplicationValidator._validate_single_application(app, i)
 
@@ -153,6 +177,11 @@ def export_applications(apps: List[Dict[str, Any]], output_dir: str) -> None:
             return
 
         for i, app in enumerate(apps):
+            # Skip if app is a list instead of a dictionary
+            if isinstance(app, list):
+                print(f"Skipping export of application {i+1}: Found a list instead of a dictionary")
+                continue
+
             name = app.get("metadata", {}).get("name", f"app-{i+1}")
             output_file = os.path.join(output_dir, f"{name}.yaml")
             with open(output_file, "w") as f:
@@ -179,6 +208,10 @@ def validate_plain_appset_sources(apps: List[Dict[str, Any]]) -> bool:
     valid = True
 
     for app in apps:
+        # Skip if app is a list instead of a dictionary
+        if isinstance(app, list):
+            continue
+
         if not app.get("metadata", {}).get("name", "").startswith("in-cluster-"):
             continue