feat: Send unsynced resources with status updates

pkg/streamline/applier/applier.go

@@ -48,6 +48,11 @@ func (in *Applier) Apply(ctx context.Context,
 	serviceErrorList := make([]client.ServiceErrorAttributes, 0)
 	toSkip := make([]unstructured.Unstructured, 0)
 
+	err := in.store.SyncServiceComponents(service.ID, resources)
+	if err != nil {
+		return componentList, serviceErrorList, err
+	}
+
 	deleteFilterFunc, err := in.getDeleteFilterFunc(service.ID)
 	if err != nil {
 		return componentList, serviceErrorList, err
@@ -189,7 +194,57 @@ func (in *Applier) Apply(ctx context.Context,
 		componentList[idx].Children = lo.ToSlicePtr(children)
 	}
 
-	return componentList, serviceErrorList, nil
+	// Append unsynced resources to the component list so all of them will be visible in the UI.
+	componentList, err = in.appendUnsyncedResources(service.ID, componentList, resources)
+	return componentList, serviceErrorList, err
+}
+
+func (in *Applier) appendUnsyncedResources(serviceId string, components []client.ComponentAttributes, resources []unstructured.Unstructured) ([]client.ComponentAttributes, error) {
+	result := make([]client.ComponentAttributes, 0)
+	keys := containers.NewSet[smcommon.Key]()
+
+	for _, component := range components {
+		result = append(result, component)
+		keys.Add(smcommon.NewStoreKeyFromComponentAttributes(component).Key())
+	}
+
+	hooks, err := in.store.GetHookComponents(serviceId)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create a map of hooks for an easy lookup.
+	keyToHookComponent := make(map[smcommon.Key]smcommon.HookComponent)
+	for _, hook := range hooks {
+		keyToHookComponent[hook.StoreKey().Key()] = hook
+	}
+
+	for _, resource := range resources {
+		key := smcommon.NewStoreKeyFromUnstructured(resource).Key()
+
+		if !keys.Has(key) {
+			// If a resource has any delete policy, check if it was already deleted.
+			// If it was deleted, then do not include it in the result.
+			deletePolicies := smcommon.ParseHookDeletePolicy(resource)
+			hook, ok := keyToHookComponent[key]
+			if len(deletePolicies) > 0 && ok && hook.HasDesiredState(deletePolicies) {
+				continue
+			}
+
+			gvk := resource.GroupVersionKind()
+			result = append(result, client.ComponentAttributes{
+				Group:     gvk.Group,
+				Version:   gvk.Version,
+				Kind:      gvk.Kind,
+				Name:      resource.GetName(),
+				Namespace: resource.GetNamespace(),
+				Synced:    false,
+				State:     lo.ToPtr(client.ComponentStatePending),
+			})
+		}
+	}
+
+	return result, nil
 }
 
 func (in *Applier) Destroy(ctx context.Context, serviceID string) ([]client.ComponentAttributes, error) {
@@ -275,7 +330,7 @@ var (
 )
 
 func (in *Applier) getDeleteFilterFunc(serviceID string) (func(resources []unstructured.Unstructured) (toDelete []unstructured.Unstructured, toApply []unstructured.Unstructured), error) {
-	components, err := in.store.GetServiceComponents(serviceID)
+	components, err := in.store.GetServiceComponents(serviceID, true)
 	if err != nil {
 		return nil, err
 	}
@@ -358,7 +413,7 @@ func (in *Applier) getDeleteFilterFunc(serviceID string) (func(resources []unstr
 }
 
 func (in *Applier) getServiceComponents(serviceID string) ([]client.ComponentAttributes, error) {
-	components, err := in.store.GetServiceComponents(serviceID)
+	components, err := in.store.GetServiceComponents(serviceID, true)
 	if err != nil {
 		return nil, err
 	}

pkg/streamline/common/hook_component.go

@@ -10,15 +10,16 @@ import (
 
 // HookComponent represents hook resources that have deletion policy set.
 type HookComponent struct {
-	UID         string
-	Group       string
-	Version     string
-	Kind        string
-	Name        string
-	Namespace   string
-	Status      string
-	ManifestSHA string
-	ServiceID   string
+	UID            string
+	Group          string
+	Version        string
+	Kind           string
+	Name           string
+	Namespace      string
+	Status         string
+	ManifestSHA    string
+	ServiceID      string
+	DeletePolicies []string
 }
 
 func (in *HookComponent) GroupVersionKind() schema.GroupVersionKind {
@@ -33,6 +34,9 @@ func (in *HookComponent) Failed() bool {
 	return in.Status == string(client.ComponentStateFailed)
 }
 
+// HasDesiredState checks if the hook has the desired state based on its delete policies.
+// Delete policies from the live resource can be passed as an argument to make check better.
+// To use the delete policies from the store, use HadDesiredState.
 func (in *HookComponent) HasDesiredState(policies []string) bool {
 	for _, policy := range policies {
 		if policy == HookDeletePolicySucceeded && in.Succeeded() {
@@ -45,6 +49,13 @@ func (in *HookComponent) HasDesiredState(policies []string) bool {
 	return false
 }
 
+// HadDesiredState checks if the hook had the desired state based on its stored delete policies.
+// It uses delete policies from the store that may have changed.
+// If possible, use HasDesiredState with live resource delete policies.
+func (in *HookComponent) HadDesiredState() bool {
+	return in.HasDesiredState(in.DeletePolicies)
+}
+
 func (in *HookComponent) HasManifestChanged(u unstructured.Unstructured) bool {
 	sha, _ := utils.HashResource(u)
 	return in.ManifestSHA != sha

pkg/streamline/common/key.go

@@ -3,6 +3,7 @@ package common
 import (
 	"fmt"
 
+	"github.com/pluralsh/console/go/client"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 )
@@ -54,3 +55,11 @@ func (in StoreKey) ReplaceGroup(group string) StoreKey {
 func NewStoreKeyFromUnstructured(u unstructured.Unstructured) StoreKey {
 	return StoreKey{GVK: u.GroupVersionKind(), Namespace: u.GetNamespace(), Name: u.GetName()}
 }
+
+func NewStoreKeyFromComponentAttributes(a client.ComponentAttributes) StoreKey {
+	return StoreKey{
+		GVK:       schema.GroupVersionKind{Group: a.Group, Version: a.Version, Kind: a.Kind},
+		Namespace: a.Namespace,
+		Name:      a.Name,
+	}
+}

pkg/streamline/common/sync_delete_policy.go

@@ -61,5 +61,9 @@ func helmHookDeletePolicy(annotations map[string]string) string {
 }
 
 func ParseHookDeletePolicy(resource unstructured.Unstructured) []string {
-	return strings.Split(strings.ReplaceAll(GetPhaseHookDeletePolicy(resource), " ", ""), ",")
+	return SplitHookDeletePolicy(GetPhaseHookDeletePolicy(resource))
+}
+
+func SplitHookDeletePolicy(policy string) []string {
+	return strings.Split(strings.ReplaceAll(policy, " ", ""), ",")
 }

pkg/streamline/store/db_queries.go

@@ -21,7 +21,8 @@ const (
 			transient_manifest_sha TEXT,
 			apply_sha TEXT,
 			server_sha TEXT,
-			manifest BOOLEAN DEFAULT 0
+			manifest BOOLEAN DEFAULT 0, -- Indicates if the component was created from an original manifest set of a service
+			applied BOOLEAN DEFAULT 0 -- Indicates if the component was already applied to the cluster
 		);
 		CREATE UNIQUE INDEX IF NOT EXISTS idx_unique_component ON component("group", version, kind, namespace, name);
 		CREATE INDEX IF NOT EXISTS idx_parent ON component(parent_uid);
@@ -57,7 +58,8 @@ const (
 			name TEXT,
 			status INT,
 			manifest_sha TEXT,
-			service_id TEXT
+			service_id TEXT,
+			delete_policies TEXT
 		);
 
 		-- Add indexes to the hook component table
@@ -77,52 +79,12 @@ const (
 		WHERE uid = ?
 	`
 
-	getComponentsByServiceID = `
-		SELECT uid, parent_uid, "group", version, kind, name, namespace, health, delete_phase, manifest
-		FROM component
-		WHERE service_id = ? AND (manifest = 1 OR (parent_uid is NULL or parent_uid = ''))
-	`
-
 	getComponentsByGVK = `
 		SELECT uid, "group", version, kind, namespace, name, server_sha, delete_phase, manifest
 		FROM component
 		WHERE "group" = ? AND version = ? AND kind = ?
 	`
 
-	setComponent = `
-		INSERT INTO component (
-			uid,
-			parent_uid,
-			"group",
-			version,
-			kind,
-			namespace,
-			name,
-			health,
-		    node,
-		    created_at,
-		    service_id
-		) VALUES (
-			?,
-			?,
-			?,
-			?,
-			?,
-			?,
-			?,
-			?,
-			?,
-		    ?,
-		    ?
-		) ON CONFLICT("group", version, kind, namespace, name) DO UPDATE SET
-			uid = excluded.uid,
-			parent_uid = excluded.parent_uid,
-			health = excluded.health,
-			node = excluded.node,
-			created_at = excluded.created_at,
-			service_id = excluded.service_id
-	`
-
 	setComponentWithSHA = `
 		INSERT INTO component (
 			uid,
@@ -137,7 +99,8 @@ const (
 		    created_at,
 		    service_id,
 		    delete_phase,
-		    server_sha
+		    server_sha,
+		    applied
 		) VALUES (
 			?,
 			?,
@@ -151,6 +114,7 @@ const (
 		    ?,
 		    ?,
 		    ?,
+		    ?,
 		    ?
 		) ON CONFLICT("group", version, kind, namespace, name) DO UPDATE SET
 			uid = excluded.uid,
@@ -160,7 +124,8 @@ const (
 			created_at = excluded.created_at,
 			service_id = excluded.service_id,
 			delete_phase = excluded.delete_phase,
-			server_sha = excluded.server_sha
+			server_sha = excluded.server_sha,
+		    applied = excluded.applied
 	`
 
 	expireSHA = `
@@ -296,12 +261,13 @@ const (
 	`
 
 	setHookComponent = `
-		INSERT INTO hook_component ("group", version, kind, namespace, name, uid, status, service_id)
-		VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+		INSERT INTO hook_component ("group", version, kind, namespace, name, uid, status, service_id, delete_policies)
+		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
 		ON CONFLICT("group", version, kind, namespace, name) DO UPDATE SET
 			uid = excluded.uid,
 			status = excluded.status,
-			service_id = excluded.service_id
+			service_id = excluded.service_id,
+			delete_policies = excluded.delete_policies
 	`
 
 	setHookComponentWithManifestSHA = `