feat(weavier-gcp): add gcs backup (#634)

walkoss · web-flow · commit a7c495eb98e3 · 2023-04-09T17:05:31.000-04:00
diff --git a/weaviate/helm/weaviate/Chart.yaml b/weaviate/helm/weaviate/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: weaviate
 description: helm chart for weaviate
 type: application
-version: 0.1.0
+version: 0.1.1
 appVersion: 1.18.0
 dependencies:
 - name: weaviate
diff --git a/weaviate/helm/weaviate/templates/serviceaccount.yaml b/weaviate/helm/weaviate/templates/serviceaccount.yaml
diff --git a/weaviate/helm/weaviate/values.yaml b/weaviate/helm/weaviate/values.yaml
@@ -2,18 +2,6 @@
 # # This is a YAML-formatted file.
 # # Declare variables to be passed into your templates.
 
-nameOverride: ""
-fullnameOverride: ""
-
-serviceAccount:
-  # Specifies whether a service account should be created
-  create: true
-  # Annotations to add to the service account
-  annotations: {}
-  # The name of the service account to use.
-  # If not set and create is true, a name is generated using the fullname template
-  name: ""
-
 ingress:
   enabled: true
   className: "nginx"
@@ -36,4 +24,3 @@ weaviate:
         protocol: TCP
         port: 80
     type: ClusterIP
-    annotations: {}
diff --git a/weaviate/helm/weaviate/values.yaml.tpl b/weaviate/helm/weaviate/values.yaml.tpl
@@ -1,3 +1,5 @@
+{{ $isGcp := or (eq .Provider "google") (eq .Provider "gcp") }}
+
 ingress:
   hosts:
   - host: {{ .Values.hostname }}
@@ -8,3 +10,13 @@ ingress:
   - secretName: weaviate-tls
     hosts:
       - {{ .Values.hostname }}
+
+weaviate:
+  backups:
+    {{ if $isGcp }}
+    gcs:
+      enabled: true
+      envconfig:
+        BACKUP_GCS_BUCKET: {{ .Values.weaviateBucket }}
+        BACKUP_GCS_USE_AUTH: "true"
+    {{ end }}
diff --git a/weaviate/plural/recipes/weaviate-gcp.yaml b/weaviate/plural/recipes/weaviate-gcp.yaml
@@ -12,6 +12,10 @@ sections:
     - name: hostname
       type: DOMAIN
       documentation: the fully qualified domain name your Weaviate instance will be available at
+    - name: weaviateBucket
+      type: BUCKET
+      documentation: gcs bucket for storing weaviate backups
+      default: weaviate
   items:
   - type: TERRAFORM
     name: kube
diff --git a/weaviate/terraform/aws/deps.yaml b/weaviate/terraform/aws/deps.yaml
@@ -0,0 +1,13 @@
+apiVersion: plural.sh/v1alpha1
+kind: Dependencies
+metadata:
+  description: weaviate aws setup
+  version: 0.1.0
+spec:
+  dependencies:
+  - name: aws-bootstrap
+    repo: bootstrap
+    type: terraform
+    version: '>= 0.1.1'
+  providers:
+  - aws
diff --git a/weaviate/terraform/aws/main.tf b/weaviate/terraform/aws/main.tf
@@ -8,3 +8,13 @@ resource "kubernetes_namespace" "weaviate" {
   }
 }
 
+resource "kubernetes_service_account" "weaviate" {
+  metadata {
+    name      = "weaviate"
+    namespace = var.namespace
+  }
+
+  depends_on = [
+    kubernetes_namespace.weaviate
+  ]
+}
diff --git a/weaviate/terraform/aws/terraform.tfvars b/weaviate/terraform/aws/terraform.tfvars
diff --git a/weaviate/terraform/aws/variables.tf b/weaviate/terraform/aws/variables.tf
diff --git a/weaviate/terraform/azure/deps.yaml b/weaviate/terraform/azure/deps.yaml
@@ -0,0 +1,13 @@
+apiVersion: plural.sh/v1alpha1
+kind: Dependencies
+metadata:
+  description: weaviate azure setup
+  version: 0.1.0
+spec:
+  dependencies:
+  - name: azure-bootstrap
+    repo: bootstrap
+    type: terraform
+    version: '>= 0.1.1'
+  providers:
+  - azure
diff --git a/weaviate/terraform/azure/main.tf b/weaviate/terraform/azure/main.tf
@@ -0,0 +1,20 @@
+resource "kubernetes_namespace" "weaviate" {
+  metadata {
+    name = var.namespace
+    labels = {
+      "app.kubernetes.io/managed-by" = "plural"
+      "app.plural.sh/name"           = "weaviate"
+    }
+  }
+}
+
+resource "kubernetes_service_account" "weaviate" {
+  metadata {
+    name      = "weaviate"
+    namespace = var.namespace
+  }
+
+  depends_on = [
+    kubernetes_namespace.weaviate
+  ]
+}
diff --git a/weaviate/terraform/azure/terraform.tfvars b/weaviate/terraform/azure/terraform.tfvars
@@ -0,0 +1 @@
+namespace = {{ .Namespace | quote }}
diff --git a/weaviate/terraform/azure/variables.tf b/weaviate/terraform/azure/variables.tf
@@ -0,0 +1,4 @@
+variable "namespace" {
+  type    = string
+  default = "weaviate"
+}
diff --git a/weaviate/terraform/gcp/deps.yaml b/weaviate/terraform/gcp/deps.yaml
@@ -0,0 +1,13 @@
+apiVersion: plural.sh/v1alpha1
+kind: Dependencies
+metadata:
+  description: weaviate gcp setup
+  version: 0.1.0
+spec:
+  dependencies:
+  - name: gcp-bootstrap
+    repo: bootstrap
+    type: terraform
+    version: '>= 0.1.1'
+  providers:
+  - gcp
diff --git a/weaviate/terraform/gcp/main.tf b/weaviate/terraform/gcp/main.tf
@@ -0,0 +1,43 @@
+resource "kubernetes_namespace" "weaviate" {
+  metadata {
+    name = var.namespace
+    labels = {
+      "app.kubernetes.io/managed-by" = "plural"
+      "app.plural.sh/name"           = "weaviate"
+    }
+  }
+}
+
+module "weaviate-workload-identity" {
+  source              = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity"
+  name                = "${var.cluster_name}-weaviate-sa"
+  namespace           = var.namespace
+  project_id          = var.project_id
+  use_existing_k8s_sa = true
+  annotate_k8s_sa     = false
+  k8s_sa_name         = "weaviate"
+  roles               = ["roles/storage.admin"]
+}
+
+module "gcs_buckets" {
+  source = "github.com/pluralsh/module-library//terraform/gcs-buckets"
+
+  project_id            = var.project_id
+  bucket_names          = [var.weaviate_bucket]
+  service_account_email = module.weaviate-workload-identity.gcp_service_account_email
+  location              = var.bucket_location
+}
+
+resource "kubernetes_service_account" "weaviate" {
+  metadata {
+    name      = "weaviate"
+    namespace = var.namespace
+    annotations = {
+      "iam.gke.io/gcp-service-account" = module.weaviate-workload-identity.gcp_service_account_email
+    }
+  }
+
+  depends_on = [
+    kubernetes_namespace.weaviate
+  ]
+}
diff --git a/weaviate/terraform/gcp/terraform.tfvars b/weaviate/terraform/gcp/terraform.tfvars
@@ -0,0 +1,5 @@
+namespace = {{ .Namespace | quote }}
+weaviate_bucket = {{ .Values.weaviateBucket | quote }}
+cluster_name = {{ .Cluster | quote }}
+project_id = {{ .Project | quote }}
+bucket_location = {{ .Context.BucketLocation | quote }}
diff --git a/weaviate/terraform/gcp/variables.tf b/weaviate/terraform/gcp/variables.tf
@@ -0,0 +1,20 @@
+variable "namespace" {
+  type    = string
+  default = "weaviate"
+}
+
+variable "weaviate_bucket" {
+  type = string
+}
+
+variable "cluster_name" {
+  type = string
+}
+
+variable "project_id" {
+  type = string
+}
+
+variable "bucket_location" {
+  type = string
+}
diff --git a/weaviate/terraform/kube/deps.yaml b/weaviate/terraform/kube/deps.yaml

Original file line number	Diff line number	Diff line change
`@@ -8,3 +8,13 @@ resource "kubernetes_namespace" "weaviate" {`
`8`	`8`	`}`
`9`	`9`	`}`
`10`	`10`
	`11`	`+resource "kubernetes_service_account" "weaviate" {`
	`12`	`+ metadata {`
	`13`	`+ name = "weaviate"`
	`14`	`+ namespace = var.namespace`
	`15`	`+ }`
	`16`	`+`
	`17`	`+ depends_on = [`
	`18`	`+ kubernetes_namespace.weaviate`
	`19`	`+ ]`
	`20`	`+}`