fix(dagster-gcp): fix dagster logs on gcp + remove user code example (#630)

Author: walkoss

dagster/Dockerfile

@@ -0,0 +1,3 @@
+FROM gcr.io/pluralsh/dagster/user-code-example:1.2.2
+
+RUN pip install dagster-gcp

dagster/helm/dagster/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: dagster
 description: A Helm chart for Kubernetes
 type: application
-version: 0.1.28
+version: 0.1.29
 appVersion: 1.2.2
 dependencies:
 - name: postgres

dagster/helm/dagster/values.yaml

@@ -96,5 +96,3 @@ dagster:
           - "--python-file"
           - "/example_project/example_repo/repo.py"
         port: 3030
-        envSecrets:
-          - name: dagster-aws-env

dagster/helm/dagster/values.yaml.tpl

@@ -1,3 +1,5 @@
+{{ $isGcp := or (eq .Provider "google") (eq .Provider "gcp") }}
+
 global:
   application:
     links:
@@ -61,7 +63,7 @@ dagster:
       {{ end }}
   {{ end }}
 
-  {{ if eq .Provider "gcp" }}
+  {{ if $isGcp }}
   computeLogManager:
     type: GCSComputeLogManager
     config:
@@ -75,14 +77,16 @@ dagster:
       eks.amazonaws.com/role-arn: "arn:aws:iam::{{ .Project }}:role/{{ .Cluster }}-dagster"
   {{ end }}
 
-  {{ if eq .Provider "gcp" }}
+  {{ if $isGcp }}
   serviceAccount:
     annotations:
       iam.gke.io/gcp-service-account: {{ importValue "Terraform" "service_account_email" }}
   {{ end }}
 
+  {{ if not $isGcp }}
   runLauncher:
     config:
       k8sRunLauncher:
         envSecrets:
-        - name: dagster-aws-env
+        - name: dagster-aws-env
+  {{ end }}

dagster/terraform/gcp/deps.yaml

@@ -2,7 +2,7 @@ apiVersion: plural.sh/v1alpha1
 kind: Dependencies
 metadata:
   description: dagster gcp setup 
-  version: 0.1.6
+  version: 0.1.7
 spec:
   dependencies:
   - name: gcp-bootstrap

dagster/terraform/gcp/main.tf

@@ -26,25 +26,6 @@ module "gcs_buckets" {
   project_id            = var.project_id
   bucket_names          = [var.dagster_bucket]
   service_account_email = module.dagster-workload-identity.gcp_service_account_email
+  location              = var.bucket_location
 }
 
-resource "google_storage_hmac_key" "dagster" {
-  service_account_email = module.dagster-workload-identity.gcp_service_account_email
-}
-
-resource "google_service_account_key" "dagster_key" {
-  service_account_id = module.dagster-workload-identity.gcp_service_account.name
-}
-
-resource "kubernetes_secret" "dagster_s3_secret" {
-  metadata {
-    name = "dagster-aws-env"
-    namespace = kubernetes_namespace.dagster.id
-  }
-
-  data = {
-    "AWS_ACCESS_KEY_ID" = google_storage_hmac_key.dagster.access_id
-    "AWS_SECRET_ACCESS_KEY" = google_storage_hmac_key.dagster.secret
-    "GOOGLE_APPLICATION_CREDENTIALS" = base64decode(google_service_account_key.dagster_key.private_key)
-  }
-}

dagster/terraform/gcp/terraform.tfvars

@@ -1,4 +1,5 @@
 namespace = {{ .Namespace | quote }}
 dagster_bucket = {{ .Values.dagsterBucket | quote }}
 cluster_name = {{ .Cluster | quote }}
-project_id = {{ .Project | quote }}
+project_id = {{ .Project | quote }}
+bucket_location = {{ .Context.BucketLocation | quote }}

dagster/terraform/gcp/variables.tf

@@ -13,4 +13,8 @@ variable "cluster_name" {
 
 variable "project_id" {
   type = string
+}
+
+variable "bucket_location" {
+  type = string
 }