From a2635c5b6b3768d0e6534b62eda47774ded8d9d9 Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Tue, 6 Feb 2024 12:00:05 -0500 Subject: [PATCH 01/35] Begin to port back linc-temp infra to central repo --- .gitignore | 2 + terraform/api.tf | 68 ++++++++--------- terraform/domain.tf | 63 ++++++++++----- terraform/main.tf | 24 +++--- .../log_bucket.tf | 14 ++-- .../main.tf | 76 +++++++++---------- .../outputs.tf | 4 +- .../variables.tf | 0 .../versions.tf | 0 terraform/redirector.tf | 4 +- terraform/sentry.tf | 4 +- terraform/sponsored_bucket.tf | 14 ++-- terraform/sponsored_iam.tf | 4 +- terraform/staging_bucket.tf | 16 ++-- terraform/staging_pipeline.tf | 40 +++++----- 15 files changed, 179 insertions(+), 154 deletions(-) rename terraform/modules/{dandiset_bucket => lincset_bucket}/log_bucket.tf (80%) rename terraform/modules/{dandiset_bucket => lincset_bucket}/main.tf (68%) rename terraform/modules/{dandiset_bucket => lincset_bucket}/outputs.tf (70%) rename terraform/modules/{dandiset_bucket => lincset_bucket}/variables.tf (100%) rename terraform/modules/{dandiset_bucket => lincset_bucket}/versions.tf (100%) diff --git a/.gitignore b/.gitignore index 6304eb3..c377a8b 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,5 @@ override.tf.json # Ignore CLI configuration files .terraformrc terraform.rc + +.idea/ \ No newline at end of file diff --git a/terraform/api.tf b/terraform/api.tf index a8ea0a0..66f0499 100644 --- a/terraform/api.tf +++ b/terraform/api.tf @@ -1,18 +1,18 @@ -data "heroku_team" "dandi" { - name = "dandi" +data "heroku_team" "linc-brain-mit" { + name = "linc-brain-mit" } module "api" { source = "girder/girder4/heroku" version = "0.13.0" - project_slug = "dandi-api" - heroku_team_name = data.heroku_team.dandi.name - route53_zone_id = aws_route53_zone.dandi.zone_id + project_slug = "linc-staging-terraform" + heroku_team_name = data.heroku_team.linc-brain-mit.name + route53_zone_id = aws_route53_zone.linc-brain-mit.zone_id subdomain_name = "api" - heroku_web_dyno_size = "standard-2x" - heroku_worker_dyno_size = "standard-2x" + heroku_web_dyno_size = "standard-1x" + heroku_worker_dyno_size = "standard-1x" heroku_postgresql_plan = "standard-0" heroku_cloudamqp_plan = "squirrel-1" heroku_papertrail_plan = "liatorp" @@ -20,38 +20,38 @@ module "api" { heroku_web_dyno_quantity = 1 heroku_worker_dyno_quantity = 1 - django_default_from_email = "admin@api.dandiarchive.org" - django_cors_origin_whitelist = ["https://dandiarchive.org"] - django_cors_origin_regex_whitelist = ["^https:\\/\\/[0-9a-z\\-]+--gui-dandiarchive-org\\.netlify\\.app$"] + django_default_from_email = "admin@api.lincbrain.org" + django_cors_origin_whitelist = ["https://lincbrain.org"] + django_cors_origin_regex_whitelist = ["^https:\\/\\/[0-9a-z\\-]+\\.netlify\\.app$"] - additional_django_vars = { - DJANGO_CONFIGURATION = "HerokuProductionConfiguration" - DJANGO_DANDI_DANDISETS_BUCKET_NAME = module.sponsored_dandiset_bucket.bucket_name - DJANGO_DANDI_DANDISETS_BUCKET_PREFIX = "" - DJANGO_DANDI_DANDISETS_EMBARGO_BUCKET_NAME = module.sponsored_embargo_bucket.bucket_name - DJANGO_DANDI_DANDISETS_EMBARGO_BUCKET_PREFIX = "" - DJANGO_DANDI_DANDISETS_LOG_BUCKET_NAME = module.sponsored_dandiset_bucket.log_bucket_name - DJANGO_DANDI_DANDISETS_EMBARGO_LOG_BUCKET_NAME = module.sponsored_embargo_bucket.log_bucket_name - DJANGO_DANDI_DOI_API_URL = "https://api.datacite.org/dois" - DJANGO_DANDI_DOI_API_USER = "dartlib.dandi" - DJANGO_DANDI_DOI_API_PREFIX = "10.48324" - DJANGO_DANDI_DOI_PUBLISH = "true" - DJANGO_SENTRY_DSN = data.sentry_key.this.dsn_public - DJANGO_SENTRY_ENVIRONMENT = "production" - DJANGO_CELERY_WORKER_CONCURRENCY = "4" - DJANGO_DANDI_WEB_APP_URL = "https://dandiarchive.org" - DJANGO_DANDI_API_URL = "https://api.dandiarchive.org" - DJANGO_DANDI_JUPYTERHUB_URL = "https://hub.dandiarchive.org/" - } - additional_sensitive_django_vars = { - DJANGO_DANDI_DOI_API_PASSWORD = var.doi_api_password - } + additional_django_vars = { + DJANGO_CONFIGURATION = "HerokuProductionConfiguration" + DJANGO_DANDI_DANDISETS_BUCKET_NAME = module.sponsored_lincset_bucket.bucket_name + DJANGO_DANDI_DANDISETS_BUCKET_PREFIX = "" + DJANGO_DANDI_DANDISETS_EMBARGO_BUCKET_NAME = module.sponsored_embargo_bucket.bucket_name + DJANGO_DANDI_DANDISETS_EMBARGO_BUCKET_PREFIX = "" + DJANGO_DANDI_DANDISETS_LOG_BUCKET_NAME = module.sponsored_lincset_bucket.log_bucket_name + DJANGO_DANDI_DANDISETS_EMBARGO_LOG_BUCKET_NAME = module.sponsored_embargo_bucket.log_bucket_name + DJANGO_DANDI_DOI_API_URL = "https://api.datacite.org/dois" + DJANGO_DANDI_DOI_API_USER = "temp.dandi" + DJANGO_DANDI_DOI_API_PREFIX = "temp" + DJANGO_DANDI_DOI_PUBLISH = "true" + DJANGO_SENTRY_DSN = "https://833c159dc622528b21b4ce4adef6dbf8@o4506237212033024.ingest.sentry.io/4506237213212672" + DJANGO_SENTRY_ENVIRONMENT = "production" + DJANGO_CELERY_WORKER_CONCURRENCY = "4" + DJANGO_DANDI_WEB_APP_URL = "https://lincbrain.org" + DJANGO_DANDI_API_URL = "https://api.lincbrain.org" + DJANGO_DANDI_JUPYTERHUB_URL = "https://hub.lincbrain.org" + } + additional_sensitive_django_vars = { + DJANGO_DANDI_DOI_API_PASSWORD = "temp" + } } resource "heroku_formation" "api_checksum_worker" { app_id = module.api.heroku_app_id type = "checksum-worker" - size = "standard-2x" + size = "standard-1x" quantity = 1 } @@ -64,4 +64,4 @@ resource "heroku_formation" "api_analytics_worker" { data "aws_iam_user" "api" { user_name = module.api.heroku_iam_user_id -} +} \ No newline at end of file diff --git a/terraform/domain.tf b/terraform/domain.tf index 525ccd5..07b8176 100644 --- a/terraform/domain.tf +++ b/terraform/domain.tf @@ -1,41 +1,62 @@ -resource "aws_route53_zone" "dandi" { - name = "dandiarchive.org" -} - -resource "aws_route53_record" "acm_validation" { - zone_id = aws_route53_zone.dandi.zone_id - name = "_cbe41dfe1888c2bb5c157cacc35e1722" - type = "CNAME" - ttl = "300" - records = ["_46df7ee9a9c17698aedbb737f220c63a.mzlfeqexyx.acm-validations.aws"] +resource "aws_route53_zone" "linc-brain-mit" { + name = "lincbrain.org" } resource "aws_route53_record" "gui" { - zone_id = aws_route53_zone.dandi.zone_id + zone_id = aws_route53_zone.linc-brain-mit.zone_id name = "" # apex type = "A" ttl = "300" - records = ["75.2.60.5"] # Netlify's load balancer, which will proxy to our app + records = ["75.2.60.5"] # Netlify's load balancer, which will proxy to our app -- https://docs.netlify.com/domains-https/custom-domains/configure-external-dns/#configure-an-apex-domain } resource "aws_route53_record" "gui-staging" { - zone_id = aws_route53_zone.dandi.zone_id + zone_id = aws_route53_zone.linc-brain-mit.zone_id name = "gui-staging" type = "CNAME" ttl = "300" - records = ["gui-staging-dandiarchive-org.netlify.com"] + records = ["staging--gui-staging-lincbrain-org.netlify.app"] } -resource "aws_route53_record" "www" { - zone_id = aws_route53_zone.dandi.zone_id - name = "www" - type = "CNAME" +resource "aws_acm_certificate" "cert" { + domain_name = "lincbrain.org" + validation_method = "DNS" + + subject_alternative_names = [ + "*.lincbrain.org", + "*--gui-staging-lincbrain-org.netlify.app" + ] +} + + +resource "aws_route53_record" "validation" { + for_each = { + for domain_validation_option in aws_acm_certificate.cert.domain_validation_options : domain_validation_option.domain_name => { + name = domain_validation_option.resource_record_name + record = domain_validation_option.resource_record_value + type = domain_validation_option.resource_record_type + } + } + + zone_id = aws_route53_zone.linc-brain-mit.zone_id + name = each.value.name + type = each.value.type + records = [each.value.record] ttl = "300" - records = ["dandi.github.io"] + + lifecycle { + create_before_destroy = true + ignore_changes = [records, name, type] + } +} + +resource "aws_acm_certificate_validation" "cert" { + certificate_arn = aws_acm_certificate.cert.arn + validation_record_fqdns = [for record in aws_route53_record.validation : record.fqdn] } resource "aws_route53_record" "email" { - zone_id = aws_route53_zone.dandi.zone_id + zone_id = aws_route53_zone.linc-brain-mit.zone_id name = "" # apex type = "MX" ttl = "300" @@ -46,7 +67,7 @@ resource "aws_route53_record" "email" { } resource "aws_route53_record" "email-spf" { - zone_id = aws_route53_zone.dandi.zone_id + zone_id = aws_route53_zone.linc-brain-mit.zone_id name = "" # apex type = "TXT" ttl = "300" diff --git a/terraform/main.tf b/terraform/main.tf index 577673e..86d47c7 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -1,37 +1,39 @@ terraform { backend "remote" { - organization = "dandi" + organization = "linc-brain-mit" workspaces { - name = "dandi-prod" + name = "linc-archive-terraform" } } } -// This is the "project" account, the primary account with most resources provider "aws" { - region = "us-east-2" - allowed_account_ids = ["278212569472"] - # Must set AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY envvars + region = "us-east-1" + allowed_account_ids = ["151312473579"] + + assume_role { + role_arn = "arn:aws:iam::151312473579:role/linc-infrastructure" + } } // The "sponsored" account, the Amazon-sponsored account with the public bucket provider "aws" { alias = "sponsored" - region = "us-east-2" - allowed_account_ids = ["769362853226"] + region = "us-east-1" + allowed_account_ids = ["151312473579"] # TODO: Aaron make new ID // This will authenticate using credentials from the project account, then assume the - // "dandi-infrastructure" role from the sponsored account to manage resources there + // "linc-infrastructure" role from the sponsored account to manage resources there assume_role { - role_arn = "arn:aws:iam::769362853226:role/dandi-infrastructure" + role_arn = "arn:aws:iam::151312473579:role/linc-infrastructure" } # Must set AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY envvars for project account } provider "heroku" { - # Must set HEROKU_EMAIL, HEROKU_API_KEY envvars + } provider "sentry" { diff --git a/terraform/modules/dandiset_bucket/log_bucket.tf b/terraform/modules/lincset_bucket/log_bucket.tf similarity index 80% rename from terraform/modules/dandiset_bucket/log_bucket.tf rename to terraform/modules/lincset_bucket/log_bucket.tf index 431a30b..c6adcee 100644 --- a/terraform/modules/dandiset_bucket/log_bucket.tf +++ b/terraform/modules/lincset_bucket/log_bucket.tf @@ -18,7 +18,7 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "log_bucket" { } } -data "aws_iam_policy_document" "dandiset_log_bucket_policy" { +data "aws_iam_policy_document" "lincset_log_bucket_policy" { statement { resources = [ "${aws_s3_bucket.log_bucket.arn}", @@ -52,7 +52,7 @@ data "aws_iam_policy_document" "dandiset_log_bucket_policy" { condition { test = "ArnLike" variable = "aws:SourceArn" - values = [aws_s3_bucket.dandiset_bucket.arn] + values = [aws_s3_bucket.lincset_bucket.arn] } principals { @@ -62,14 +62,14 @@ data "aws_iam_policy_document" "dandiset_log_bucket_policy" { } } -resource "aws_s3_bucket_policy" "dandiset_log_bucket_policy" { +resource "aws_s3_bucket_policy" "lincset_log_bucket_policy" { provider = aws bucket = aws_s3_bucket.log_bucket.id - policy = data.aws_iam_policy_document.dandiset_log_bucket_policy.json + policy = data.aws_iam_policy_document.lincset_log_bucket_policy.json } -data "aws_iam_policy_document" "dandiset_log_bucket_owner" { +data "aws_iam_policy_document" "lincset_log_bucket_owner" { version = "2008-10-17" // TODO: gate behind a "cross account" flag, since this is technically only @@ -87,12 +87,12 @@ data "aws_iam_policy_document" "dandiset_log_bucket_owner" { } } -resource "aws_iam_user_policy" "dandiset_log_bucket_owner" { +resource "aws_iam_user_policy" "lincset_log_bucket_owner" { // The Heroku IAM user will always be in the project account provider = aws.project name = "${var.log_bucket_name}-ownership-policy" user = var.heroku_user.user_name - policy = data.aws_iam_policy_document.dandiset_log_bucket_owner.json + policy = data.aws_iam_policy_document.lincset_log_bucket_owner.json } diff --git a/terraform/modules/dandiset_bucket/main.tf b/terraform/modules/lincset_bucket/main.tf similarity index 68% rename from terraform/modules/dandiset_bucket/main.tf rename to terraform/modules/lincset_bucket/main.tf index c3948ff..92afc96 100644 --- a/terraform/modules/dandiset_bucket/main.tf +++ b/terraform/modules/lincset_bucket/main.tf @@ -4,7 +4,7 @@ data "aws_caller_identity" "sponsored_account" { data "aws_caller_identity" "current" {} -resource "aws_s3_bucket" "dandiset_bucket" { +resource "aws_s3_bucket" "lincset_bucket" { bucket = var.bucket_name @@ -13,8 +13,8 @@ resource "aws_s3_bucket" "dandiset_bucket" { } } -resource "aws_s3_bucket_server_side_encryption_configuration" "dandiset_bucket" { - bucket = aws_s3_bucket.dandiset_bucket.id +resource "aws_s3_bucket_server_side_encryption_configuration" "lincset_bucket" { + bucket = aws_s3_bucket.lincset_bucket.id rule { apply_server_side_encryption_by_default { @@ -23,8 +23,8 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "dandiset_bucket" } } -resource "aws_s3_bucket_cors_configuration" "dandiset_bucket" { - bucket = aws_s3_bucket.dandiset_bucket.id +resource "aws_s3_bucket_cors_configuration" "lincset_bucket" { + bucket = aws_s3_bucket.lincset_bucket.id cors_rule { allowed_origins = [ @@ -46,57 +46,57 @@ resource "aws_s3_bucket_cors_configuration" "dandiset_bucket" { } } -resource "aws_s3_bucket_logging" "dandiset_bucket" { - bucket = aws_s3_bucket.dandiset_bucket.id +resource "aws_s3_bucket_logging" "lincset_bucket" { + bucket = aws_s3_bucket.lincset_bucket.id target_bucket = aws_s3_bucket.log_bucket.id target_prefix = "" } -resource "aws_s3_bucket_versioning" "dandiset_bucket" { +resource "aws_s3_bucket_versioning" "lincset_bucket" { count = var.versioning ? 1 : 0 - bucket = aws_s3_bucket.dandiset_bucket.id + bucket = aws_s3_bucket.lincset_bucket.id versioning_configuration { status = "Enabled" } } -resource "aws_s3_bucket_ownership_controls" "dandiset_bucket" { - bucket = aws_s3_bucket.dandiset_bucket.id +resource "aws_s3_bucket_ownership_controls" "lincset_bucket" { + bucket = aws_s3_bucket.lincset_bucket.id rule { object_ownership = "BucketOwnerPreferred" } } -resource "aws_s3_bucket_acl" "dandiset_bucket" { - depends_on = [aws_s3_bucket_ownership_controls.dandiset_bucket] +resource "aws_s3_bucket_acl" "lincset_bucket" { + depends_on = [aws_s3_bucket_ownership_controls.lincset_bucket] - bucket = aws_s3_bucket.dandiset_bucket.id + bucket = aws_s3_bucket.lincset_bucket.id // Public access is granted via a bucket policy, not a canned ACL acl = "private" } -resource "aws_iam_user_policy" "dandiset_bucket_owner" { +resource "aws_iam_user_policy" "lincset_bucket_owner" { // The Heroku IAM user will always be in the project account provider = aws.project name = "${var.bucket_name}-ownership-policy" user = var.heroku_user.user_name - policy = data.aws_iam_policy_document.dandiset_bucket_owner.json + policy = data.aws_iam_policy_document.lincset_bucket_owner.json } -data "aws_iam_policy_document" "dandiset_bucket_owner" { +data "aws_iam_policy_document" "lincset_bucket_owner" { version = "2008-10-17" statement { resources = [ - "${aws_s3_bucket.dandiset_bucket.arn}", - "${aws_s3_bucket.dandiset_bucket.arn}/*", + "${aws_s3_bucket.lincset_bucket.arn}", + "${aws_s3_bucket.lincset_bucket.arn}/*", ] actions = [ @@ -111,8 +111,8 @@ data "aws_iam_policy_document" "dandiset_bucket_owner" { content { resources = [ - "${aws_s3_bucket.dandiset_bucket.arn}", - "${aws_s3_bucket.dandiset_bucket.arn}/*", + "${aws_s3_bucket.lincset_bucket.arn}", + "${aws_s3_bucket.lincset_bucket.arn}/*", ] actions = ["s3:PutObject"] @@ -121,8 +121,8 @@ data "aws_iam_policy_document" "dandiset_bucket_owner" { statement { resources = [ - "${aws_s3_bucket.dandiset_bucket.arn}", - "${aws_s3_bucket.dandiset_bucket.arn}/*", + "${aws_s3_bucket.lincset_bucket.arn}", + "${aws_s3_bucket.lincset_bucket.arn}/*", ] actions = ["s3:*"] @@ -135,14 +135,14 @@ data "aws_iam_policy_document" "dandiset_bucket_owner" { } } -resource "aws_s3_bucket_policy" "dandiset_bucket_policy" { +resource "aws_s3_bucket_policy" "lincset_bucket_policy" { provider = aws - bucket = aws_s3_bucket.dandiset_bucket.id - policy = data.aws_iam_policy_document.dandiset_bucket_policy.json + bucket = aws_s3_bucket.lincset_bucket.id + policy = data.aws_iam_policy_document.lincset_bucket_policy.json } -data "aws_iam_policy_document" "dandiset_bucket_policy" { +data "aws_iam_policy_document" "lincset_bucket_policy" { version = "2008-10-17" dynamic "statement" { @@ -150,8 +150,8 @@ data "aws_iam_policy_document" "dandiset_bucket_policy" { content { resources = [ - "${aws_s3_bucket.dandiset_bucket.arn}", - "${aws_s3_bucket.dandiset_bucket.arn}/*", + "${aws_s3_bucket.lincset_bucket.arn}", + "${aws_s3_bucket.lincset_bucket.arn}/*", ] actions = [ @@ -179,7 +179,7 @@ data "aws_iam_policy_document" "dandiset_bucket_policy" { "s3:PutObject", ] resources = [ - "${aws_s3_bucket.dandiset_bucket.arn}/*", + "${aws_s3_bucket.lincset_bucket.arn}/*", ] condition { test = "StringEquals" @@ -194,15 +194,15 @@ data "aws_iam_policy_document" "dandiset_bucket_policy" { condition { test = "ArnLike" variable = "aws:SourceArn" - values = [aws_s3_bucket.dandiset_bucket.arn] + values = [aws_s3_bucket.lincset_bucket.arn] } } } statement { resources = [ - "${aws_s3_bucket.dandiset_bucket.arn}", - "${aws_s3_bucket.dandiset_bucket.arn}/*", + "${aws_s3_bucket.lincset_bucket.arn}", + "${aws_s3_bucket.lincset_bucket.arn}/*", ] actions = [ @@ -219,8 +219,8 @@ data "aws_iam_policy_document" "dandiset_bucket_policy" { statement { resources = [ - "${aws_s3_bucket.dandiset_bucket.arn}", - "${aws_s3_bucket.dandiset_bucket.arn}/*", + "${aws_s3_bucket.lincset_bucket.arn}", + "${aws_s3_bucket.lincset_bucket.arn}/*", ] actions = ["s3:*"] @@ -244,7 +244,7 @@ data "aws_iam_policy_document" "dandiset_bucket_policy" { sid = "PreventDeletionOfObjectVersions" resources = [ - "${aws_s3_bucket.dandiset_bucket.arn}/*" + "${aws_s3_bucket.lincset_bucket.arn}/*" ] actions = [ @@ -266,11 +266,11 @@ data "aws_iam_policy_document" "dandiset_bucket_policy" { # after 30 days. resource "aws_s3_bucket_lifecycle_configuration" "expire_deleted_objects" { # Must have bucket versioning enabled first - depends_on = [aws_s3_bucket_versioning.dandiset_bucket] + depends_on = [aws_s3_bucket_versioning.lincset_bucket] count = var.trailing_delete ? 1 : 0 - bucket = aws_s3_bucket.dandiset_bucket.id + bucket = aws_s3_bucket.lincset_bucket.id # Based on https://docs.aws.amazon.com/AmazonS3/latest/userguide/lifecycle-configuration-examples.html#lifecycle-config-conceptual-ex7 rule { diff --git a/terraform/modules/dandiset_bucket/outputs.tf b/terraform/modules/lincset_bucket/outputs.tf similarity index 70% rename from terraform/modules/dandiset_bucket/outputs.tf rename to terraform/modules/lincset_bucket/outputs.tf index 53a0f8a..65e5c02 100644 --- a/terraform/modules/dandiset_bucket/outputs.tf +++ b/terraform/modules/lincset_bucket/outputs.tf @@ -1,5 +1,5 @@ output "bucket_name" { - value = aws_s3_bucket.dandiset_bucket.id + value = aws_s3_bucket.lincset_bucket.id description = "The S3 bucket name." } @@ -9,6 +9,6 @@ output "log_bucket_name" { } output "bucket_arn" { - value = aws_s3_bucket.dandiset_bucket.arn + value = aws_s3_bucket.lincset_bucket.arn description = "The S3 bucket ARN." } diff --git a/terraform/modules/dandiset_bucket/variables.tf b/terraform/modules/lincset_bucket/variables.tf similarity index 100% rename from terraform/modules/dandiset_bucket/variables.tf rename to terraform/modules/lincset_bucket/variables.tf diff --git a/terraform/modules/dandiset_bucket/versions.tf b/terraform/modules/lincset_bucket/versions.tf similarity index 100% rename from terraform/modules/dandiset_bucket/versions.tf rename to terraform/modules/lincset_bucket/versions.tf diff --git a/terraform/redirector.tf b/terraform/redirector.tf index 09bb45f..9f565bc 100644 --- a/terraform/redirector.tf +++ b/terraform/redirector.tf @@ -1,8 +1,8 @@ # Record to point gui.dandiarchive.org to the Netlify hosted redirector resource "aws_route53_record" "redirector" { - zone_id = aws_route53_zone.dandi.zone_id + zone_id = aws_route53_zone.linc-brain-mit.zone_id name = "gui" type = "CNAME" ttl = "300" - records = ["redirect-dandiarchive-org.netlify.com"] + records = ["redirect-lincbrain-org.netlify.com"] } diff --git a/terraform/sentry.tf b/terraform/sentry.tf index dbbb251..d9cb573 100644 --- a/terraform/sentry.tf +++ b/terraform/sentry.tf @@ -1,10 +1,10 @@ data "sentry_organization" "this" { - slug = "dandiarchive" + slug = "lincbrain" } data "sentry_team" "this" { organization = data.sentry_organization.this.id - slug = "dandidevs" + slug = "linc-brain-devs" } data "sentry_project" "this" { diff --git a/terraform/sponsored_bucket.tf b/terraform/sponsored_bucket.tf index a1d32ce..4b14e1f 100644 --- a/terraform/sponsored_bucket.tf +++ b/terraform/sponsored_bucket.tf @@ -1,12 +1,12 @@ -module "sponsored_dandiset_bucket" { - source = "./modules/dandiset_bucket" - bucket_name = "dandiarchive" +module "sponsored_lincset_bucket" { + source = "./modules/lincset_bucket" + bucket_name = "linc-brain-mit" public = true versioning = true trailing_delete = false allow_cross_account_heroku_put_object = true heroku_user = data.aws_iam_user.api - log_bucket_name = "dandiarchive-logs" + log_bucket_name = "linc-brain-mit-logs" providers = { aws = aws.sponsored aws.project = aws @@ -14,12 +14,12 @@ module "sponsored_dandiset_bucket" { } module "sponsored_embargo_bucket" { - source = "./modules/dandiset_bucket" - bucket_name = "dandiarchive-embargo" + source = "./modules/lincset_bucket" + bucket_name = "linc-brain-mit-embargo" versioning = false trailing_delete = false heroku_user = data.aws_iam_user.api - log_bucket_name = "dandiarchive-embargo-logs" + log_bucket_name = "linc-brain-mit-embargo-logs" providers = { aws = aws.sponsored aws.project = aws diff --git a/terraform/sponsored_iam.tf b/terraform/sponsored_iam.tf index 1339be1..b0d9908 100644 --- a/terraform/sponsored_iam.tf +++ b/terraform/sponsored_iam.tf @@ -37,8 +37,8 @@ data "aws_iam_policy_document" "sponsored_writers" { "s3:GetObjectVersion", ] resources = [ - "${module.sponsored_dandiset_bucket.bucket_arn}/*", - module.sponsored_dandiset_bucket.bucket_arn, + "${module.sponsored_lincset_bucket.bucket_arn}/*", + module.sponsored_lincset_bucket.bucket_arn, ] } } diff --git a/terraform/staging_bucket.tf b/terraform/staging_bucket.tf index b6ceeea..87087b7 100644 --- a/terraform/staging_bucket.tf +++ b/terraform/staging_bucket.tf @@ -1,12 +1,12 @@ -module "staging_dandiset_bucket" { - source = "./modules/dandiset_bucket" - bucket_name = "dandi-api-staging-dandisets" +module "staging_lincset_bucket" { + source = "./modules/lincset_bucket" + bucket_name = "linc-api-staging-lincsets" public = true versioning = true trailing_delete = true allow_heroku_put_object = true heroku_user = data.aws_iam_user.api_staging - log_bucket_name = "dandi-api-staging-dandiset-logs" + log_bucket_name = "linc-api-staging-lincset-logs" providers = { aws = aws aws.project = aws @@ -14,14 +14,14 @@ module "staging_dandiset_bucket" { } module "staging_embargo_bucket" { - source = "./modules/dandiset_bucket" - bucket_name = "dandi-api-staging-embargo-dandisets" + source = "./modules/lincset_bucket" + bucket_name = "linc-api-staging-embargo-lincsets" versioning = false trailing_delete = false heroku_user = data.aws_iam_user.api_staging - log_bucket_name = "dandi-api-staging-embargo-dandisets-logs" + log_bucket_name = "linc-api-staging-embargo-lincset-logs" providers = { aws = aws aws.project = aws } -} +} \ No newline at end of file diff --git a/terraform/staging_pipeline.tf b/terraform/staging_pipeline.tf index fdd3b0a..c784e46 100644 --- a/terraform/staging_pipeline.tf +++ b/terraform/staging_pipeline.tf @@ -5,10 +5,10 @@ module "api_staging" { source = "girder/girder4/heroku" version = "0.13.0" - project_slug = "dandi-api-staging" - heroku_team_name = data.heroku_team.dandi.name - route53_zone_id = aws_route53_zone.dandi.zone_id - subdomain_name = "api-staging" + project_slug = "linc-brain-staging" + heroku_team_name = data.heroku_team.linc-brain-mit.name + route53_zone_id = aws_route53_zone.linc-brain-mit.zone_id + subdomain_name = "staging-api" heroku_web_dyno_size = "basic" heroku_worker_dyno_size = "basic" @@ -19,31 +19,31 @@ module "api_staging" { heroku_web_dyno_quantity = 1 heroku_worker_dyno_quantity = 1 - django_default_from_email = "admin@api-staging.dandiarchive.org" - django_cors_origin_whitelist = ["https://gui-staging.dandiarchive.org"] - django_cors_origin_regex_whitelist = ["^https:\\/\\/[0-9a-z\\-]+--gui-staging-dandiarchive-org\\.netlify\\.app$"] + django_default_from_email = "admin@staging-api.lincbrain.org" + django_cors_origin_whitelist = ["https://gui-staging.lincbrain.org", "https://staging--gui-staging-lincbrain-org.netlify.app"] + django_cors_origin_regex_whitelist = ["https://staging--gui-staging-lincbrain-org.netlify.app"] additional_django_vars = { DJANGO_CONFIGURATION = "HerokuStagingConfiguration" - DJANGO_DANDI_DANDISETS_BUCKET_NAME = module.staging_dandiset_bucket.bucket_name + DJANGO_DANDI_DANDISETS_BUCKET_NAME = module.staging_lincset_bucket.bucket_name DJANGO_DANDI_DANDISETS_BUCKET_PREFIX = "" DJANGO_DANDI_DANDISETS_EMBARGO_BUCKET_NAME = module.staging_embargo_bucket.bucket_name DJANGO_DANDI_DANDISETS_EMBARGO_BUCKET_PREFIX = "" - DJANGO_DANDI_DANDISETS_LOG_BUCKET_NAME = module.staging_dandiset_bucket.log_bucket_name + DJANGO_DANDI_DANDISETS_LOG_BUCKET_NAME = module.staging_lincset_bucket.log_bucket_name DJANGO_DANDI_DANDISETS_EMBARGO_LOG_BUCKET_NAME = module.staging_embargo_bucket.log_bucket_name DJANGO_DANDI_DOI_API_URL = "https://api.test.datacite.org/dois" DJANGO_DANDI_DOI_API_USER = "dartlib.dandi" DJANGO_DANDI_DOI_API_PREFIX = "10.80507" DJANGO_DANDI_DOI_PUBLISH = "false" - DJANGO_SENTRY_DSN = data.sentry_key.this.dsn_public + DJANGO_SENTRY_DSN = "https://833c159dc622528b21b4ce4adef6dbf8@o4506237212033024.ingest.sentry.io/4506237213212672" DJANGO_SENTRY_ENVIRONMENT = "staging" DJANGO_CELERY_WORKER_CONCURRENCY = "2" - DJANGO_DANDI_WEB_APP_URL = "https://gui-staging.dandiarchive.org" - DJANGO_DANDI_API_URL = "https://api-staging.dandiarchive.org" - DJANGO_DANDI_JUPYTERHUB_URL = "https://hub.dandiarchive.org/" + DJANGO_DANDI_WEB_APP_URL = "https://staging--gui-staging-lincbrain-org.netlify.app/" + DJANGO_DANDI_API_URL = "https://staging-api.lincbrain.org/" + DJANGO_DANDI_JUPYTERHUB_URL = "https://hub.lincbrain.org/" } additional_sensitive_django_vars = { - DJANGO_DANDI_DOI_API_PASSWORD = var.test_doi_api_password + DJANGO_DANDI_DOI_API_PASSWORD = "temp" } } @@ -65,23 +65,23 @@ data "aws_iam_user" "api_staging" { user_name = module.api_staging.heroku_iam_user_id } -resource "heroku_pipeline" "dandi_pipeline" { - name = "dandi-pipeline" +resource "heroku_pipeline" "linc_pipeline" { + name = "linc-pipeline" owner { - id = data.heroku_team.dandi.id + id = data.heroku_team.linc-brain-mit.id type = "team" } } resource "heroku_pipeline_coupling" "staging" { app_id = module.api_staging.heroku_app_id - pipeline = heroku_pipeline.dandi_pipeline.id + pipeline = heroku_pipeline.linc_pipeline.id stage = "staging" } resource "heroku_pipeline_coupling" "production" { app_id = module.api.heroku_app_id - pipeline = heroku_pipeline.dandi_pipeline.id + pipeline = heroku_pipeline.linc_pipeline.id stage = "production" -} +} \ No newline at end of file From d8e36e58ef86a9011d9f7a888c4a0cc70b7a8101 Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Tue, 6 Feb 2024 12:02:08 -0500 Subject: [PATCH 02/35] couple more changes --- README.md | 4 ++-- terraform/redirector.tf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 94c68dc..b96ba14 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,2 @@ -# dandi-infrastructure -Deployment infrastructure for the DANDI Archive. +# linc-archive-infrastructure +Deployment infrastructure for the LINC Brain Archive. diff --git a/terraform/redirector.tf b/terraform/redirector.tf index 9f565bc..bd2f078 100644 --- a/terraform/redirector.tf +++ b/terraform/redirector.tf @@ -1,4 +1,4 @@ -# Record to point gui.dandiarchive.org to the Netlify hosted redirector +# Record to point gui.lincbrain.org to the Netlify hosted redirector resource "aws_route53_record" "redirector" { zone_id = aws_route53_zone.linc-brain-mit.zone_id name = "gui" From 8c08b97e26c317c6ba4c9371df631863c5d5a954 Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Tue, 6 Feb 2024 12:23:29 -0500 Subject: [PATCH 03/35] bump some values --- terraform/api.tf | 2 +- terraform/variables.tf | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/terraform/api.tf b/terraform/api.tf index 66f0499..b017c4d 100644 --- a/terraform/api.tf +++ b/terraform/api.tf @@ -6,7 +6,7 @@ module "api" { source = "girder/girder4/heroku" version = "0.13.0" - project_slug = "linc-staging-terraform" + project_slug = "linc-brain-prod" heroku_team_name = data.heroku_team.linc-brain-mit.name route53_zone_id = aws_route53_zone.linc-brain-mit.zone_id subdomain_name = "api" diff --git a/terraform/variables.tf b/terraform/variables.tf index 9d26cb7..4cdb22e 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -1,9 +1,11 @@ variable "doi_api_password" { type = string description = "The password for the Datacite API, used to mint new DOIs during publish." + default = "yourdefaultpassword" } variable "test_doi_api_password" { type = string description = "The password for the Datacite Test API, used to mint new DOIs on staging during publish." + default = "yourtestdefaultpassword" } From 19d49296e6c9bb1d410c15777d6d4da10ad771a9 Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Tue, 6 Feb 2024 16:11:52 -0500 Subject: [PATCH 04/35] include sentry vars --- terraform/sentry.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/sentry.tf b/terraform/sentry.tf index d9cb573..d621e5d 100644 --- a/terraform/sentry.tf +++ b/terraform/sentry.tf @@ -1,15 +1,15 @@ data "sentry_organization" "this" { - slug = "lincbrain" + slug = "mit-m3" } data "sentry_team" "this" { organization = data.sentry_organization.this.id - slug = "linc-brain-devs" + slug = "mit" } data "sentry_project" "this" { organization = data.sentry_organization.this.id - slug = "dandi-api" + slug = "linc-api" } data "sentry_key" "this" { From ef26355587dbd1479b4edac814789fbb053e07bc Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Tue, 6 Feb 2024 16:29:27 -0500 Subject: [PATCH 05/35] resolve acm terraform issue --- terraform/domain.tf | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/terraform/domain.tf b/terraform/domain.tf index 07b8176..8aedaca 100644 --- a/terraform/domain.tf +++ b/terraform/domain.tf @@ -10,21 +10,20 @@ resource "aws_route53_record" "gui" { records = ["75.2.60.5"] # Netlify's load balancer, which will proxy to our app -- https://docs.netlify.com/domains-https/custom-domains/configure-external-dns/#configure-an-apex-domain } -resource "aws_route53_record" "gui-staging" { - zone_id = aws_route53_zone.linc-brain-mit.zone_id - name = "gui-staging" - type = "CNAME" - ttl = "300" - records = ["staging--gui-staging-lincbrain-org.netlify.app"] -} +# resource "aws_route53_record" "gui-staging" { +# zone_id = aws_route53_zone.linc-brain-mit.zone_id +# name = "gui-staging" +# type = "CNAME" +# ttl = "300" +# records = ["staging--gui-staging-lincbrain-org.netlify.app"] +# } resource "aws_acm_certificate" "cert" { domain_name = "lincbrain.org" validation_method = "DNS" subject_alternative_names = [ - "*.lincbrain.org", - "*--gui-staging-lincbrain-org.netlify.app" + "*.lincbrain.org" ] } From 9e9a79710b40f25edd03e84135ed22c02554c424 Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Tue, 6 Feb 2024 17:11:23 -0500 Subject: [PATCH 06/35] Temp ACL alteration for S3 --- terraform/modules/lincset_bucket/main.tf | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/terraform/modules/lincset_bucket/main.tf b/terraform/modules/lincset_bucket/main.tf index 92afc96..15e1f01 100644 --- a/terraform/modules/lincset_bucket/main.tf +++ b/terraform/modules/lincset_bucket/main.tf @@ -71,14 +71,14 @@ resource "aws_s3_bucket_ownership_controls" "lincset_bucket" { } } -resource "aws_s3_bucket_acl" "lincset_bucket" { - depends_on = [aws_s3_bucket_ownership_controls.lincset_bucket] - - bucket = aws_s3_bucket.lincset_bucket.id - - // Public access is granted via a bucket policy, not a canned ACL - acl = "private" -} +# resource "aws_s3_bucket_acl" "lincset_bucket" { +# depends_on = [aws_s3_bucket_ownership_controls.lincset_bucket] +# +# bucket = aws_s3_bucket.lincset_bucket.id +# +# // Public access is granted via a bucket policy, not a canned ACL +# acl = "private" +# } resource "aws_iam_user_policy" "lincset_bucket_owner" { // The Heroku IAM user will always be in the project account From 280de88e576424bdb5111bdc3f3560e61d94be5d Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Wed, 7 Feb 2024 11:13:23 -0500 Subject: [PATCH 07/35] bump module --- terraform/modules/lincset_bucket/main.tf | 248 +++++++++++------------ 1 file changed, 124 insertions(+), 124 deletions(-) diff --git a/terraform/modules/lincset_bucket/main.tf b/terraform/modules/lincset_bucket/main.tf index 15e1f01..c1c15b9 100644 --- a/terraform/modules/lincset_bucket/main.tf +++ b/terraform/modules/lincset_bucket/main.tf @@ -135,131 +135,131 @@ data "aws_iam_policy_document" "lincset_bucket_owner" { } } -resource "aws_s3_bucket_policy" "lincset_bucket_policy" { - provider = aws - - bucket = aws_s3_bucket.lincset_bucket.id - policy = data.aws_iam_policy_document.lincset_bucket_policy.json -} - -data "aws_iam_policy_document" "lincset_bucket_policy" { - version = "2008-10-17" - - dynamic "statement" { - for_each = var.public ? [1] : [] - - content { - resources = [ - "${aws_s3_bucket.lincset_bucket.arn}", - "${aws_s3_bucket.lincset_bucket.arn}/*", - ] - - actions = [ - "s3:Get*", - "s3:List*", - ] - - principals { - identifiers = ["*"] - type = "*" - } - } - } - - dynamic "statement" { - for_each = var.allow_cross_account_heroku_put_object ? [1] : [] - - content { - sid = "S3PolicyStmt-DO-NOT-MODIFY-1569973164923" - principals { - identifiers = ["s3.amazonaws.com"] - type = "Service" - } - actions = [ - "s3:PutObject", - ] - resources = [ - "${aws_s3_bucket.lincset_bucket.arn}/*", - ] - condition { - test = "StringEquals" - variable = "aws:SourceAccount" - values = [data.aws_caller_identity.sponsored_account.account_id] - } - condition { - test = "StringEquals" - variable = "s3:x-amz-acl" - values = ["bucket-owner-full-control"] - } - condition { - test = "ArnLike" - variable = "aws:SourceArn" - values = [aws_s3_bucket.lincset_bucket.arn] - } - } - } - - statement { - resources = [ - "${aws_s3_bucket.lincset_bucket.arn}", - "${aws_s3_bucket.lincset_bucket.arn}/*", - ] - - actions = [ - "s3:Get*", - "s3:List*", - "s3:Delete*", - ] - - principals { - type = "AWS" - identifiers = [var.heroku_user.arn] - } - } - - statement { - resources = [ - "${aws_s3_bucket.lincset_bucket.arn}", - "${aws_s3_bucket.lincset_bucket.arn}/*", - ] - - actions = ["s3:*"] - - condition { - test = "StringEquals" - variable = "s3:x-amz-acl" - values = ["bucket-owner-full-control"] - } - - principals { - type = "AWS" - identifiers = [var.heroku_user.arn] - } - } - - dynamic "statement" { - for_each = var.trailing_delete ? [1] : [] - - content { - sid = "PreventDeletionOfObjectVersions" - - resources = [ - "${aws_s3_bucket.lincset_bucket.arn}/*" - ] - - actions = [ - "s3:DeleteObjectVersion", - ] - - effect = "Deny" +# resource "aws_s3_bucket_policy" "lincset_bucket_policy" { +# provider = aws +# +# bucket = aws_s3_bucket.lincset_bucket.id +# policy = data.aws_iam_policy_document.lincset_bucket_policy.json +# } - principals { - identifiers = ["*"] - type = "*" - } - } - } -} +# data "aws_iam_policy_document" "lincset_bucket_policy" { +# version = "2008-10-17" +# +# dynamic "statement" { +# for_each = var.public ? [1] : [] +# +# content { +# resources = [ +# "${aws_s3_bucket.lincset_bucket.arn}", +# "${aws_s3_bucket.lincset_bucket.arn}/*", +# ] +# +# actions = [ +# "s3:Get*", +# "s3:List*", +# ] +# +# principals { +# identifiers = ["*"] +# type = "*" +# } +# } +# } +# +# dynamic "statement" { +# for_each = var.allow_cross_account_heroku_put_object ? [1] : [] +# +# content { +# sid = "S3PolicyStmt-DO-NOT-MODIFY-1569973164923" +# principals { +# identifiers = ["s3.amazonaws.com"] +# type = "Service" +# } +# actions = [ +# "s3:PutObject", +# ] +# resources = [ +# "${aws_s3_bucket.lincset_bucket.arn}/*", +# ] +# condition { +# test = "StringEquals" +# variable = "aws:SourceAccount" +# values = [data.aws_caller_identity.sponsored_account.account_id] +# } +# condition { +# test = "StringEquals" +# variable = "s3:x-amz-acl" +# values = ["bucket-owner-full-control"] +# } +# condition { +# test = "ArnLike" +# variable = "aws:SourceArn" +# values = [aws_s3_bucket.lincset_bucket.arn] +# } +# } +# } +# +# statement { +# resources = [ +# "${aws_s3_bucket.lincset_bucket.arn}", +# "${aws_s3_bucket.lincset_bucket.arn}/*", +# ] +# +# actions = [ +# "s3:Get*", +# "s3:List*", +# "s3:Delete*", +# ] +# +# principals { +# type = "AWS" +# identifiers = [var.heroku_user.arn] +# } +# } +# +# statement { +# resources = [ +# "${aws_s3_bucket.lincset_bucket.arn}", +# "${aws_s3_bucket.lincset_bucket.arn}/*", +# ] +# +# actions = ["s3:*"] +# +# condition { +# test = "StringEquals" +# variable = "s3:x-amz-acl" +# values = ["bucket-owner-full-control"] +# } +# +# principals { +# type = "AWS" +# identifiers = [var.heroku_user.arn] +# } +# } +# +# dynamic "statement" { +# for_each = var.trailing_delete ? [1] : [] +# +# content { +# sid = "PreventDeletionOfObjectVersions" +# +# resources = [ +# "${aws_s3_bucket.lincset_bucket.arn}/*" +# ] +# +# actions = [ +# "s3:DeleteObjectVersion", +# ] +# +# effect = "Deny" +# +# principals { +# identifiers = ["*"] +# type = "*" +# } +# } +# } +# } # S3 lifecycle policy that permanently deletes objects with delete markers From eb405cf0b6203a1d2e810876889f24b64b11082a Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Wed, 7 Feb 2024 11:15:41 -0500 Subject: [PATCH 08/35] revert --- terraform/.terraform.lock.hcl | 7 +- terraform/modules/lincset_bucket/main.tf | 248 +++++++++++------------ 2 files changed, 130 insertions(+), 125 deletions(-) diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl index 66378d7..47cb290 100644 --- a/terraform/.terraform.lock.hcl +++ b/terraform/.terraform.lock.hcl @@ -3,9 +3,10 @@ provider "registry.terraform.io/hashicorp/aws" { version = "5.17.0" - constraints = ">= 4.9.0" + constraints = ">= 4.9.0, >= 4.30.0" hashes = [ "h1:U+EDfeUqefebA1h7KyBMD1xH0h311LMi7wijPDPkC/0=", + "h1:rplvK7UGP2FuzM44t2eRX+QYYPC0aUIoKdi5XayRI8M=", "zh:0087b9dd2c9c638fd63e527e5b9b70988008e263d480a199f180efe5a4f070f0", "zh:0fd532a4fd03ddef11f0502ff9fe4343443e1ae805cb088825a71d6d48906ec7", "zh:16411e731100cd15f7e165f53c23be784b2c86c2fcfd34781e0642d17090d342", @@ -28,6 +29,7 @@ provider "registry.terraform.io/hashicorp/local" { version = "2.4.0" hashes = [ "h1:R97FTYETo88sT2VHfMgkPU3lzCsZLunPftjSI5vfKe8=", + "h1:ZUEYUmm2t4vxwzxy1BvN1wL6SDWrDxfH7pxtzX8c6d0=", "zh:53604cd29cb92538668fe09565c739358dc53ca56f9f11312b9d7de81e48fab9", "zh:66a46e9c508716a1c98efbf793092f03d50049fa4a83cd6b2251e9a06aca2acf", "zh:70a6f6a852dd83768d0778ce9817d81d4b3f073fab8fa570bff92dcb0824f732", @@ -46,6 +48,7 @@ provider "registry.terraform.io/hashicorp/local" { provider "registry.terraform.io/hashicorp/random" { version = "3.5.1" hashes = [ + "h1:IL9mSatmwov+e0+++YX2V6uel+dV6bn+fC/cnGDK3Ck=", "h1:VSnd9ZIPyfKHOObuQCaKfnjIHRtR7qTw19Rz8tJxm+k=", "zh:04e3fbd610cb52c1017d282531364b9c53ef72b6bc533acb2a90671957324a64", "zh:119197103301ebaf7efb91df8f0b6e0dd31e6ff943d231af35ee1831c599188d", @@ -66,6 +69,7 @@ provider "registry.terraform.io/heroku/heroku" { version = "5.2.6" constraints = ">= 5.0.0" hashes = [ + "h1:YQl8iAwo1sEzUd4zhLs87nuk3La/iNyLa4G0eLfvSvA=", "h1:a8S25Rq0oLoUeIjpNpEaeuTlw0vua4o1vTPbrjU4aSg=", "zh:1a50b1749f49377edf368695309a8945435e8c2803adb981a382fa87a4eda1f0", "zh:1d2c157bf1619acdc0f02f0f43c4852d38fce1e75b23a8c4f32a0751cc31fa7a", @@ -90,6 +94,7 @@ provider "registry.terraform.io/jianyuan/sentry" { version = "0.12.2" hashes = [ "h1:7VdazIutMx8oGuy8S6iC1rL2IaU8S6qFzZO0dVzta9A=", + "h1:e7ldfvSDL4VSTNGNvr8439/ttvY9KpVzAcg4tDKk304=", "zh:0dde99e7b343fa01f8eefc378171fb8621bedb20f59157d6cc8e3d46c738105f", "zh:1b0d79eb5343187724c85996b3972d00daf242569395f6ca2e1a88286146223e", "zh:22b6766f1fae35823b3881c198e3965c690cb87309af1527df9cc5781f7633d4", diff --git a/terraform/modules/lincset_bucket/main.tf b/terraform/modules/lincset_bucket/main.tf index c1c15b9..15e1f01 100644 --- a/terraform/modules/lincset_bucket/main.tf +++ b/terraform/modules/lincset_bucket/main.tf @@ -135,131 +135,131 @@ data "aws_iam_policy_document" "lincset_bucket_owner" { } } -# resource "aws_s3_bucket_policy" "lincset_bucket_policy" { -# provider = aws -# -# bucket = aws_s3_bucket.lincset_bucket.id -# policy = data.aws_iam_policy_document.lincset_bucket_policy.json -# } +resource "aws_s3_bucket_policy" "lincset_bucket_policy" { + provider = aws -# data "aws_iam_policy_document" "lincset_bucket_policy" { -# version = "2008-10-17" -# -# dynamic "statement" { -# for_each = var.public ? [1] : [] -# -# content { -# resources = [ -# "${aws_s3_bucket.lincset_bucket.arn}", -# "${aws_s3_bucket.lincset_bucket.arn}/*", -# ] -# -# actions = [ -# "s3:Get*", -# "s3:List*", -# ] -# -# principals { -# identifiers = ["*"] -# type = "*" -# } -# } -# } -# -# dynamic "statement" { -# for_each = var.allow_cross_account_heroku_put_object ? [1] : [] -# -# content { -# sid = "S3PolicyStmt-DO-NOT-MODIFY-1569973164923" -# principals { -# identifiers = ["s3.amazonaws.com"] -# type = "Service" -# } -# actions = [ -# "s3:PutObject", -# ] -# resources = [ -# "${aws_s3_bucket.lincset_bucket.arn}/*", -# ] -# condition { -# test = "StringEquals" -# variable = "aws:SourceAccount" -# values = [data.aws_caller_identity.sponsored_account.account_id] -# } -# condition { -# test = "StringEquals" -# variable = "s3:x-amz-acl" -# values = ["bucket-owner-full-control"] -# } -# condition { -# test = "ArnLike" -# variable = "aws:SourceArn" -# values = [aws_s3_bucket.lincset_bucket.arn] -# } -# } -# } -# -# statement { -# resources = [ -# "${aws_s3_bucket.lincset_bucket.arn}", -# "${aws_s3_bucket.lincset_bucket.arn}/*", -# ] -# -# actions = [ -# "s3:Get*", -# "s3:List*", -# "s3:Delete*", -# ] -# -# principals { -# type = "AWS" -# identifiers = [var.heroku_user.arn] -# } -# } -# -# statement { -# resources = [ -# "${aws_s3_bucket.lincset_bucket.arn}", -# "${aws_s3_bucket.lincset_bucket.arn}/*", -# ] -# -# actions = ["s3:*"] -# -# condition { -# test = "StringEquals" -# variable = "s3:x-amz-acl" -# values = ["bucket-owner-full-control"] -# } -# -# principals { -# type = "AWS" -# identifiers = [var.heroku_user.arn] -# } -# } -# -# dynamic "statement" { -# for_each = var.trailing_delete ? [1] : [] -# -# content { -# sid = "PreventDeletionOfObjectVersions" -# -# resources = [ -# "${aws_s3_bucket.lincset_bucket.arn}/*" -# ] -# -# actions = [ -# "s3:DeleteObjectVersion", -# ] -# -# effect = "Deny" -# -# principals { -# identifiers = ["*"] -# type = "*" -# } -# } -# } -# } + bucket = aws_s3_bucket.lincset_bucket.id + policy = data.aws_iam_policy_document.lincset_bucket_policy.json +} + +data "aws_iam_policy_document" "lincset_bucket_policy" { + version = "2008-10-17" + + dynamic "statement" { + for_each = var.public ? [1] : [] + + content { + resources = [ + "${aws_s3_bucket.lincset_bucket.arn}", + "${aws_s3_bucket.lincset_bucket.arn}/*", + ] + + actions = [ + "s3:Get*", + "s3:List*", + ] + + principals { + identifiers = ["*"] + type = "*" + } + } + } + + dynamic "statement" { + for_each = var.allow_cross_account_heroku_put_object ? [1] : [] + + content { + sid = "S3PolicyStmt-DO-NOT-MODIFY-1569973164923" + principals { + identifiers = ["s3.amazonaws.com"] + type = "Service" + } + actions = [ + "s3:PutObject", + ] + resources = [ + "${aws_s3_bucket.lincset_bucket.arn}/*", + ] + condition { + test = "StringEquals" + variable = "aws:SourceAccount" + values = [data.aws_caller_identity.sponsored_account.account_id] + } + condition { + test = "StringEquals" + variable = "s3:x-amz-acl" + values = ["bucket-owner-full-control"] + } + condition { + test = "ArnLike" + variable = "aws:SourceArn" + values = [aws_s3_bucket.lincset_bucket.arn] + } + } + } + + statement { + resources = [ + "${aws_s3_bucket.lincset_bucket.arn}", + "${aws_s3_bucket.lincset_bucket.arn}/*", + ] + + actions = [ + "s3:Get*", + "s3:List*", + "s3:Delete*", + ] + + principals { + type = "AWS" + identifiers = [var.heroku_user.arn] + } + } + + statement { + resources = [ + "${aws_s3_bucket.lincset_bucket.arn}", + "${aws_s3_bucket.lincset_bucket.arn}/*", + ] + + actions = ["s3:*"] + + condition { + test = "StringEquals" + variable = "s3:x-amz-acl" + values = ["bucket-owner-full-control"] + } + + principals { + type = "AWS" + identifiers = [var.heroku_user.arn] + } + } + + dynamic "statement" { + for_each = var.trailing_delete ? [1] : [] + + content { + sid = "PreventDeletionOfObjectVersions" + + resources = [ + "${aws_s3_bucket.lincset_bucket.arn}/*" + ] + + actions = [ + "s3:DeleteObjectVersion", + ] + + effect = "Deny" + + principals { + identifiers = ["*"] + type = "*" + } + } + } +} # S3 lifecycle policy that permanently deletes objects with delete markers From d2a2c2d3896709df3ec2cc686750d7f194336809 Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Wed, 7 Feb 2024 14:35:28 -0500 Subject: [PATCH 09/35] include object lock on the main bucket where dandisets are stored --- terraform/modules/lincset_bucket/main.tf | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/terraform/modules/lincset_bucket/main.tf b/terraform/modules/lincset_bucket/main.tf index 15e1f01..50e9be0 100644 --- a/terraform/modules/lincset_bucket/main.tf +++ b/terraform/modules/lincset_bucket/main.tf @@ -5,12 +5,22 @@ data "aws_caller_identity" "sponsored_account" { data "aws_caller_identity" "current" {} resource "aws_s3_bucket" "lincset_bucket" { - bucket = var.bucket_name lifecycle { prevent_destroy = true } + + object_lock_configuration { + object_lock_enabled = "Enabled" + + rule { + default_retention { + mode = "GOVERNANCE" + # No days or years specified, implying indefinite retention + } + } + } } resource "aws_s3_bucket_server_side_encryption_configuration" "lincset_bucket" { From d17581a7de1278a78de1a5ac6a5cd83826864b3f Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Wed, 7 Feb 2024 15:00:07 -0500 Subject: [PATCH 10/35] use not deprecated object lock definition --- terraform/modules/lincset_bucket/main.tf | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/terraform/modules/lincset_bucket/main.tf b/terraform/modules/lincset_bucket/main.tf index 50e9be0..eb089a6 100644 --- a/terraform/modules/lincset_bucket/main.tf +++ b/terraform/modules/lincset_bucket/main.tf @@ -11,14 +11,14 @@ resource "aws_s3_bucket" "lincset_bucket" { prevent_destroy = true } - object_lock_configuration { - object_lock_enabled = "Enabled" +} - rule { - default_retention { - mode = "GOVERNANCE" - # No days or years specified, implying indefinite retention - } +resource "aws_s3_bucket_object_lock_configuration" "lincset_bucket" { + bucket = var.bucket_name + + rule { + default_retention { + mode = "GOVERNANCE" } } } From 6592867ccd81ea8c6a93d37a56f9812968f04172 Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Wed, 7 Feb 2024 15:28:39 -0500 Subject: [PATCH 11/35] lincset setting --- terraform/modules/lincset_bucket/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/modules/lincset_bucket/main.tf b/terraform/modules/lincset_bucket/main.tf index eb089a6..a1210f7 100644 --- a/terraform/modules/lincset_bucket/main.tf +++ b/terraform/modules/lincset_bucket/main.tf @@ -10,7 +10,6 @@ resource "aws_s3_bucket" "lincset_bucket" { lifecycle { prevent_destroy = true } - } resource "aws_s3_bucket_object_lock_configuration" "lincset_bucket" { @@ -23,6 +22,7 @@ resource "aws_s3_bucket_object_lock_configuration" "lincset_bucket" { } } + resource "aws_s3_bucket_server_side_encryption_configuration" "lincset_bucket" { bucket = aws_s3_bucket.lincset_bucket.id From 12fa7d23c741c46c8805ae0b9fb9375fa48a0e0b Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Wed, 7 Feb 2024 15:39:04 -0500 Subject: [PATCH 12/35] be more explicit with object lock --- terraform/modules/lincset_bucket/main.tf | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/terraform/modules/lincset_bucket/main.tf b/terraform/modules/lincset_bucket/main.tf index a1210f7..24e1137 100644 --- a/terraform/modules/lincset_bucket/main.tf +++ b/terraform/modules/lincset_bucket/main.tf @@ -7,21 +7,24 @@ data "aws_caller_identity" "current" {} resource "aws_s3_bucket" "lincset_bucket" { bucket = var.bucket_name - lifecycle { - prevent_destroy = true + versioning { + enabled = true } -} -resource "aws_s3_bucket_object_lock_configuration" "lincset_bucket" { - bucket = var.bucket_name + object_lock_configuration { + object_lock_enabled = "Enabled" - rule { - default_retention { - mode = "GOVERNANCE" + rule { + default_retention { + mode = "GOVERNANCE" + } } } -} + lifecycle { + prevent_destroy = true + } +} resource "aws_s3_bucket_server_side_encryption_configuration" "lincset_bucket" { bucket = aws_s3_bucket.lincset_bucket.id From 2a4545d2247c9877736a8f8aa1d1aac38c7619a7 Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Wed, 7 Feb 2024 17:05:49 -0500 Subject: [PATCH 13/35] update email address django config --- terraform/api.tf | 2 +- terraform/modules/lincset_bucket/main.tf | 16 +--------------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/terraform/api.tf b/terraform/api.tf index b017c4d..3c8cf06 100644 --- a/terraform/api.tf +++ b/terraform/api.tf @@ -20,7 +20,7 @@ module "api" { heroku_web_dyno_quantity = 1 heroku_worker_dyno_quantity = 1 - django_default_from_email = "admin@api.lincbrain.org" + django_default_from_email = "admin@lincbrain.org" django_cors_origin_whitelist = ["https://lincbrain.org"] django_cors_origin_regex_whitelist = ["^https:\\/\\/[0-9a-z\\-]+\\.netlify\\.app$"] diff --git a/terraform/modules/lincset_bucket/main.tf b/terraform/modules/lincset_bucket/main.tf index 24e1137..2350530 100644 --- a/terraform/modules/lincset_bucket/main.tf +++ b/terraform/modules/lincset_bucket/main.tf @@ -7,24 +7,10 @@ data "aws_caller_identity" "current" {} resource "aws_s3_bucket" "lincset_bucket" { bucket = var.bucket_name - versioning { - enabled = true - } - - object_lock_configuration { - object_lock_enabled = "Enabled" - - rule { - default_retention { - mode = "GOVERNANCE" - } - } - } - lifecycle { prevent_destroy = true } -} +}` resource "aws_s3_bucket_server_side_encryption_configuration" "lincset_bucket" { bucket = aws_s3_bucket.lincset_bucket.id From f50483cf1e02711c871b789dbbe1c8b637af666d Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Wed, 7 Feb 2024 17:11:24 -0500 Subject: [PATCH 14/35] trailing comma --- terraform/modules/lincset_bucket/main.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/terraform/modules/lincset_bucket/main.tf b/terraform/modules/lincset_bucket/main.tf index 2350530..e9e4370 100644 --- a/terraform/modules/lincset_bucket/main.tf +++ b/terraform/modules/lincset_bucket/main.tf @@ -10,7 +10,8 @@ resource "aws_s3_bucket" "lincset_bucket" { lifecycle { prevent_destroy = true } -}` + +} resource "aws_s3_bucket_server_side_encryption_configuration" "lincset_bucket" { bucket = aws_s3_bucket.lincset_bucket.id From d97c98211df6d230cd3ec4df562b0493b7d3f19e Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Fri, 9 Feb 2024 14:58:44 -0500 Subject: [PATCH 15/35] Create new s3 buckets without object lock for prod and staging --- terraform/modules/lincset_bucket/main.tf | 16 ++++++++-------- terraform/sponsored_bucket.tf | 4 ++-- terraform/staging_bucket.tf | 8 ++++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/terraform/modules/lincset_bucket/main.tf b/terraform/modules/lincset_bucket/main.tf index e9e4370..afbb3cc 100644 --- a/terraform/modules/lincset_bucket/main.tf +++ b/terraform/modules/lincset_bucket/main.tf @@ -71,14 +71,14 @@ resource "aws_s3_bucket_ownership_controls" "lincset_bucket" { } } -# resource "aws_s3_bucket_acl" "lincset_bucket" { -# depends_on = [aws_s3_bucket_ownership_controls.lincset_bucket] -# -# bucket = aws_s3_bucket.lincset_bucket.id -# -# // Public access is granted via a bucket policy, not a canned ACL -# acl = "private" -# } +resource "aws_s3_bucket_acl" "lincset_bucket" { + depends_on = [aws_s3_bucket_ownership_controls.lincset_bucket] + + bucket = aws_s3_bucket.lincset_bucket.id + + // Public access is granted via a bucket policy, not a canned ACL + acl = "private" +} resource "aws_iam_user_policy" "lincset_bucket_owner" { // The Heroku IAM user will always be in the project account diff --git a/terraform/sponsored_bucket.tf b/terraform/sponsored_bucket.tf index 4b14e1f..40cbd67 100644 --- a/terraform/sponsored_bucket.tf +++ b/terraform/sponsored_bucket.tf @@ -1,6 +1,6 @@ module "sponsored_lincset_bucket" { source = "./modules/lincset_bucket" - bucket_name = "linc-brain-mit" + bucket_name = "linc-brain-mit-prod" public = true versioning = true trailing_delete = false @@ -15,7 +15,7 @@ module "sponsored_lincset_bucket" { module "sponsored_embargo_bucket" { source = "./modules/lincset_bucket" - bucket_name = "linc-brain-mit-embargo" + bucket_name = "linc-brain-mit-embargo-prod" versioning = false trailing_delete = false heroku_user = data.aws_iam_user.api diff --git a/terraform/staging_bucket.tf b/terraform/staging_bucket.tf index 87087b7..5fd6a47 100644 --- a/terraform/staging_bucket.tf +++ b/terraform/staging_bucket.tf @@ -1,12 +1,12 @@ module "staging_lincset_bucket" { source = "./modules/lincset_bucket" - bucket_name = "linc-api-staging-lincsets" + bucket_name = "linc-brain-mit-staging" public = true versioning = true trailing_delete = true allow_heroku_put_object = true heroku_user = data.aws_iam_user.api_staging - log_bucket_name = "linc-api-staging-lincset-logs" + log_bucket_name = "linc-brain-mit-staging-logs" providers = { aws = aws aws.project = aws @@ -15,11 +15,11 @@ module "staging_lincset_bucket" { module "staging_embargo_bucket" { source = "./modules/lincset_bucket" - bucket_name = "linc-api-staging-embargo-lincsets" + bucket_name = "linc-brain-mit-embargo-staging" versioning = false trailing_delete = false heroku_user = data.aws_iam_user.api_staging - log_bucket_name = "linc-api-staging-embargo-lincset-logs" + log_bucket_name = "linc-brain-mit-staging-embargo-logs" providers = { aws = aws aws.project = aws From 5afc4a7dfb138e6a7bc1c723a9bdb6f7945a1ef9 Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Fri, 9 Feb 2024 15:02:53 -0500 Subject: [PATCH 16/35] temp delete --- terraform/modules/lincset_bucket/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/modules/lincset_bucket/main.tf b/terraform/modules/lincset_bucket/main.tf index afbb3cc..61d2360 100644 --- a/terraform/modules/lincset_bucket/main.tf +++ b/terraform/modules/lincset_bucket/main.tf @@ -8,7 +8,7 @@ resource "aws_s3_bucket" "lincset_bucket" { bucket = var.bucket_name lifecycle { - prevent_destroy = true + prevent_destroy = false } } From 3a74ef2a08f6b46039da2e5474dc27489504ca7f Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Fri, 9 Feb 2024 15:05:01 -0500 Subject: [PATCH 17/35] other flag --- terraform/modules/lincset_bucket/log_bucket.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/modules/lincset_bucket/log_bucket.tf b/terraform/modules/lincset_bucket/log_bucket.tf index c6adcee..ca18477 100644 --- a/terraform/modules/lincset_bucket/log_bucket.tf +++ b/terraform/modules/lincset_bucket/log_bucket.tf @@ -4,7 +4,7 @@ resource "aws_s3_bucket" "log_bucket" { bucket = var.log_bucket_name lifecycle { - prevent_destroy = true + prevent_destroy = false } } From 29be98ee80697f00b41ed5ceffa016a9242f2dd5 Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Fri, 9 Feb 2024 15:18:26 -0500 Subject: [PATCH 18/35] revert flags --- terraform/modules/lincset_bucket/log_bucket.tf | 2 +- terraform/modules/lincset_bucket/main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/modules/lincset_bucket/log_bucket.tf b/terraform/modules/lincset_bucket/log_bucket.tf index ca18477..c6adcee 100644 --- a/terraform/modules/lincset_bucket/log_bucket.tf +++ b/terraform/modules/lincset_bucket/log_bucket.tf @@ -4,7 +4,7 @@ resource "aws_s3_bucket" "log_bucket" { bucket = var.log_bucket_name lifecycle { - prevent_destroy = false + prevent_destroy = true } } diff --git a/terraform/modules/lincset_bucket/main.tf b/terraform/modules/lincset_bucket/main.tf index 61d2360..afbb3cc 100644 --- a/terraform/modules/lincset_bucket/main.tf +++ b/terraform/modules/lincset_bucket/main.tf @@ -8,7 +8,7 @@ resource "aws_s3_bucket" "lincset_bucket" { bucket = var.bucket_name lifecycle { - prevent_destroy = false + prevent_destroy = true } } From 96c689ab0fa2d107691107a6b86307ed47ac24f8 Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Mon, 12 Feb 2024 14:15:10 -0500 Subject: [PATCH 19/35] remove trailing slash --- terraform/staging_pipeline.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/staging_pipeline.tf b/terraform/staging_pipeline.tf index c784e46..bcdcb58 100644 --- a/terraform/staging_pipeline.tf +++ b/terraform/staging_pipeline.tf @@ -39,7 +39,7 @@ module "api_staging" { DJANGO_SENTRY_ENVIRONMENT = "staging" DJANGO_CELERY_WORKER_CONCURRENCY = "2" DJANGO_DANDI_WEB_APP_URL = "https://staging--gui-staging-lincbrain-org.netlify.app/" - DJANGO_DANDI_API_URL = "https://staging-api.lincbrain.org/" + DJANGO_DANDI_API_URL = "https://staging-api.lincbrain.org" DJANGO_DANDI_JUPYTERHUB_URL = "https://hub.lincbrain.org/" } additional_sensitive_django_vars = { From 9897fa69fc4622fc08c01942a7daa5d77ef0ee58 Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Mon, 12 Feb 2024 14:39:58 -0500 Subject: [PATCH 20/35] correct env vars --- terraform/staging_pipeline.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/staging_pipeline.tf b/terraform/staging_pipeline.tf index bcdcb58..42473af 100644 --- a/terraform/staging_pipeline.tf +++ b/terraform/staging_pipeline.tf @@ -20,7 +20,7 @@ module "api_staging" { heroku_worker_dyno_quantity = 1 django_default_from_email = "admin@staging-api.lincbrain.org" - django_cors_origin_whitelist = ["https://gui-staging.lincbrain.org", "https://staging--gui-staging-lincbrain-org.netlify.app"] + django_cors_origin_whitelist = ["https://gui-staging.lincbrain.org", "https://staging--lincbrain-org.netlify.app"] django_cors_origin_regex_whitelist = ["https://staging--gui-staging-lincbrain-org.netlify.app"] additional_django_vars = { @@ -38,7 +38,7 @@ module "api_staging" { DJANGO_SENTRY_DSN = "https://833c159dc622528b21b4ce4adef6dbf8@o4506237212033024.ingest.sentry.io/4506237213212672" DJANGO_SENTRY_ENVIRONMENT = "staging" DJANGO_CELERY_WORKER_CONCURRENCY = "2" - DJANGO_DANDI_WEB_APP_URL = "https://staging--gui-staging-lincbrain-org.netlify.app/" + DJANGO_DANDI_WEB_APP_URL = "https://staging--lincbrain-org.netlify.app/" DJANGO_DANDI_API_URL = "https://staging-api.lincbrain.org" DJANGO_DANDI_JUPYTERHUB_URL = "https://hub.lincbrain.org/" } From 5471e75b37485f7e93075b1a656bd7d8325f872b Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Mon, 12 Feb 2024 20:34:29 -0500 Subject: [PATCH 21/35] remove trailing slash from staging API --- terraform/staging_pipeline.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/staging_pipeline.tf b/terraform/staging_pipeline.tf index 42473af..784fb69 100644 --- a/terraform/staging_pipeline.tf +++ b/terraform/staging_pipeline.tf @@ -38,7 +38,7 @@ module "api_staging" { DJANGO_SENTRY_DSN = "https://833c159dc622528b21b4ce4adef6dbf8@o4506237212033024.ingest.sentry.io/4506237213212672" DJANGO_SENTRY_ENVIRONMENT = "staging" DJANGO_CELERY_WORKER_CONCURRENCY = "2" - DJANGO_DANDI_WEB_APP_URL = "https://staging--lincbrain-org.netlify.app/" + DJANGO_DANDI_WEB_APP_URL = "https://staging--lincbrain-org.netlify.app" DJANGO_DANDI_API_URL = "https://staging-api.lincbrain.org" DJANGO_DANDI_JUPYTERHUB_URL = "https://hub.lincbrain.org/" } From 07838702faf35126cbcfeaf30db17b54fa234d52 Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Wed, 14 Feb 2024 10:38:15 -0500 Subject: [PATCH 22/35] Bump Heroku dyno compute values to match DANDI --- terraform/api.tf | 8 ++++---- terraform/sponsored_bucket.tf | 2 +- terraform/staging_bucket.tf | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/terraform/api.tf b/terraform/api.tf index 3c8cf06..04352a7 100644 --- a/terraform/api.tf +++ b/terraform/api.tf @@ -11,8 +11,8 @@ module "api" { route53_zone_id = aws_route53_zone.linc-brain-mit.zone_id subdomain_name = "api" - heroku_web_dyno_size = "standard-1x" - heroku_worker_dyno_size = "standard-1x" + heroku_web_dyno_size = "standard-2x" + heroku_worker_dyno_size = "standard-2x" heroku_postgresql_plan = "standard-0" heroku_cloudamqp_plan = "squirrel-1" heroku_papertrail_plan = "liatorp" @@ -20,7 +20,7 @@ module "api" { heroku_web_dyno_quantity = 1 heroku_worker_dyno_quantity = 1 - django_default_from_email = "admin@lincbrain.org" + django_default_from_email = "admin@api.lincbrain.org" django_cors_origin_whitelist = ["https://lincbrain.org"] django_cors_origin_regex_whitelist = ["^https:\\/\\/[0-9a-z\\-]+\\.netlify\\.app$"] @@ -51,7 +51,7 @@ module "api" { resource "heroku_formation" "api_checksum_worker" { app_id = module.api.heroku_app_id type = "checksum-worker" - size = "standard-1x" + size = "standard-2x" quantity = 1 } diff --git a/terraform/sponsored_bucket.tf b/terraform/sponsored_bucket.tf index 40cbd67..765107a 100644 --- a/terraform/sponsored_bucket.tf +++ b/terraform/sponsored_bucket.tf @@ -1,7 +1,7 @@ module "sponsored_lincset_bucket" { source = "./modules/lincset_bucket" bucket_name = "linc-brain-mit-prod" - public = true + public = false versioning = true trailing_delete = false allow_cross_account_heroku_put_object = true diff --git a/terraform/staging_bucket.tf b/terraform/staging_bucket.tf index 5fd6a47..0a73926 100644 --- a/terraform/staging_bucket.tf +++ b/terraform/staging_bucket.tf @@ -1,7 +1,7 @@ module "staging_lincset_bucket" { source = "./modules/lincset_bucket" bucket_name = "linc-brain-mit-staging" - public = true + public = false versioning = true trailing_delete = true allow_heroku_put_object = true From 6cdd67a8a7dd0e7c4e96fb0f18f9077875976623 Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Tue, 12 Mar 2024 14:59:15 -0400 Subject: [PATCH 23/35] Change region for buckets from us-east-1 to us-east-2 --- terraform/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/main.tf b/terraform/main.tf index 86d47c7..e6e1883 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -9,7 +9,7 @@ terraform { } provider "aws" { - region = "us-east-1" + region = "us-east-2" allowed_account_ids = ["151312473579"] assume_role { @@ -20,7 +20,7 @@ provider "aws" { // The "sponsored" account, the Amazon-sponsored account with the public bucket provider "aws" { alias = "sponsored" - region = "us-east-1" + region = "us-east-2" allowed_account_ids = ["151312473579"] # TODO: Aaron make new ID // This will authenticate using credentials from the project account, then assume the From f9453b1d71ec784462755d9052636fc8bb246a5f Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Tue, 12 Mar 2024 15:05:44 -0400 Subject: [PATCH 24/35] use target --- terraform/api.tf | 8 ++++---- terraform/main.tf | 14 ++++++++++++-- terraform/sponsored_bucket.tf | 28 ++++++++++++++++++++++++++++ terraform/staging_bucket.tf | 28 ++++++++++++++++++++++++++++ terraform/staging_pipeline.tf | 8 ++++---- 5 files changed, 76 insertions(+), 10 deletions(-) diff --git a/terraform/api.tf b/terraform/api.tf index 04352a7..432754a 100644 --- a/terraform/api.tf +++ b/terraform/api.tf @@ -26,12 +26,12 @@ module "api" { additional_django_vars = { DJANGO_CONFIGURATION = "HerokuProductionConfiguration" - DJANGO_DANDI_DANDISETS_BUCKET_NAME = module.sponsored_lincset_bucket.bucket_name + DJANGO_DANDI_DANDISETS_BUCKET_NAME = module.sponsored_lincset_bucket-us-east-2.bucket_name DJANGO_DANDI_DANDISETS_BUCKET_PREFIX = "" - DJANGO_DANDI_DANDISETS_EMBARGO_BUCKET_NAME = module.sponsored_embargo_bucket.bucket_name + DJANGO_DANDI_DANDISETS_EMBARGO_BUCKET_NAME = module.sponsored_embargo_bucket-us-east-2.bucket_name DJANGO_DANDI_DANDISETS_EMBARGO_BUCKET_PREFIX = "" - DJANGO_DANDI_DANDISETS_LOG_BUCKET_NAME = module.sponsored_lincset_bucket.log_bucket_name - DJANGO_DANDI_DANDISETS_EMBARGO_LOG_BUCKET_NAME = module.sponsored_embargo_bucket.log_bucket_name + DJANGO_DANDI_DANDISETS_LOG_BUCKET_NAME = module.sponsored_lincset_bucket-us-east-2.log_bucket_name + DJANGO_DANDI_DANDISETS_EMBARGO_LOG_BUCKET_NAME = module.sponsored_embargo_bucket-us-east-2.log_bucket_name DJANGO_DANDI_DOI_API_URL = "https://api.datacite.org/dois" DJANGO_DANDI_DOI_API_USER = "temp.dandi" DJANGO_DANDI_DOI_API_PREFIX = "temp" diff --git a/terraform/main.tf b/terraform/main.tf index e6e1883..4a3cc71 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -9,7 +9,7 @@ terraform { } provider "aws" { - region = "us-east-2" + region = "us-east-1" allowed_account_ids = ["151312473579"] assume_role { @@ -17,10 +17,20 @@ provider "aws" { } } +provider "aws" { + region = "us-east-2" + alias = "target" + allowed_account_ids = ["151312473579"] + + assume_role { + role_arn = "arn:aws:iam::151312473579:role/linc-infrastructure" + } +} + // The "sponsored" account, the Amazon-sponsored account with the public bucket provider "aws" { alias = "sponsored" - region = "us-east-2" + region = "us-east-1" allowed_account_ids = ["151312473579"] # TODO: Aaron make new ID // This will authenticate using credentials from the project account, then assume the diff --git a/terraform/sponsored_bucket.tf b/terraform/sponsored_bucket.tf index 765107a..9734807 100644 --- a/terraform/sponsored_bucket.tf +++ b/terraform/sponsored_bucket.tf @@ -25,3 +25,31 @@ module "sponsored_embargo_bucket" { aws.project = aws } } + +module "sponsored_lincset_bucket-us-east-2" { + source = "./modules/lincset_bucket" + bucket_name = "linc-brain-mit-prod-us-east-2" + public = false + versioning = true + trailing_delete = false + allow_cross_account_heroku_put_object = true + heroku_user = data.aws_iam_user.api + log_bucket_name = "linc-brain-mit-logs-us-east-2" + providers = { + aws = aws.target + aws.project = aws + } +} + +module "sponsored_embargo_bucket-us-east-2" { + source = "./modules/lincset_bucket" + bucket_name = "linc-brain-mit-embargo-prod-us-east-2" + versioning = false + trailing_delete = false + heroku_user = data.aws_iam_user.api + log_bucket_name = "linc-brain-mit-embargo-logs-us-east-2" + providers = { + aws = aws.target + aws.project = aws + } +} diff --git a/terraform/staging_bucket.tf b/terraform/staging_bucket.tf index 0a73926..892aea5 100644 --- a/terraform/staging_bucket.tf +++ b/terraform/staging_bucket.tf @@ -24,4 +24,32 @@ module "staging_embargo_bucket" { aws = aws aws.project = aws } +} + +module "staging_lincset_bucket-us-east-2" { + source = "./modules/lincset_bucket" + bucket_name = "linc-brain-mit-staging-us-east-2" + public = false + versioning = true + trailing_delete = true + allow_heroku_put_object = true + heroku_user = data.aws_iam_user.api_staging + log_bucket_name = "linc-brain-mit-staging-logs-us-east-2" + providers = { + aws = aws.target + aws.project = aws + } +} + +module "staging_embargo_bucket-us-east-2" { + source = "./modules/lincset_bucket" + bucket_name = "linc-brain-mit-embargo-staging-us-east-2" + versioning = false + trailing_delete = false + heroku_user = data.aws_iam_user.api_staging + log_bucket_name = "linc-brain-mit-staging-embargo-logs-us-east-2" + providers = { + aws = aws.target + aws.project = aws + } } \ No newline at end of file diff --git a/terraform/staging_pipeline.tf b/terraform/staging_pipeline.tf index 784fb69..3e6b19d 100644 --- a/terraform/staging_pipeline.tf +++ b/terraform/staging_pipeline.tf @@ -25,12 +25,12 @@ module "api_staging" { additional_django_vars = { DJANGO_CONFIGURATION = "HerokuStagingConfiguration" - DJANGO_DANDI_DANDISETS_BUCKET_NAME = module.staging_lincset_bucket.bucket_name + DJANGO_DANDI_DANDISETS_BUCKET_NAME = module.staging_lincset_bucket-us-east-2.bucket_name DJANGO_DANDI_DANDISETS_BUCKET_PREFIX = "" - DJANGO_DANDI_DANDISETS_EMBARGO_BUCKET_NAME = module.staging_embargo_bucket.bucket_name + DJANGO_DANDI_DANDISETS_EMBARGO_BUCKET_NAME = module.staging_embargo_bucket-us-east-2.bucket_name DJANGO_DANDI_DANDISETS_EMBARGO_BUCKET_PREFIX = "" - DJANGO_DANDI_DANDISETS_LOG_BUCKET_NAME = module.staging_lincset_bucket.log_bucket_name - DJANGO_DANDI_DANDISETS_EMBARGO_LOG_BUCKET_NAME = module.staging_embargo_bucket.log_bucket_name + DJANGO_DANDI_DANDISETS_LOG_BUCKET_NAME = module.staging_lincset_bucket-us-east-2.log_bucket_name + DJANGO_DANDI_DANDISETS_EMBARGO_LOG_BUCKET_NAME = module.staging_embargo_bucket-us-east-2.log_bucket_name DJANGO_DANDI_DOI_API_URL = "https://api.test.datacite.org/dois" DJANGO_DANDI_DOI_API_USER = "dartlib.dandi" DJANGO_DANDI_DOI_API_PREFIX = "10.80507" From 48d7ece16b1aa71a0d0ebf5b2b4397c18ed633e7 Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Tue, 12 Mar 2024 16:09:07 -0400 Subject: [PATCH 25/35] update linc bucket configs to us-east-2 alternative --- terraform/modules/lincset_bucket/main.tf | 294 +++++++++++++++++++++++ terraform/sponsored_iam.tf | 48 ++++ 2 files changed, 342 insertions(+) diff --git a/terraform/modules/lincset_bucket/main.tf b/terraform/modules/lincset_bucket/main.tf index afbb3cc..9c9ed61 100644 --- a/terraform/modules/lincset_bucket/main.tf +++ b/terraform/modules/lincset_bucket/main.tf @@ -290,3 +290,297 @@ resource "aws_s3_bucket_lifecycle_configuration" "expire_deleted_objects" { status = "Enabled" } } + + +data "aws_caller_identity" "sponsored_account_us_east_2" { + provider = aws +} + +# data "aws_caller_identity" "current" {} + +resource "aws_s3_bucket" "lincset_bucket_us_east_2" { + bucket = var.bucket_name + + lifecycle { + prevent_destroy = true + } + +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "lincset_bucket_us_east_2" { + bucket = aws_s3_bucket.lincset_bucket.id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} + +resource "aws_s3_bucket_cors_configuration" "lincset_bucket_us_east_2" { + bucket = aws_s3_bucket.lincset_bucket.id + + cors_rule { + allowed_origins = [ + "*", + ] + allowed_methods = [ + "PUT", + "POST", + "GET", + "DELETE", + ] + allowed_headers = [ + "*" + ] + expose_headers = [ + "ETag", + ] + max_age_seconds = 3000 + } +} + +resource "aws_s3_bucket_logging" "lincset_bucket_us_east_2" { + bucket = aws_s3_bucket.lincset_bucket.id + + target_bucket = aws_s3_bucket.log_bucket.id + target_prefix = "" +} + +resource "aws_s3_bucket_versioning" "lincset_bucket_us_east_2" { + count = var.versioning ? 1 : 0 + + bucket = aws_s3_bucket.lincset_bucket.id + + versioning_configuration { + status = "Enabled" + } +} + +resource "aws_s3_bucket_ownership_controls" "lincset_bucket_us_east_2" { + bucket = aws_s3_bucket.lincset_bucket_us_east_2.id + + rule { + object_ownership = "BucketOwnerPreferred" + } +} + +resource "aws_s3_bucket_acl" "lincset_bucket" { + depends_on = [aws_s3_bucket_ownership_controls.lincset_bucket] + + bucket = aws_s3_bucket.lincset_bucket_us_east_2.id + + // Public access is granted via a bucket policy, not a canned ACL + acl = "private" +} + +resource "aws_iam_user_policy" "lincset_bucket_owner_us_east_2" { + // The Heroku IAM user will always be in the project account + provider = aws.project + + name = "${var.bucket_name}-ownership-policy" + user = var.heroku_user.user_name + + policy = data.aws_iam_policy_document.lincset_bucket_owner_us_east_2.json +} + +data "aws_iam_policy_document" "lincset_bucket_owner_us_east_2" { + version = "2008-10-17" + + statement { + resources = [ + "${aws_s3_bucket.lincset_bucket_us_east_2.arn}", + "${aws_s3_bucket.lincset_bucket_us_east_2.arn}/*", + ] + + actions = [ + "s3:Get*", + "s3:List*", + "s3:Delete*", + ] + } + + dynamic "statement" { + for_each = var.allow_heroku_put_object ? [1] : [] + content { + + resources = [ + "${aws_s3_bucket.lincset_bucket_us_east_2.arn}", + "${aws_s3_bucket.lincset_bucket_us_east_2.arn}/*", + ] + + actions = ["s3:PutObject"] + } + } + + statement { + resources = [ + "${aws_s3_bucket.lincset_bucket_us_east_2.arn}", + "${aws_s3_bucket.lincset_bucket_us_east_2.arn}/*", + ] + + actions = ["s3:*"] + + condition { + test = "StringEquals" + variable = "s3:x-amz-acl" + values = ["bucket-owner-full-control"] + } + } +} + +resource "aws_s3_bucket_policy" "lincset_bucket_policy_us_east_2" { + provider = aws + + bucket = aws_s3_bucket.lincset_bucket_us_east_2.id + policy = data.aws_iam_policy_document.lincset_bucket_policy_us_east_2.json +} + +data "aws_iam_policy_document" "lincset_bucket_policy_us_east_2" { + version = "2008-10-17" + + dynamic "statement" { + for_each = var.public ? [1] : [] + + content { + resources = [ + "${aws_s3_bucket.lincset_bucket_us_east_2.arn}", + "${aws_s3_bucket.lincset_bucket_us_east_2.arn}/*", + ] + + actions = [ + "s3:Get*", + "s3:List*", + ] + + principals { + identifiers = ["*"] + type = "*" + } + } + } + + dynamic "statement" { + for_each = var.allow_cross_account_heroku_put_object ? [1] : [] + + content { + sid = "S3PolicyStmt-DO-NOT-MODIFY-1569973164923" + principals { + identifiers = ["s3.amazonaws.com"] + type = "Service" + } + actions = [ + "s3:PutObject", + ] + resources = [ + "${aws_s3_bucket.lincset_bucket_us_east_2.arn}/*", + ] + condition { + test = "StringEquals" + variable = "aws:SourceAccount" + values = [data.aws_caller_identity.sponsored_account.account_id] + } + condition { + test = "StringEquals" + variable = "s3:x-amz-acl" + values = ["bucket-owner-full-control"] + } + condition { + test = "ArnLike" + variable = "aws:SourceArn" + values = [aws_s3_bucket.lincset_bucket_us_east_2.arn] + } + } + } + + statement { + resources = [ + "${aws_s3_bucket.lincset_bucket_us_east_2.arn}", + "${aws_s3_bucket.lincset_bucket_us_east_2.arn}/*", + ] + + actions = [ + "s3:Get*", + "s3:List*", + "s3:Delete*", + ] + + principals { + type = "AWS" + identifiers = [var.heroku_user.arn] + } + } + + statement { + resources = [ + "${aws_s3_bucket.lincset_bucket_us_east_2.arn}", + "${aws_s3_bucket.lincset_bucket_us_east_2.arn}/*", + ] + + actions = ["s3:*"] + + condition { + test = "StringEquals" + variable = "s3:x-amz-acl" + values = ["bucket-owner-full-control"] + } + + principals { + type = "AWS" + identifiers = [var.heroku_user.arn] + } + } + + dynamic "statement" { + for_each = var.trailing_delete ? [1] : [] + + content { + sid = "PreventDeletionOfObjectVersions" + + resources = [ + "${aws_s3_bucket.lincset_bucket_us_east_2.arn}/*" + ] + + actions = [ + "s3:DeleteObjectVersion", + ] + + effect = "Deny" + + principals { + identifiers = ["*"] + type = "*" + } + } + } +} + + +# S3 lifecycle policy that permanently deletes objects with delete markers +# after 30 days. +resource "aws_s3_bucket_lifecycle_configuration" "expire_deleted_objects_us_east_2" { + # Must have bucket versioning enabled first + depends_on = [aws_s3_bucket_versioning.lincset_bucket_us_east_2] + + count = var.trailing_delete ? 1 : 0 + + bucket = aws_s3_bucket.lincset_bucket_us_east_2.id + + # Based on https://docs.aws.amazon.com/AmazonS3/latest/userguide/lifecycle-configuration-examples.html#lifecycle-config-conceptual-ex7 + rule { + id = "ExpireOldDeleteMarkers" + filter {} + + # Expire objects with delete markers after 30 days + noncurrent_version_expiration { + noncurrent_days = 30 + } + + # Also delete any delete markers associated with the expired object + expiration { + expired_object_delete_marker = true + } + + status = "Enabled" + } +} diff --git a/terraform/sponsored_iam.tf b/terraform/sponsored_iam.tf index b0d9908..7cfa3f1 100644 --- a/terraform/sponsored_iam.tf +++ b/terraform/sponsored_iam.tf @@ -42,3 +42,51 @@ data "aws_iam_policy_document" "sponsored_writers" { ] } } + +resource "aws_iam_group" "sponsored_writers_us_east_2" { + provider = aws.target + + name = "writers" +} + +resource "aws_iam_group_policy" "sponsored_writers_us_east_2" { + provider = aws.target + + name = "bucket-write" + group = aws_iam_group.sponsored_writers_us_east_2.name + policy = data.aws_iam_policy_document.sponsored_writers_us_east_2.json +} + +data "aws_iam_policy_document" "sponsored_writers_us_east_2" { + version = "2012-10-17" + statement { + sid = "VisualEditor0" + actions = [ + "s3:DeleteObjectTagging", + "s3:ListBucketByTags", + "s3:ListBucketMultipartUploads", + "s3:GetBucketTagging", + "s3:ListBucketVersions", + "s3:PutObjectVersionTagging", + "s3:ListBucket", + "s3:DeleteObjectVersionTagging", + "s3:GetBucketVersioning", + "s3:GetObjectVersionTorrent", + "s3:PutObject", + "s3:GetObject", + "s3:PutBucketTagging", + "s3:GetObjectTagging", + "s3:PutObjectTagging", + "s3:DeleteObject", + "s3:GetBucketLocation", + "s3:GetObjectVersion", + ] + resources = [ + "${module.sponsored_lincset_bucket_us_east_2.bucket_arn}/*", + module.sponsored_lincset_bucket_us_east_2.bucket_arn, + ] + } +} + + + From 2bda5126b3d75f89d0e5f3b9a0439f214accb00d Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Tue, 12 Mar 2024 16:12:13 -0400 Subject: [PATCH 26/35] separate acl --- terraform/modules/lincset_bucket/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/modules/lincset_bucket/main.tf b/terraform/modules/lincset_bucket/main.tf index 9c9ed61..adc0b68 100644 --- a/terraform/modules/lincset_bucket/main.tf +++ b/terraform/modules/lincset_bucket/main.tf @@ -365,8 +365,8 @@ resource "aws_s3_bucket_ownership_controls" "lincset_bucket_us_east_2" { } } -resource "aws_s3_bucket_acl" "lincset_bucket" { - depends_on = [aws_s3_bucket_ownership_controls.lincset_bucket] +resource "aws_s3_bucket_acl" "lincset_bucket_us_east_2" { + depends_on = [aws_s3_bucket_ownership_controls.lincset_bucket_us_east_2] bucket = aws_s3_bucket.lincset_bucket_us_east_2.id From b8eae5aa2a2113db4f4363482dc3d5881a5797ed Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Tue, 12 Mar 2024 16:16:08 -0400 Subject: [PATCH 27/35] more separation --- terraform/modules/lincset_bucket/outputs.tf | 15 +++++++++++++++ terraform/sponsored_bucket.tf | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/terraform/modules/lincset_bucket/outputs.tf b/terraform/modules/lincset_bucket/outputs.tf index 65e5c02..e7f9cf3 100644 --- a/terraform/modules/lincset_bucket/outputs.tf +++ b/terraform/modules/lincset_bucket/outputs.tf @@ -12,3 +12,18 @@ output "bucket_arn" { value = aws_s3_bucket.lincset_bucket.arn description = "The S3 bucket ARN." } + +output "bucket_name_us_east_2" { + value = aws_s3_bucket.lincset_bucket_us_east_2.id + description = "The S3 bucket name." +} + +output "log_bucket_name_us_east_2" { + value = aws_s3_bucket.log_bucket_us_east_2.id + description = "The S3 log bucket name." +} + +output "bucket_arn_us_east_2" { + value = aws_s3_bucket.lincset_bucket_us_east_2.arn + description = "The S3 bucket ARN." +} diff --git a/terraform/sponsored_bucket.tf b/terraform/sponsored_bucket.tf index 9734807..bb49f06 100644 --- a/terraform/sponsored_bucket.tf +++ b/terraform/sponsored_bucket.tf @@ -26,7 +26,7 @@ module "sponsored_embargo_bucket" { } } -module "sponsored_lincset_bucket-us-east-2" { +module "sponsored_lincset_bucket_us_east_2" { source = "./modules/lincset_bucket" bucket_name = "linc-brain-mit-prod-us-east-2" public = false From ae2cfedaf36bbd82f4b4bc0988b508dbb8e16809 Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Tue, 12 Mar 2024 16:31:16 -0400 Subject: [PATCH 28/35] trivial change to invoke terraform plan --- terraform/modules/lincset_bucket/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/modules/lincset_bucket/main.tf b/terraform/modules/lincset_bucket/main.tf index adc0b68..68a3bd0 100644 --- a/terraform/modules/lincset_bucket/main.tf +++ b/terraform/modules/lincset_bucket/main.tf @@ -584,3 +584,4 @@ resource "aws_s3_bucket_lifecycle_configuration" "expire_deleted_objects_us_east status = "Enabled" } } + From 607cde8f37fac1d6cea8dee847df8d3b94813499 Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Tue, 12 Mar 2024 16:34:05 -0400 Subject: [PATCH 29/35] more fixes --- terraform/api.tf | 8 ++++---- terraform/staging_bucket.tf | 2 +- terraform/staging_pipeline.tf | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/terraform/api.tf b/terraform/api.tf index 432754a..01fa0ee 100644 --- a/terraform/api.tf +++ b/terraform/api.tf @@ -26,12 +26,12 @@ module "api" { additional_django_vars = { DJANGO_CONFIGURATION = "HerokuProductionConfiguration" - DJANGO_DANDI_DANDISETS_BUCKET_NAME = module.sponsored_lincset_bucket-us-east-2.bucket_name + DJANGO_DANDI_DANDISETS_BUCKET_NAME = module.sponsored_lincset_bucket_us_east_2.bucket_name DJANGO_DANDI_DANDISETS_BUCKET_PREFIX = "" - DJANGO_DANDI_DANDISETS_EMBARGO_BUCKET_NAME = module.sponsored_embargo_bucket-us-east-2.bucket_name + DJANGO_DANDI_DANDISETS_EMBARGO_BUCKET_NAME = module.sponsored_embargo_bucket_us_east_2.bucket_name DJANGO_DANDI_DANDISETS_EMBARGO_BUCKET_PREFIX = "" - DJANGO_DANDI_DANDISETS_LOG_BUCKET_NAME = module.sponsored_lincset_bucket-us-east-2.log_bucket_name - DJANGO_DANDI_DANDISETS_EMBARGO_LOG_BUCKET_NAME = module.sponsored_embargo_bucket-us-east-2.log_bucket_name + DJANGO_DANDI_DANDISETS_LOG_BUCKET_NAME = module.sponsored_lincset_bucket_us_east_2.log_bucket_name + DJANGO_DANDI_DANDISETS_EMBARGO_LOG_BUCKET_NAME = module.sponsored_embargo_bucket_us_east_2.log_bucket_name DJANGO_DANDI_DOI_API_URL = "https://api.datacite.org/dois" DJANGO_DANDI_DOI_API_USER = "temp.dandi" DJANGO_DANDI_DOI_API_PREFIX = "temp" diff --git a/terraform/staging_bucket.tf b/terraform/staging_bucket.tf index 892aea5..bb7fb72 100644 --- a/terraform/staging_bucket.tf +++ b/terraform/staging_bucket.tf @@ -26,7 +26,7 @@ module "staging_embargo_bucket" { } } -module "staging_lincset_bucket-us-east-2" { +module "staging_lincset_bucket_us_east_2" { source = "./modules/lincset_bucket" bucket_name = "linc-brain-mit-staging-us-east-2" public = false diff --git a/terraform/staging_pipeline.tf b/terraform/staging_pipeline.tf index 3e6b19d..87d05f0 100644 --- a/terraform/staging_pipeline.tf +++ b/terraform/staging_pipeline.tf @@ -25,12 +25,12 @@ module "api_staging" { additional_django_vars = { DJANGO_CONFIGURATION = "HerokuStagingConfiguration" - DJANGO_DANDI_DANDISETS_BUCKET_NAME = module.staging_lincset_bucket-us-east-2.bucket_name + DJANGO_DANDI_DANDISETS_BUCKET_NAME = module.staging_lincset_bucket_us_east_2.bucket_name DJANGO_DANDI_DANDISETS_BUCKET_PREFIX = "" - DJANGO_DANDI_DANDISETS_EMBARGO_BUCKET_NAME = module.staging_embargo_bucket-us-east-2.bucket_name + DJANGO_DANDI_DANDISETS_EMBARGO_BUCKET_NAME = module.staging_embargo_bucket_us_east_2.bucket_name DJANGO_DANDI_DANDISETS_EMBARGO_BUCKET_PREFIX = "" - DJANGO_DANDI_DANDISETS_LOG_BUCKET_NAME = module.staging_lincset_bucket-us-east-2.log_bucket_name - DJANGO_DANDI_DANDISETS_EMBARGO_LOG_BUCKET_NAME = module.staging_embargo_bucket-us-east-2.log_bucket_name + DJANGO_DANDI_DANDISETS_LOG_BUCKET_NAME = module.staging_lincset_bucket_us_east_2.log_bucket_name + DJANGO_DANDI_DANDISETS_EMBARGO_LOG_BUCKET_NAME = module.staging_embargo_bucket_us_east_2.log_bucket_name DJANGO_DANDI_DOI_API_URL = "https://api.test.datacite.org/dois" DJANGO_DANDI_DOI_API_USER = "dartlib.dandi" DJANGO_DANDI_DOI_API_PREFIX = "10.80507" From b75fcad23c5a0d9ff9f1783eb46f6d792a670ebb Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Tue, 12 Mar 2024 16:35:55 -0400 Subject: [PATCH 30/35] more cleanup --- terraform/staging_bucket.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/staging_bucket.tf b/terraform/staging_bucket.tf index bb7fb72..3efb64c 100644 --- a/terraform/staging_bucket.tf +++ b/terraform/staging_bucket.tf @@ -41,7 +41,7 @@ module "staging_lincset_bucket_us_east_2" { } } -module "staging_embargo_bucket-us-east-2" { +module "staging_embargo_bucket_us_east_2" { source = "./modules/lincset_bucket" bucket_name = "linc-brain-mit-embargo-staging-us-east-2" versioning = false From 0537e21c56144fead7dd6bb8a4b034ebf9319cdd Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Tue, 12 Mar 2024 16:37:14 -0400 Subject: [PATCH 31/35] more progress --- terraform/sponsored_bucket.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/sponsored_bucket.tf b/terraform/sponsored_bucket.tf index bb49f06..3a5ff00 100644 --- a/terraform/sponsored_bucket.tf +++ b/terraform/sponsored_bucket.tf @@ -41,7 +41,7 @@ module "sponsored_lincset_bucket_us_east_2" { } } -module "sponsored_embargo_bucket-us-east-2" { +module "sponsored_embargo_bucket_us_east_2" { source = "./modules/lincset_bucket" bucket_name = "linc-brain-mit-embargo-prod-us-east-2" versioning = false From 65af591c4c59ca5e8e965d8885459b13b69ee340 Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Tue, 12 Mar 2024 16:40:52 -0400 Subject: [PATCH 32/35] Resolve logging error --- .../modules/lincset_bucket/log_bucket.tf | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/terraform/modules/lincset_bucket/log_bucket.tf b/terraform/modules/lincset_bucket/log_bucket.tf index c6adcee..f0bc916 100644 --- a/terraform/modules/lincset_bucket/log_bucket.tf +++ b/terraform/modules/lincset_bucket/log_bucket.tf @@ -96,3 +96,103 @@ resource "aws_iam_user_policy" "lincset_log_bucket_owner" { policy = data.aws_iam_policy_document.lincset_log_bucket_owner.json } + +# data "aws_canonical_user_id" "log_bucket_owner_account" {} + +resource "aws_s3_bucket" "log_bucket_us_east_2" { + bucket = aws_s3_bucket.log_bucket_us_east_2.id + + lifecycle { + prevent_destroy = true + } +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "log_bucket_us_east_2" { + bucket = aws_s3_bucket.log_bucket_us_east_2.id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} + +data "aws_iam_policy_document" "lincset_log_bucket_policy_us_east_2" { + statement { + resources = [ + "${aws_s3_bucket.log_bucket_us_east_2.arn}", + "${aws_s3_bucket.log_bucket_us_east_2.arn}/*", + ] + + actions = [ + # Needed for the app to process logs for collecting download analytics + "s3:GetObject", + "s3:ListBucket", + ] + + principals { + type = "AWS" + identifiers = [var.heroku_user.arn] + } + } + + statement { + sid = "S3ServerAccessLogsPolicy" + effect = "Allow" + resources = ["${aws_s3_bucket.log_bucket_us_east_2.arn}/*"] + actions = ["s3:PutObject"] + + condition { + test = "StringEquals" + variable = "aws:SourceAccount" + values = [data.aws_caller_identity.current.account_id] + } + + condition { + test = "ArnLike" + variable = "aws:SourceArn" + values = [aws_s3_bucket.lincset_bucket_us_east_2.arn] + } + + principals { + type = "Service" + identifiers = ["logging.s3.amazonaws.com"] + } + } +} + +resource "aws_s3_bucket_policy" "lincset_log_bucket_policy_us_east_2" { + provider = aws + + bucket = aws_s3_bucket.log_bucket.id + policy = data.aws_iam_policy_document.lincset_log_bucket_policy_us_east_2.json +} + +data "aws_iam_policy_document" "lincset_log_bucket_owner_us_east_2" { + version = "2008-10-17" + + // TODO: gate behind a "cross account" flag, since this is technically only + // needed for sponsored log bucket. + statement { + resources = [ + "${aws_s3_bucket.log_bucket_us_east_2.arn}", + "${aws_s3_bucket.log_bucket_us_east_2.arn}/*", + ] + + actions = [ + "s3:GetObject", + "s3:ListBucket", + ] + } +} + +resource "aws_iam_user_policy" "lincset_log_bucket_owner_us_east_2" { + // The Heroku IAM user will always be in the project account + provider = aws.project + + name = "${var.log_bucket_name}-ownership-policy" + user = var.heroku_user.user_name + + policy = data.aws_iam_policy_document.lincset_log_bucket_owner_us_east_2.json +} + From 53cd126b68871d1f71f49aa6acf402cc46b90d0d Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Tue, 12 Mar 2024 16:43:11 -0400 Subject: [PATCH 33/35] Remove self reference --- terraform/modules/lincset_bucket/log_bucket.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/modules/lincset_bucket/log_bucket.tf b/terraform/modules/lincset_bucket/log_bucket.tf index f0bc916..ba241ee 100644 --- a/terraform/modules/lincset_bucket/log_bucket.tf +++ b/terraform/modules/lincset_bucket/log_bucket.tf @@ -100,7 +100,7 @@ resource "aws_iam_user_policy" "lincset_log_bucket_owner" { # data "aws_canonical_user_id" "log_bucket_owner_account" {} resource "aws_s3_bucket" "log_bucket_us_east_2" { - bucket = aws_s3_bucket.log_bucket_us_east_2.id + bucket = var.log_bucket_name lifecycle { prevent_destroy = true From d598cbc09da83eafc9e5b0cf2353ab78341076a3 Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Thu, 14 Mar 2024 11:31:46 -0400 Subject: [PATCH 34/35] Include correct references for bucket id for us east 2 --- terraform/modules/lincset_bucket/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/modules/lincset_bucket/main.tf b/terraform/modules/lincset_bucket/main.tf index 68a3bd0..6382b4c 100644 --- a/terraform/modules/lincset_bucket/main.tf +++ b/terraform/modules/lincset_bucket/main.tf @@ -318,7 +318,7 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "lincset_bucket_us } resource "aws_s3_bucket_cors_configuration" "lincset_bucket_us_east_2" { - bucket = aws_s3_bucket.lincset_bucket.id + bucket = aws_s3_bucket.lincset_bucket_us_east_2.id cors_rule { allowed_origins = [ @@ -341,7 +341,7 @@ resource "aws_s3_bucket_cors_configuration" "lincset_bucket_us_east_2" { } resource "aws_s3_bucket_logging" "lincset_bucket_us_east_2" { - bucket = aws_s3_bucket.lincset_bucket.id + bucket = aws_s3_bucket.lincset_bucket_us_east_2.id target_bucket = aws_s3_bucket.log_bucket.id target_prefix = "" @@ -350,7 +350,7 @@ resource "aws_s3_bucket_logging" "lincset_bucket_us_east_2" { resource "aws_s3_bucket_versioning" "lincset_bucket_us_east_2" { count = var.versioning ? 1 : 0 - bucket = aws_s3_bucket.lincset_bucket.id + bucket = aws_s3_bucket.lincset_bucket_us_east_2.id versioning_configuration { status = "Enabled" From aaba1bf525db4f2bc2bd9c4dff8a0dc91d4bc163 Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Thu, 11 Jul 2024 09:16:10 -0400 Subject: [PATCH 35/35] Include environment variables for CloudFront and WebKnossos --- terraform/api.tf | 7 +++++++ terraform/staging_pipeline.tf | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/terraform/api.tf b/terraform/api.tf index 01fa0ee..a1d8e71 100644 --- a/terraform/api.tf +++ b/terraform/api.tf @@ -25,6 +25,10 @@ module "api" { django_cors_origin_regex_whitelist = ["^https:\\/\\/[0-9a-z\\-]+\\.netlify\\.app$"] additional_django_vars = { + CLOUDFRONT_BASE_URL = "lincbrain.org" + CLOUDFRONT_NEUROGLANCER_URL = "https://neuroglancer.lincbrain.org" + CLOUDFRONT_PEM_KEY_ID = "K3OG4MF62CGEDN" + CLOUDFRONT_PRIVATE_PEM_S3_LOCATION = "cloudfront/private_key_prod_new.pem" DJANGO_CONFIGURATION = "HerokuProductionConfiguration" DJANGO_DANDI_DANDISETS_BUCKET_NAME = module.sponsored_lincset_bucket_us_east_2.bucket_name DJANGO_DANDI_DANDISETS_BUCKET_PREFIX = "" @@ -42,6 +46,9 @@ module "api" { DJANGO_DANDI_WEB_APP_URL = "https://lincbrain.org" DJANGO_DANDI_API_URL = "https://api.lincbrain.org" DJANGO_DANDI_JUPYTERHUB_URL = "https://hub.lincbrain.org" + WEBKNOSSOS_API_URL = "https://webknossos.lincbrain.org" + WEBKNOSSOS_ORGANIZATION_DISPLAY_NAME = "LINC" + WEBKNOSSOS_ORGANIZATION_NAME = "LINC" } additional_sensitive_django_vars = { DJANGO_DANDI_DOI_API_PASSWORD = "temp" diff --git a/terraform/staging_pipeline.tf b/terraform/staging_pipeline.tf index 87d05f0..00461cd 100644 --- a/terraform/staging_pipeline.tf +++ b/terraform/staging_pipeline.tf @@ -24,6 +24,10 @@ module "api_staging" { django_cors_origin_regex_whitelist = ["https://staging--gui-staging-lincbrain-org.netlify.app"] additional_django_vars = { + CLOUDFRONT_BASE_URL = "lincbrain.org" + CLOUDFRONT_NEUROGLANCER_URL = "https://neuroglancer-staging.lincbrain.org" + CLOUDFRONT_PEM_KEY_ID = "KZQ92MU8PCLJ8" + CLOUDFRONT_PRIVATE_PEM_S3_LOCATION = "cloudfront/private_key_staging_new.pem" DJANGO_CONFIGURATION = "HerokuStagingConfiguration" DJANGO_DANDI_DANDISETS_BUCKET_NAME = module.staging_lincset_bucket_us_east_2.bucket_name DJANGO_DANDI_DANDISETS_BUCKET_PREFIX = "" @@ -41,6 +45,9 @@ module "api_staging" { DJANGO_DANDI_WEB_APP_URL = "https://staging--lincbrain-org.netlify.app" DJANGO_DANDI_API_URL = "https://staging-api.lincbrain.org" DJANGO_DANDI_JUPYTERHUB_URL = "https://hub.lincbrain.org/" + WEBKNOSSOS_API_URL = "https://webknossos-staging.lincbrain.org" + WEBKNOSSOS_ORGANIZATION_DISPLAY_NAME = "LINC Staging" + WEBKNOSSOS_ORGANIZATION_NAME = "LINC_Staging" } additional_sensitive_django_vars = { DJANGO_DANDI_DOI_API_PASSWORD = "temp"