Litestram (#33)

* litestram - litestream restore script * litestram - set busy timeout pragma on sqlite db * litestram - restore and replicate scripts, replicate before new deploy * litestram - remove litestream automatic backup from terraform in favor of simple copy to s3 - litestream issues with gcp:fuse
Plebeian-Technology · Jun 6, 2024 · a15c33d · a15c33d
1 parent e989183
commit a15c33d
Show file tree

Hide file tree

Showing 9 changed files with 85 additions and 44 deletions.
diff --git a/.gitignore b/.gitignore
@@ -17,24 +17,19 @@
 # Ignore all logfiles and tempfiles.
 /log/*
 /tmp/*
-!/log/.keep
-!/tmp/.keep
 
 # Ignore pidfiles, but keep the directory.
 /tmp/pids/*
 !/tmp/pids/
-!/tmp/pids/.keep
 
 # Ignore storage (uploaded files in development and any SQLite databases).
 /storage/*
-!/storage/.keep
 !/storage/seeds/data/locales.json
 !/storage/seeds/data/united_states/congress
 !/storage/seeds/data/united_states/maryland/baltimore
 !/storage/geojson/baltimore-maryland-united_states.geojson
 /tmp/storage/*
 !/tmp/storage/
-!/tmp/storage/.keep
 
 /public/assets
 
@@ -54,8 +49,14 @@ doc/
 
 /app/assets/.well-known
 /app/assets/builds/*
-!/app/assets/builds/.keep
 
 tf/secrets.tfvars
 tf/.terraform*
 tf/plans
+
+litestream/config/*
+litestream/dbs/*
+
+
+
+!**/**/.keep
diff --git a/app/frontend/entrypoints/application.tsx b/app/frontend/entrypoints/application.tsx
@@ -1,5 +1,5 @@
 import { SentryUtil } from "app/frontend/sway_utils/sentry";
-SentryUtil.init();
+SentryUtil.init(); // only fulfilled in prod
 
 import { InertiaProgress } from "@inertiajs/progress";
 import { createInertiaApp } from "@inertiajs/react";

diff --git a/config/database.yml b/config/database.yml
@@ -38,6 +38,8 @@ default: &default
   pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %>
   password: <%= ENV['SWAY_DATABASE_PASSWORD'] %>
   username: sway
+  # time to wait (in milliseconds) to obtain a write lock before raising an exception
+  # https://www.sqlite.org/pragma.html#pragma_busy_timeout
   timeout: 5000
   pragmas: # https://fractaledmind.github.io/2023/09/07/enhancing-rails-sqlite-fine-tuning/
     # level of database durability, 2 = "FULL" (sync on every write), other values include 1 = "NORMAL" (sync every 1000 written pages) and 0 = "NONE"
@@ -56,6 +58,7 @@ default: &default
     # increase the local connection cache to 2000 pages
     # https://www.sqlite.org/pragma.html#pragma_cache_size
     cache_size: 2000
+    busy_timeout: 5000
 
 development:
   <<: *default

diff --git a/litestream/config/.keep b/litestream/config/.keep
diff --git a/litestream/dbs/.keep b/litestream/dbs/.keep
diff --git a/litestream/replicate.sh b/litestream/replicate.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env zsh
+
+export GOOGLE_APPLICATION_CREDENTIALS="${HOME}/.config/gcloud/application_default_credentials.json"
+
+echo ""
+echo "###################################################################################################"
+echo ""
+echo "replicate.sh - Replicating the production db to Digial Ocean via Litestream"
+echo ""
+
+rm -rf litestream/dbs
+mkdir -p litestream/dbs
+touch litestream/dbs/.keep
+
+gsutil cp gs://sway-sqlite/production.sqlite3 litestream/dbs/production.sqlite3
+
+litestream replicate -exec "sleep 10" -config litestream/config/replicate.yml
+
+rm -rf litestream/dbs
+mkdir -p litestream/dbs
+touch litestream/dbs/.keep
+
+echo ""
+echo "replicate.sh - Finished replicating the production db to Digial Ocean via Litestream"
+echo ""
+echo "###################################################################################################"
+echo ""
diff --git a/litestream/restore.sh b/litestream/restore.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env zsh
+
+echo ""
+echo "###################################################################################################"
+echo ""
+echo "restore.sh - Restoring the production db from Digial Ocean via Litestream to storage/production.db"
+echo ""
+
+litestream restore -config litestream/config/restore.yml storage/production.db
+
+echo ""
+echo "restore.sh - Finished restoring the production db from Digial Ocean via Litestream to stroage/production.db"
+echo ""
+echo "###################################################################################################"
+echo ""
diff --git a/scripts/deploy.sh b/scripts/deploy.sh
@@ -16,11 +16,12 @@ export $(cat .env.github | xargs)
 SECRET_KEY_BASE_DUMMY=1 ./bin/rails assets:clobber
 RAILS_ENV=production SECRET_KEY_BASE_DUMMY=1 ./bin/rails assets:clobber
 
+./litestream/replicate.sh
+
 gcloud storage cp --recursive $(pwd)/storage/geojson gs://sway-sqlite/
 
 gcloud storage cp --recursive $(pwd)/storage/seeds/data gs://sway-sqlite/seeds/
 
-
 # Cloud Run requires AMD64 images
 docker buildx build . -f docker/dockerfiles/production.dockerfile --platform linux/amd64 -t us-central1-docker.pkg.dev/sway-421916/sway/sway:latest --push --compress
 

diff --git a/tf/modules/cloud_run/main.tf b/tf/modules/cloud_run/main.tf
@@ -230,21 +230,21 @@ resource "google_cloud_run_service_iam_policy" "noauth" {
   policy_data = data.google_iam_policy.noauth.policy_data
 }
 
-resource "google_cloud_run_domain_mapping" "app" {
-  name     = "${var.environment == "prod" ? "app" : var.environment}.sway.vote"
-  location = var.region
-  project  = var.project
-
-  metadata {
-    namespace = var.project
-  }
-
-  spec {
-    route_name       = var.environment == "prod" ? "sway" : "sway-${var.environment}"
-    force_override   = false
-    certificate_mode = "AUTOMATIC"
-  }
-}
+# resource "google_cloud_run_domain_mapping" "app" {
+#   name     = "${var.environment == "prod" ? "app" : var.environment}.sway.vote"
+#   location = var.region
+#   project  = var.project
+
+#   metadata {
+#     namespace = var.project
+#   }
+
+#   spec {
+#     route_name       = var.environment == "prod" ? "sway" : "sway-${var.environment}"
+#     force_override   = false
+#     certificate_mode = "AUTOMATIC"
+#   }
+# }
 
 #######################################################################################################################################
 # SQLITE BACKUP WITH LITESTREAM
@@ -280,38 +280,37 @@ resource "google_cloud_run_v2_job" "backup" {
       execution_environment = "EXECUTION_ENVIRONMENT_GEN2"
       timeout = "60s"
       service_account = "cloud-job-executor@${var.project}.iam.gserviceaccount.com"
+      max_retries = 0
 
       containers {
-        image = "litestream/litestream"
+        image = "amazon/aws-cli"
         name  = local.digitalocean_bucket_name
 
         command = [
-          "litestream",
-          "replicate",
+          "aws",
+          "s3",
+          "cp",
           "/sway/${var.environment == "prod" ? "production" : var.environment}.sqlite3",
-          "s3://${local.digitalocean_bucket_name}.nyc3.digitaloceanspaces.com/${var.environment}"
+          "s3://${local.digitalocean_bucket_name}/production-${timestamp()}.db",
+          "--endpoint-url",
+          "https://nyc3.digitaloceanspaces.com",
+          "--region",
+          "nyc3"
         ]
 
+
         env {
-          name = "SWAY_DATABASE_PASSWORD"
-          value_source {
-            secret_key_ref {
-              secret  = var.secrets.SWAY_DATABASE_PASSWORD
-              version = "latest"
-            }
-          }
-        }
-        env {
-          name = "LITESTREAM_ACCESS_KEY_ID"
+          name = "AWS_ACCESS_KEY_ID"
           value_source {
             secret_key_ref {
               secret  = var.secrets.LITESTREAM_ACCESS_KEY_ID
               version = "latest"
             }
           }
         }
+
         env {
-          name = "LITESTREAM_SECRET_ACCESS_KEY"
+          name = "AWS_SECRET_ACCESS_KEY"
           value_source {
             secret_key_ref {
               secret  = var.secrets.LITESTREAM_SECRET_ACCESS_KEY
@@ -322,7 +321,7 @@ resource "google_cloud_run_v2_job" "backup" {
 
         resources {
           limits = {
-            cpu    = "1"
+            cpu    = "1000m"
             memory = "1Gi"
           }
         }
@@ -331,12 +330,10 @@ resource "google_cloud_run_v2_job" "backup" {
           name       = local.google_bucket_name
           mount_path = "/sway"
         }
-
       }
 
       volumes {
         name = local.google_bucket_name
-
         gcs {
           bucket    = local.google_bucket_name
           read_only = false
@@ -369,9 +366,6 @@ resource "google_cloud_scheduler_job" "backup_job" {
   http_target {
     http_method = "POST"
     uri         = "https://us-${var.region}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${var.project}/jobs/${google_cloud_run_v2_job.backup.name}:run"
-    headers = {
-      "User-Agent" = "Google-Cloud-Scheduler"
-    }
     oauth_token {
       service_account_email = "cloud-job-executor@${var.project}.iam.gserviceaccount.com"
       scope                 = "https://www.googleapis.com/auth/cloud-platform"