chore(docker): add postgresql service properly

.env.tpl

@@ -2,14 +2,20 @@ ANSIBLE_VAULT_KEY=${ANSIBLE_VAULT_KEY}
 ENV=${ENV}
 FLOWER_PORT=${FLOWER_PORT}
 FLOWER_PASSWORD=${FLOWER_PASSWORD}
+HOST_POSTGRES_DUMP=${HOST_POSTGRES_DUMP}
 MAPBOX_API_KEY=${MAPBOX_API_KEY}
+POSTGRES_DUMP_HOST=${POSTGRES_DUMP_HOST}
+POSTGRES_DUMP_CONTAINER=${POSTGRES_DUMP_CONTAINER}
 POSTGRES_HOST=${POSTGRES_HOST}
 POSTGRES_PORT=${POSTGRES_PORT}
 POSTGRES_USER=${POSTGRES_USER}
 POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
 POSTGRES_DB=${POSTGRES_DB}
-POSTGRES_DB_PRIVATE=${POSTGRES_DB_PRIVATE}
-POSTGRES_DB_SANDBOX=${POSTGRES_DB_SANDBOX}
+POSTGRES_EPIGRAPH_USER=${POSTGRES_EPIGRAPH_USER}
+POSTGRES_EPIGRAPH_PASSWORD=${POSTGRES_EPIGRAPH_PASSWORD}
+POSTGRES_EPIGRAPH_DB=${POSTGRES_EPIGRAPH_DB}
+POSTGRES_EPIGRAPH_DB_PRIVATE=${POSTGRES_DB_EPIGRAPH_PRIVATE}
+POSTGRES_EPIGRAPH_DB_SANDBOX=${POSTGRES_DB_EPIGRAPH_SANDBOX}
 RECAPTCHA_PUBLIC_KEY=${RECAPTCHA_PUBLIC_KEY}
 RECAPTCHA_PRIVATE_KEY=${RECAPTCHA_PRIVATE_KEY}
 REDIS_HOST=${REDIS_HOST}

.github/workflows/main.yaml

@@ -10,13 +10,17 @@ env:
   ENV: dev
   FLOWER_PORT: 28888
   FLOWER_PASSWORD: flowerpass
+  POSTGRES_DUMP_HOST: /tmp/dump
   POSTGRES_HOST: epigraphhub-db
   POSTGRES_PORT: 25432
-  POSTGRES_USER: dev_epigraph
-  POSTGRES_PASSWORD: dev_epigraph
-  POSTGRES_DB: dev_epigraphhub
-  POSTGRES_DB_PRIVATE: dev_privatehub
-  POSTGRES_DB_SANDBOX: dev_sandbox
+  POSTGRES_USER: postgres
+  POSTGRES_PASSWORD: postgres
+  POSTGRES_DB: postgres
+  POSTGRES_EPIGRAPH_USER: dev_epigraph
+  POSTGRES_EPIGRAPH_PASSWORD: dev_epigraph
+  POSTGRES_EPIGRAPH_DB: dev_epigraphhub
+  POSTGRES_EPIGRAPH_DB_PRIVATE: dev_privatehub
+  POSTGRES_EPIGRAPH_DB_SANDBOX: dev_sandbox
   REDIS_HOST: epigraphhub-redis
   REDIS_PORT: 6379
   REDIS_PASSWORD: redispass

Data_Collection/CRON_scripts/config.py

@@ -8,10 +8,10 @@
 
 DB_HOST = os.environ.get("POSTGRES_HOST")
 DB_PORT = os.environ.get("POSTGRES_PORT")
-DB_USER = os.environ.get("POSTGRES_USER")
-DB_PASSWORD = os.environ.get("POSTGRES_PASSWORD")
-DB_NAME = os.environ.get("POSTGRES_DB")
-DB_NAME_PRIVATE = os.environ.get("POSTGRES_DB_PRIVATE")
+DB_USER = os.environ.get("POSTGRES_EPIGRAPH_USER")
+DB_PASSWORD = os.environ.get("POSTGRES_EPIGRAPH_PASSWORD")
+DB_NAME = os.environ.get("POSTGRES_EPIGRAPH_DB")
+DB_NAME_PRIVATE = os.environ.get("POSTGRES_EPIGRAPH_DB_PRIVATE")
 
 DB_URI = (
     f"postgresql://{DB_USER}:{DB_PASSWORD}"

Makefile

@@ -4,12 +4,12 @@ SERVICE:=epigraphhub-superset
 ENV:=dev
 CONSOLE:=bash
 CRON:=
+FILE:=
 
 DOCKER=docker-compose \
 	--env-file .env \
 	--project-name eph-$(ENV) \
-	--file docker/compose-base.yaml \
-	--file docker/compose-$(ENV).yaml
+	--file docker/compose.yaml
 
 # DOCKER
 
@@ -30,6 +30,11 @@ docker-stop:
 	$(DOCKER) stop ${SERVICES}
 
 
+.PHONY:docker-down
+docker-down:
+	$(DOCKER) down --volumes
+
+
 .PHONY:docker-restart
 docker-restart: docker-stop docker-start
 	echo "[II] Docker services restarted!"
@@ -38,6 +43,10 @@ docker-restart: docker-stop docker-start
 docker-logs-follow:
 	$(DOCKER) logs --follow --tail 300 ${SERVICES}
 
+.PHONY:docker-logs-follow
+docker-logs-follow:
+	$(DOCKER) logs --follow --tail 100 ${SERVICES}
+
 .PHONY:docker-logs
 docker-logs:
 	$(DOCKER) logs --tail 300 ${SERVICES}
@@ -60,7 +69,7 @@ docker-wait-all:
 docker-dev-prepare-db:
 	# used for development
 	$(DOCKER) exec -T epigraphhub-superset \
-		bash /opt/EpiGraphHub/docker/postgresql/prepare-db.sh
+		bash /opt/EpiGraphHub/docker/postgresql/dev/prepare-db.sh
 
 
 .PHONY:docker-run-cron
@@ -100,6 +109,18 @@ docker-run-bash:
 	$(DOCKER) run --rm ${SERVICE} bash
 
 
+.PHONY:docker-db-dump
+docker-db-dump:
+	$(DOCKER) exec -T epigraphhub-superset bash \
+		/opt/EpiGraphHub/docker/postgresql/dump.sh
+
+
+.PHONY:docker-db-restore
+docker-db-restore:
+	$(DOCKER) exec -T epigraphhub-superset bash \
+		/opt/EpiGraphHub/docker/postgresql/restore.sh ${FILE}
+
+
 # conda
 
 .ONESHELL:

docker/compose-base.yaml → docker/compose.yaml

@@ -38,9 +38,11 @@ services:
     volumes:
       - ..:/opt/EpiGraphHub
       - ${SUPERSET_DB_PATH_DIR_HOST}:/opt/data/superset/
+      - ${POSTGRES_DUMP_HOST}:${POSTGRES_DUMP_CONTAINER}
     ports:
       - ${SUPERSET_PORT}:8088
     depends_on:
+      - epigraphhub-db
       - epigraphhub-redis
       - epigraphhub-celery
       - epigraphhub-celery-beat
@@ -167,3 +169,4 @@ services:
 
 volumes:
   redis:
+  pgdata:

docker/postgresql/dev/prepare-db.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+# =================================
+# NOTE: USE IT JUST FOR DEVELOPMENT
+# =================================
+
+set -ex
+
+DOCKER_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd ../.. && pwd )"
+PSQL_CONNECTION="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}"
+
+echo "[II] CREATE DATABASE"
+psql "${PSQL_CONNECTION}/postgres" < ${DOCKER_DIR}/sql/dev/database.sql
+
+echo "[II] LOAD EPIGRAPHHUB DUMP"
+psql "${PSQL_CONNECTION}/${POSTGRES_EPIGRAPH_DB}" < ${DOCKER_DIR}/sql/dev/epigraphhub.sql
+
+echo "[II] LOAD PRIVATE DUMP"
+psql "${PSQL_CONNECTION}/${POSTGRES_EPIGRAPH_DB_PRIVATE}" < ${DOCKER_DIR}/sql/dev/privatehub.sql

docker/postgresql/dump.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+PROJECT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd ../.. && pwd )"
+
+if [ -f ${PROJECT_DIR}/.env ]; then
+    # Load Environment Variables
+    export $(cat ${PROJECT_DIR}/.env | grep -v '#' | sed 's/\r$//' | awk '/=/ {print $1}' )
+fi
+
+POSTGRES_DUMP_HOST=${POSTGRES_DUMP_HOST:-/tmp/dump}
+mkdir -p ${POSTGRES_DUMP_HOST}
+
+set -ex
+
+PGPASSWORD=${POSTGRES_PASSWORD} pg_dumpall \
+  --host ${POSTGRES_HOST} \
+  --port ${POSTGRES_PORT} \
+  --user ${POSTGRES_USER} \
+  > ${POSTGRES_DUMP_HOST}/$(date --iso-8601).dump

docker/postgresql/restore.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+PROJECT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd ../.. && pwd )"
+
+if [ -f ${PROJECT_DIR}/.env ]; then
+    # Load Environment Variables
+    export $(cat ${PROJECT_DIR}/.env | grep -v '#' | sed 's/\r$//' | awk '/=/ {print $1}' )
+fi
+
+POSTGRES_DUMP_HOST=${POSTGRES_DUMP_HOST:-/tmp/dump}
+
+set -ex
+
+PGPASSWORD=${POSTGRES_PASSWORD} psql \
+  --host ${POSTGRES_HOST} \
+  --port ${POSTGRES_PORT} \
+  --user ${POSTGRES_USER} \
+  -f ${POSTGRES_DUMP_HOST}/${1} ${POSTGRES_DB}