From dbc36b14dd1a133f00df0b18e593daeafc9ef015 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Tue, 8 Oct 2024 12:11:56 +0200 Subject: [PATCH] Fix deployment problems --- ansible/deploy.yml | 3 +- ansible/migrate.yml | 9 ++-- ansible/roles/deploy/tasks/main.yml | 1 + db/partition.sql | 3 -- db/partitions_generated.sql | 73 +++++++++++++++++++++++++++++ ingestion/Cargo.toml | 2 + ingestion/src/main.rs | 17 +++++-- 7 files changed, 95 insertions(+), 13 deletions(-) delete mode 100644 db/partition.sql create mode 100644 db/partitions_generated.sql diff --git a/ansible/deploy.yml b/ansible/deploy.yml index 6ed6c9c..6fec7c5 100644 --- a/ansible/deploy.yml +++ b/ansible/deploy.yml @@ -1,6 +1,7 @@ --- - name: Deploy binaries - hosts: lard-a # deploy to lard-a for now, might need to define a separate VM later? + # deploy to lard-a for now, might need to define a separate VM later? + hosts: lard-a remote_user: ubuntu roles: - role: deploy diff --git a/ansible/migrate.yml b/ansible/migrate.yml index 1f20af7..cf6019c 100644 --- a/ansible/migrate.yml +++ b/ansible/migrate.yml @@ -2,6 +2,7 @@ - name: Perform DB migration hosts: servers remote_user: ubuntu + gather_facts: false vars: primary: lard-a @@ -18,12 +19,12 @@ become: true become_user: postgres block: - - name: Create data partition tables in lard + - name: Update public schema in lard community.postgresql.postgresql_script: db: lard - path: /etc/postgresql/16/db/partition.sql + path: /etc/postgresql/16/db/public.sql - - name: Update public schema in lard + - name: Create partition tables in lard community.postgresql.postgresql_script: db: lard - path: /etc/postgresql/16/db/public.sql + path: /etc/postgresql/16/db/partitions_generated.sql diff --git a/ansible/roles/deploy/tasks/main.yml b/ansible/roles/deploy/tasks/main.yml index 6228c72..e2f8e7d 100644 --- a/ansible/roles/deploy/tasks/main.yml +++ b/ansible/roles/deploy/tasks/main.yml @@ -43,6 +43,7 @@ name: lard_ingestion state: restarted enabled: true + become: true - name: Unset environment variables ansible.builtin.command: systemctl unset-environment LARD_CONN_STRING STINFO_CONN_STRING diff --git a/db/partition.sql b/db/partition.sql deleted file mode 100644 index 22355ab..0000000 --- a/db/partition.sql +++ /dev/null @@ -1,3 +0,0 @@ --- TODO: this is just for testing, need to think about how to deal with partitions -CREATE TABLE data_y2000_to_y2050 PARTITION OF public.data - FOR VALUES FROM ('2000-01-01 00:00:00+00') TO ('2050-01-01 00:00:00+00'); diff --git a/db/partitions_generated.sql b/db/partitions_generated.sql new file mode 100644 index 0000000..070a914 --- /dev/null +++ b/db/partitions_generated.sql @@ -0,0 +1,73 @@ +-- Generated by simple script for testing +CREATE TABLE IF NOT EXISTS data_y1950_to_y2000 PARTITION OF public.data +FOR VALUES FROM ('1950-01-01 00:00:00+00') TO ('2000-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS data_y2000_to_y2010 PARTITION OF public.data +FOR VALUES FROM ('2000-01-01 00:00:00+00') TO ('2010-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS data_y2010_to_y2015 PARTITION OF public.data +FOR VALUES FROM ('2010-01-01 00:00:00+00') TO ('2015-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS data_y2015_to_y2016 PARTITION OF public.data +FOR VALUES FROM ('2015-01-01 00:00:00+00') TO ('2016-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS data_y2016_to_y2017 PARTITION OF public.data +FOR VALUES FROM ('2016-01-01 00:00:00+00') TO ('2017-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS data_y2017_to_y2018 PARTITION OF public.data +FOR VALUES FROM ('2017-01-01 00:00:00+00') TO ('2018-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS data_y2018_to_y2019 PARTITION OF public.data +FOR VALUES FROM ('2018-01-01 00:00:00+00') TO ('2019-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS data_y2019_to_y2020 PARTITION OF public.data +FOR VALUES FROM ('2019-01-01 00:00:00+00') TO ('2020-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS data_y2020_to_y2021 PARTITION OF public.data +FOR VALUES FROM ('2020-01-01 00:00:00+00') TO ('2021-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS data_y2021_to_y2022 PARTITION OF public.data +FOR VALUES FROM ('2021-01-01 00:00:00+00') TO ('2022-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS data_y2022_to_y2023 PARTITION OF public.data +FOR VALUES FROM ('2022-01-01 00:00:00+00') TO ('2023-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS data_y2023_to_y2024 PARTITION OF public.data +FOR VALUES FROM ('2023-01-01 00:00:00+00') TO ('2024-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS data_y2024_to_y2025 PARTITION OF public.data +FOR VALUES FROM ('2024-01-01 00:00:00+00') TO ('2025-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS data_y2025_to_y2026 PARTITION OF public.data +FOR VALUES FROM ('2025-01-01 00:00:00+00') TO ('2026-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS data_y2026_to_y2027 PARTITION OF public.data +FOR VALUES FROM ('2026-01-01 00:00:00+00') TO ('2027-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS data_y2027_to_y2028 PARTITION OF public.data +FOR VALUES FROM ('2027-01-01 00:00:00+00') TO ('2028-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS data_y2028_to_y2029 PARTITION OF public.data +FOR VALUES FROM ('2028-01-01 00:00:00+00') TO ('2029-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS data_y2029_to_y2030 PARTITION OF public.data +FOR VALUES FROM ('2029-01-01 00:00:00+00') TO ('2030-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS nonscalar_data_y1950_to_y2000 PARTITION OF public.nonscalar_data +FOR VALUES FROM ('1950-01-01 00:00:00+00') TO ('2000-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS nonscalar_data_y2000_to_y2010 PARTITION OF public.nonscalar_data +FOR VALUES FROM ('2000-01-01 00:00:00+00') TO ('2010-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS nonscalar_data_y2010_to_y2015 PARTITION OF public.nonscalar_data +FOR VALUES FROM ('2010-01-01 00:00:00+00') TO ('2015-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS nonscalar_data_y2015_to_y2016 PARTITION OF public.nonscalar_data +FOR VALUES FROM ('2015-01-01 00:00:00+00') TO ('2016-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS nonscalar_data_y2016_to_y2017 PARTITION OF public.nonscalar_data +FOR VALUES FROM ('2016-01-01 00:00:00+00') TO ('2017-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS nonscalar_data_y2017_to_y2018 PARTITION OF public.nonscalar_data +FOR VALUES FROM ('2017-01-01 00:00:00+00') TO ('2018-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS nonscalar_data_y2018_to_y2019 PARTITION OF public.nonscalar_data +FOR VALUES FROM ('2018-01-01 00:00:00+00') TO ('2019-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS nonscalar_data_y2019_to_y2020 PARTITION OF public.nonscalar_data +FOR VALUES FROM ('2019-01-01 00:00:00+00') TO ('2020-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS nonscalar_data_y2020_to_y2021 PARTITION OF public.nonscalar_data +FOR VALUES FROM ('2020-01-01 00:00:00+00') TO ('2021-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS nonscalar_data_y2021_to_y2022 PARTITION OF public.nonscalar_data +FOR VALUES FROM ('2021-01-01 00:00:00+00') TO ('2022-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS nonscalar_data_y2022_to_y2023 PARTITION OF public.nonscalar_data +FOR VALUES FROM ('2022-01-01 00:00:00+00') TO ('2023-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS nonscalar_data_y2023_to_y2024 PARTITION OF public.nonscalar_data +FOR VALUES FROM ('2023-01-01 00:00:00+00') TO ('2024-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS nonscalar_data_y2024_to_y2025 PARTITION OF public.nonscalar_data +FOR VALUES FROM ('2024-01-01 00:00:00+00') TO ('2025-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS nonscalar_data_y2025_to_y2026 PARTITION OF public.nonscalar_data +FOR VALUES FROM ('2025-01-01 00:00:00+00') TO ('2026-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS nonscalar_data_y2026_to_y2027 PARTITION OF public.nonscalar_data +FOR VALUES FROM ('2026-01-01 00:00:00+00') TO ('2027-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS nonscalar_data_y2027_to_y2028 PARTITION OF public.nonscalar_data +FOR VALUES FROM ('2027-01-01 00:00:00+00') TO ('2028-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS nonscalar_data_y2028_to_y2029 PARTITION OF public.nonscalar_data +FOR VALUES FROM ('2028-01-01 00:00:00+00') TO ('2029-01-01 00:00:00+00'); +CREATE TABLE IF NOT EXISTS nonscalar_data_y2029_to_y2030 PARTITION OF public.nonscalar_data +FOR VALUES FROM ('2029-01-01 00:00:00+00') TO ('2030-01-01 00:00:00+00'); diff --git a/ingestion/Cargo.toml b/ingestion/Cargo.toml index 2dc4d20..c98585b 100644 --- a/ingestion/Cargo.toml +++ b/ingestion/Cargo.toml @@ -6,6 +6,8 @@ edition.workspace = true [features] integration_tests = [] kafka = [] +# TODO: remove this when finished with testing Obsinn ingestion +kafka_prod = [] [dependencies] axum.workspace = true diff --git a/ingestion/src/main.rs b/ingestion/src/main.rs index af176d2..a1c1cfe 100644 --- a/ingestion/src/main.rs +++ b/ingestion/src/main.rs @@ -2,14 +2,13 @@ use bb8_postgres::PostgresConnectionManager; use std::sync::{Arc, RwLock}; use tokio_postgres::NoTls; -#[cfg(feature = "kafka")] -use lard_ingestion::kvkafka; use lard_ingestion::permissions; const PARAMCONV: &str = "resources/paramconversions.csv"; #[tokio::main] async fn main() -> Result<(), Box> { + println!("LARD ingestion service starting up..."); // TODO: use clap for argument parsing let args: Vec = std::env::args().collect(); @@ -24,6 +23,7 @@ async fn main() -> Result<(), Box> { let permit_tables = Arc::new(RwLock::new(permissions::fetch_permits().await?)); let background_permit_tables = permit_tables.clone(); + println!("Spawing task to fetch permissions from StInfoSys..."); // background task to refresh permit tables every 30 mins tokio::task::spawn(async move { let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(30 * 60)); @@ -48,10 +48,17 @@ async fn main() -> Result<(), Box> { let db_pool = bb8::Pool::builder().build(manager).await?; // Spawn kvkafka reader - let kafka_group = args[1].to_string(); - #[cfg(feature = "kafka")] - tokio::spawn(kvkafka::read_and_insert(db_pool.clone(), kafka_group)); + #[cfg(feature = "kafka_prod")] + { + let kafka_group = args[1].to_string(); + println!("Spawing kvkafka reader..."); + tokio::spawn(lard_ingestion::kvkafka::read_and_insert( + db_pool.clone(), + kafka_group, + )); + } // Set up and run our server + database + println!("Ingestion server started!"); lard_ingestion::run(db_pool, PARAMCONV, permit_tables).await }