From ad012517b155d7851cf02bb92bad2c0574dcb167 Mon Sep 17 00:00:00 2001 From: Spartan Plume Date: Tue, 20 Feb 2024 17:37:57 +0100 Subject: [PATCH] Configure docker-compose for local development --- .dockerignore | 6 + .env | 4 + .gitignore | 1 + ...0463030b0f4806010b897ab05a8917edab158.json | 18 +++ Cargo.lock | 22 ++++ Cargo.toml | 2 + Dockerfile | 27 +++++ README.md | 8 ++ configuration/base.yaml | 5 + configuration/development.yaml | 4 + .../local.yaml | 14 +-- configuration/production.yaml | 4 + configuration/test.yaml | 4 + docker-compose.yaml | 41 +++++++ src/configuration.rs | 107 ++++++++++++++---- src/lib.rs | 1 + src/main.rs | 14 ++- src/routes/tournaments.rs | 2 +- src/secret_env.rs | 101 +++++++++++++++++ tests/integration.rs | 14 +-- 20 files changed, 355 insertions(+), 44 deletions(-) create mode 100644 .dockerignore create mode 100644 .sqlx/query-f3001f4e0c82679948fbd3e96e10463030b0f4806010b897ab05a8917edab158.json create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 configuration/base.yaml create mode 100644 configuration/development.yaml rename configuration.yaml => configuration/local.yaml (51%) create mode 100644 configuration/production.yaml create mode 100644 configuration/test.yaml create mode 100644 docker-compose.yaml create mode 100644 src/secret_env.rs diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..5d2a7ec --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +.env +target/ +tests/ +Dockerfile +scripts/ +migrations/ diff --git a/.env b/.env index 086d2ab..dcf37ba 100644 --- a/.env +++ b/.env @@ -1 +1,5 @@ +# For sqlx build DATABASE_URL="postgres://postgres:password@localhost:5432/tosurnament" +# For docker-compose +DB_USERNAME="tosurnament" +DB_NAME="tosurnament" diff --git a/.gitignore b/.gitignore index 0b42d2d..1f6235c 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /target +secrets/ diff --git a/.sqlx/query-f3001f4e0c82679948fbd3e96e10463030b0f4806010b897ab05a8917edab158.json b/.sqlx/query-f3001f4e0c82679948fbd3e96e10463030b0f4806010b897ab05a8917edab158.json new file mode 100644 index 0000000..943856a --- /dev/null +++ b/.sqlx/query-f3001f4e0c82679948fbd3e96e10463030b0f4806010b897ab05a8917edab158.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO tournaments (id, name, acronym, created_at, updated_at)\n VALUES ($1, $2, $3, $4, $5)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Timestamptz", + "Timestamptz" + ] + }, + "nullable": [] + }, + "hash": "f3001f4e0c82679948fbd3e96e10463030b0f4806010b897ab05a8917edab158" +} diff --git a/Cargo.lock b/Cargo.lock index 46e8d53..06ccd27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -436,6 +436,15 @@ dependencies = [ "yaml-rust", ] +[[package]] +name = "config-secret" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eac0b238d284e5d48e4763c08b8e480a21879e24a39735a707c39b368dd637" +dependencies = [ + "config", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -1932,6 +1941,17 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-aux" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a86348501c129f3ad50c2f4635a01971f76974cd8a3f335988a0f1581c082765" +dependencies = [ + "chrono", + "serde", + "serde_json", +] + [[package]] name = "serde_derive" version = "1.0.196" @@ -2563,10 +2583,12 @@ dependencies = [ "actix-web", "chrono", "config", + "config-secret", "once_cell", "reqwest", "secrecy", "serde", + "serde-aux", "sqlx", "tokio", "tracing", diff --git a/Cargo.toml b/Cargo.toml index d5f170e..5d19f33 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,8 @@ tracing-bunyan-formatter = "0.3" tracing-log = "0.2" secrecy = { version = "0.8", features = ["serde"] } tracing-actix-web = "0.7" +serde-aux = "4" +config-secret = "0.1" [dependencies.sqlx] version = "0.7" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8b2f23c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM lukemathwalker/cargo-chef:latest-rust-1 AS chef +WORKDIR /app + +FROM chef AS planner +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + +FROM chef AS builder +COPY --from=planner /app/recipe.json recipe.json +# Build dependencies - this is the caching Docker layer! +RUN cargo chef cook --release --recipe-path recipe.json +# Build application +COPY . . +ENV SQLX_OFFLINE true +RUN cargo build --release --bin tosurnament + +FROM debian:bookworm-slim AS runtime +WORKDIR /app +RUN apt-get update -y \ + && apt-get install -y --no-install-recommends openssl ca-certificates \ + && apt-get autoremove -y \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* +COPY --from=builder /app/target/release/tosurnament tosurnament +COPY configuration configuration +ENV APP_ENVIRONMENT production +ENTRYPOINT ["./tosurnament"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..59eb5bc --- /dev/null +++ b/README.md @@ -0,0 +1,8 @@ +# Tosurnament + +## Setup + +### Secrets + +Create a `secrets` folder and add the following files in it: +- `db_password.txt`: Password for your db diff --git a/configuration/base.yaml b/configuration/base.yaml new file mode 100644 index 0000000..c98c8a0 --- /dev/null +++ b/configuration/base.yaml @@ -0,0 +1,5 @@ +application: + port: 8080 +database: + host: "127.0.0.1" + port: 5432 diff --git a/configuration/development.yaml b/configuration/development.yaml new file mode 100644 index 0000000..1de1c6b --- /dev/null +++ b/configuration/development.yaml @@ -0,0 +1,4 @@ +application: + host: 0.0.0.0 +database: + require_ssl: false diff --git a/configuration.yaml b/configuration/local.yaml similarity index 51% rename from configuration.yaml rename to configuration/local.yaml index b4847a5..3b04ec9 100644 --- a/configuration.yaml +++ b/configuration/local.yaml @@ -1,7 +1,7 @@ -application_port: 8080 -database: - host: "127.0.0.1" - port: 5432 - username: "postgres" - password: "password" - database_name: "tosurnament" \ No newline at end of file +application: + host: "127.0.0.1" +database: + username: "postgres" + password: "password" + database_name: "tosurnament" + require_ssl: false diff --git a/configuration/production.yaml b/configuration/production.yaml new file mode 100644 index 0000000..cd4608a --- /dev/null +++ b/configuration/production.yaml @@ -0,0 +1,4 @@ +application: + host: 0.0.0.0 +database: + require_ssl: true diff --git a/configuration/test.yaml b/configuration/test.yaml new file mode 100644 index 0000000..cd4608a --- /dev/null +++ b/configuration/test.yaml @@ -0,0 +1,4 @@ +application: + host: 0.0.0.0 +database: + require_ssl: true diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..8df90e6 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,41 @@ +services: + db: + image: postgres:16 + restart: always + ports: + - 5432:5432 + secrets: + - db_password + environment: + - POSTGRES_USER=${DB_USERNAME} + - PGUSER=${DB_USERNAME} + - POSTGRES_PASSWORD_FILE=/run/secrets/db_password + - POSTGRES_DB=${DB_NAME} + healthcheck: + test: [ "CMD-SHELL", "pg_isready" ] + interval: 10s + timeout: 5s + retries: 5 + volumes: + - ./migrations/20240215193936_create_tournaments_table.sql:/docker-entrypoint-initdb.d/create_tournaments_table.tosurnament.sql + server: + build: + context: . + dockerfile: Dockerfile + ports: + - 8080:8080 + depends_on: + db: + condition: service_healthy + secrets: + - db_password + environment: + - APP_ENVIRONMENT=development + - APP_DATABASE__USERNAME=${DB_USERNAME} + - APP_DATABASE__PASSWORD_FILE=/run/secrets/db_password + - APP_DATABASE__DATABASE_NAME=${DB_NAME} + - APP_DATABASE__HOST=db + +secrets: + db_password: + file: secrets/db_password.txt diff --git a/src/configuration.rs b/src/configuration.rs index 62a47c5..d156545 100644 --- a/src/configuration.rs +++ b/src/configuration.rs @@ -1,9 +1,22 @@ +use crate::secret_env::SecretFileEnvironment; use secrecy::{ExposeSecret, Secret}; +use serde_aux::field_attributes::deserialize_number_from_string; +use sqlx::{ + postgres::{PgConnectOptions, PgSslMode}, + ConnectOptions, +}; #[derive(serde::Deserialize)] pub struct Settings { pub database: DatabaseSettings, - pub application_port: u16, + pub application: ApplicationSettings, +} + +#[derive(serde::Deserialize)] +pub struct ApplicationSettings { + pub host: String, + #[serde(deserialize_with = "deserialize_number_from_string")] + pub port: u16, } #[derive(serde::Deserialize)] @@ -11,39 +24,89 @@ pub struct DatabaseSettings { pub username: String, pub password: Secret, pub host: String, - pub port: String, + #[serde(deserialize_with = "deserialize_number_from_string")] + pub port: u16, pub database_name: String, + pub require_ssl: bool, } impl DatabaseSettings { - pub fn connection_string(&self) -> Secret { - Secret::new(format!( - "postgres://{}:{}@{}:{}/{}", - self.username, - self.password.expose_secret(), - self.host, - self.port, - self.database_name - )) + pub fn without_db(&self) -> PgConnectOptions { + let ssl_mode = if self.require_ssl { + PgSslMode::Require + } else { + PgSslMode::Prefer + }; + PgConnectOptions::new() + .host(&self.host) + .port(self.port) + .username(&self.username) + .password(self.password.expose_secret()) + .ssl_mode(ssl_mode) } - pub fn connection_string_without_db(&self) -> Secret { - Secret::new(format!( - "postgres://{}:{}@{}:{}", - self.username, - self.password.expose_secret(), - self.host, - self.port - )) + pub fn with_db(&self) -> PgConnectOptions { + let options = self.without_db().database(&self.database_name); + options.log_statements(tracing_log::log::LevelFilter::Trace) } } pub fn get_configuration() -> Result { + let base_path = std::env::current_dir().expect("Failed to determine the current directory"); + let configuration_dir = base_path.join("configuration"); + let environment: Environment = std::env::var("APP_ENVIRONMENT") + .unwrap_or_else(|_| "local".into()) + .try_into() + .expect("Failed to parse APP_ENVIRONMENT"); + let environment_filename = format!("{}.yaml", environment.as_str()); + let settings = config::Config::builder() - .add_source(config::File::new( - "configuration.yaml", - config::FileFormat::Yaml, + .add_source(config::File::from(configuration_dir.join("base.yaml"))) + .add_source(config::File::from( + configuration_dir.join(environment_filename), )) + .add_source( + config::Environment::with_prefix("APP") + .prefix_separator("_") + .separator("__"), + ) + .add_source( + SecretFileEnvironment::with_prefix("APP") + .prefix_separator("_") + .separator("__"), + ) .build()?; settings.try_deserialize::() } + +pub enum Environment { + Local, + Development, + Test, + Production, +} + +impl Environment { + pub fn as_str(&self) -> &'static str { + match self { + Environment::Local => "local", + Environment::Development => "development", + Environment::Test => "test", + Environment::Production => "production", + } + } +} + +impl TryFrom for Environment { + type Error = String; + + fn try_from(s: String) -> Result { + match s.to_lowercase().as_str() { + "local" => Ok(Self::Local), + "development" => Ok(Self::Development), + "test" => Ok(Self::Test), + "production" => Ok(Self::Production), + other => Err(format!("{} is not a supported environment", other)), + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 80d0e95..57acdde 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,5 @@ pub mod configuration; pub mod routes; +pub mod secret_env; pub mod startup; pub mod telemetry; diff --git a/src/main.rs b/src/main.rs index 9078b15..fa88f00 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,6 @@ use std::net::TcpListener; -use secrecy::ExposeSecret; -use sqlx::PgPool; +use sqlx::postgres::PgPoolOptions; use tosurnament::configuration::get_configuration; use tosurnament::startup::run; @@ -13,10 +12,13 @@ async fn main() -> Result<(), std::io::Error> { init_subscriber(subscriber); let configuration = get_configuration().expect("Failed to read configuration"); - let db_pool = PgPool::connect(configuration.database.connection_string().expose_secret()) - .await - .expect("Failed to connect to Postgres"); - let address = format!("127.0.0.1:{}", configuration.application_port); + let db_pool = PgPoolOptions::new() + .acquire_timeout(std::time::Duration::from_secs(2)) + .connect_lazy_with(configuration.database.with_db()); + let address = format!( + "{}:{}", + configuration.application.host, configuration.application.port + ); let listener = TcpListener::bind(address).expect("Failed to bind port"); run(listener, db_pool)?.await } diff --git a/src/routes/tournaments.rs b/src/routes/tournaments.rs index edf3387..218dfac 100644 --- a/src/routes/tournaments.rs +++ b/src/routes/tournaments.rs @@ -11,7 +11,7 @@ struct Tournament { } #[allow(clippy::async_yields_async)] -#[post("/tournament")] +#[post("/tournaments")] #[tracing::instrument(skip_all, fields(%tournament.name))] async fn create_tournament( tournament: web::Json, diff --git a/src/secret_env.rs b/src/secret_env.rs new file mode 100644 index 0000000..46cea10 --- /dev/null +++ b/src/secret_env.rs @@ -0,0 +1,101 @@ +// Code heavily inspired from config-rs (https://github.com/mehcode/config-rs) env code and modified to support secret files (for Docker for example) + +use std::env; + +use config::{ConfigError, Map, Source, Value, ValueKind}; + +#[derive(Clone, Debug, Default)] +pub struct SecretFileEnvironment { + prefix: Option, + prefix_separator: Option, + separator: Option, +} + +impl SecretFileEnvironment { + pub fn with_prefix(s: &str) -> Self { + Self { + prefix: Some(s.into()), + ..Self::default() + } + } + + pub fn prefix(mut self, s: &str) -> Self { + self.prefix = Some(s.into()); + self + } + + pub fn prefix_separator(mut self, s: &str) -> Self { + self.prefix_separator = Some(s.into()); + self + } + + pub fn separator(mut self, s: &str) -> Self { + self.separator = Some(s.into()); + self + } +} + +impl Source for SecretFileEnvironment { + fn clone_into_box(&self) -> Box { + Box::new((*self).clone()) + } + + fn collect(&self) -> Result, ConfigError> { + let mut m = Map::new(); + let uri: String = "the environment".into(); + + let separator = self.separator.as_deref().unwrap_or(""); + let prefix_separator = match (self.prefix_separator.as_deref(), self.separator.as_deref()) { + (Some(pre), _) => pre, + (None, Some(sep)) => sep, + (None, None) => "_", + }; + + // Define a prefix pattern to test and exclude from keys + let prefix_pattern = self + .prefix + .as_ref() + .map(|prefix| format!("{}{}", prefix, prefix_separator).to_lowercase()); + + let collector = |(key, value): (String, String)| { + // Treat empty environment variables as unset + if value.is_empty() { + return; + } + + let mut key = key.to_lowercase(); + let mut value = value; + + // Check for prefix + if let Some(ref prefix_pattern) = prefix_pattern { + if key.starts_with(prefix_pattern) { + // Remove this prefix from the key + key = key[prefix_pattern.len()..].to_string(); + } else { + // Skip this key + return; + } + } + + if key.ends_with("_file") { + let content = std::fs::read_to_string(value).unwrap_or("".to_string()); + if content.is_empty() { + return; + } + key = key[..key.len() - 5].to_string(); + value = content; + } + + // If separator is given replace with `.` + if !separator.is_empty() { + key = key.replace(separator, "."); + } + + m.insert(key, Value::new(Some(&uri), ValueKind::String(value))); + }; + + env::vars().for_each(collector); + + Ok(m) + } +} diff --git a/tests/integration.rs b/tests/integration.rs index 11a32f7..2a38562 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -1,7 +1,6 @@ use std::{collections::HashMap, net::TcpListener}; use once_cell::sync::Lazy; -use secrecy::ExposeSecret; use sqlx::{Connection, Executor, PgConnection, PgPool}; use uuid::Uuid; @@ -44,16 +43,15 @@ async fn spawn_app() -> TestApp { } async fn configure_database(db_config: &DatabaseSettings) -> PgPool { - let mut db_connection = - PgConnection::connect(db_config.connection_string_without_db().expose_secret()) - .await - .expect("Failed to connect to Postgres"); + let mut db_connection = PgConnection::connect_with(&db_config.without_db()) + .await + .expect("Failed to connect to Postgres"); db_connection .execute(format!(r#"CREATE DATABASE "{}";"#, db_config.database_name).as_str()) .await .expect("Failed to create database"); - let db_pool = PgPool::connect(db_config.connection_string().expose_secret()) + let db_pool = PgPool::connect_with(db_config.with_db()) .await .expect("Failed to connect to Postgres"); sqlx::migrate!("./migrations") @@ -88,7 +86,7 @@ async fn create_tournament_returns_200_for_valid_data() { body.insert("acronym", "TN"); let response = client - .post(&format!("{}/tournament", &app.address)) + .post(&format!("{}/tournaments", &app.address)) .header("Content-Type", "application/json") .json(&body) .send() @@ -119,7 +117,7 @@ async fn create_tournament_returns_400_when_data_is_missing() { invalid_body.insert(field_name, field_value); let response = client - .post(&format!("{}/tournament", &app.address)) + .post(&format!("{}/tournaments", &app.address)) .header("Content-Type", "application/json") .json(&invalid_body) .send()