diff --git a/.github/workflows/additional-checks.yml b/.github/workflows/additional-checks.yml new file mode 100644 index 0000000..3dc3c31 --- /dev/null +++ b/.github/workflows/additional-checks.yml @@ -0,0 +1,31 @@ +# .github/workflows/additional-checks.yml +name: additional checks + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: + +jobs: + security-audit: + name: security audit + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: install cargo-audit + run: cargo install cargo-audit + - name: run security audit + run: cargo audit --ignore RUSTSEC-2023-0071 + + docs: + name: documentation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - name: check documentation + env: + rustdocflags: "-d warnings" + run: cargo doc --no-deps --all-features --workspace diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml.disabled similarity index 83% rename from .github/workflows/ci.yml rename to .github/workflows/ci.yml.disabled index abb8589..0e5d97a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml.disabled @@ -2,14 +2,15 @@ name: Continuous Integration on: push: - branches: [ main, master ] + branches: [ main ] pull_request: - branches: [ main, master ] + branches: [ main ] workflow_dispatch: # Manual trigger env: CARGO_TERM_COLOR: always REDIS_URI: redis://localhost:6379 + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/testdb jobs: check: @@ -46,7 +47,6 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 - mongodb: image: mongo:6 ports: @@ -56,8 +56,22 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 + postgres: + image: postgres:15 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: testdb + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 env: MONGODB_URI: mongodb://localhost:27017/testdb + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/testdb steps: - uses: actions/checkout@v4 @@ -179,6 +193,19 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 + postgres: + image: postgres:15 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: testdb + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 steps: - uses: actions/checkout@v4 @@ -195,6 +222,12 @@ jobs: ~/.cargo/git/db/ target/ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Install SQLx CLI + run: cargo install sqlx-cli --no-default-features --features postgres + + - name: Run SQLx Migrations + run: sqlx migrate run - name: Install cargo-llvm-cov uses: taiki-e/install-action@cargo-llvm-cov @@ -211,3 +244,4 @@ jobs: verbose: true env: MONGODB_URI: mongodb://localhost:27017/testdb + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/testdb diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml new file mode 100644 index 0000000..f5ad6bb --- /dev/null +++ b/.github/workflows/code-quality.yml @@ -0,0 +1,44 @@ +# .github/workflows/code-quality.yml +name: Code Quality + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + +jobs: + fmt: + name: Formatting + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt + - name: Check formatting + run: cargo fmt --all -- --check + + clippy: + name: Clippy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + - name: Clippy check + run: cargo clippy --all-features --workspace -- -D warnings diff --git a/.github/workflows/platform-coverage.yml b/.github/workflows/platform-coverage.yml new file mode 100644 index 0000000..f9e5247 --- /dev/null +++ b/.github/workflows/platform-coverage.yml @@ -0,0 +1,111 @@ +name: Platform & Coverage + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + MONGODB_URI: mongodb://localhost:27017/testdb + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/testdb + +jobs: + cross-platform: + name: Cross-platform tests + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Run unit tests only + env: + SQLX_OFFLINE: true + run: cargo test --lib --workspace --no-default-features + + coverage: + name: Code coverage + runs-on: ubuntu-latest + services: + redis: + image: redis + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + mongodb: + image: mongo:6 + ports: + - 27017:27017 + options: >- + --health-cmd "mongosh --eval 'db.runCommand(\"ping\").ok'" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + postgres: + image: postgres:15 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: testdb + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + env: + REDIS_URI: redis://localhost:6379 + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/testdb + MONGODB_URI: mongodb://localhost:27017/testdb + + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: llvm-tools-preview + - uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Install sqlx-cli + run: cargo install sqlx-cli --no-default-features --features postgres + + - name: Run database migrations + run: cargo sqlx migrate run + + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + + - name: Generate code coverage + run: cargo llvm-cov --workspace --lcov --output-path lcov.info + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: lcov.info + fail_ci_if_error: true + verbose: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..043c8ce --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,80 @@ +# .github/workflows/test.yml +name: Tests + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + REDIS_URI: redis://localhost:6379 + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/testdb + MONGODB_URI: mongodb://localhost:27017/testdb + +jobs: + test: + name: Test Suite + runs-on: ubuntu-latest + services: + redis: + image: redis + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + mongodb: + image: mongo:6 + ports: + - 27017:27017 + options: >- + --health-cmd "mongosh --eval 'db.runCommand(\"ping\").ok'" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + postgres: + image: postgres:15 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: testdb + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Install sqlx-cli + run: cargo install sqlx-cli --no-default-features --features postgres + + - name: Run database migrations + run: cargo sqlx migrate run + + - name: Cargo check + run: cargo check --all-features --workspace + + - name: Run unit tests + run: cargo test --lib --all-features --workspace + + - name: Run integration tests + run: cargo test --test '*' --all-features --workspace diff --git a/.sqlx/query-1e339e959f8d2cdac13b3e2b452d2f718c0fd6cf6202d5c9139fb1afda123d29.json b/.sqlx/query-1e339e959f8d2cdac13b3e2b452d2f718c0fd6cf6202d5c9139fb1afda123d29.json new file mode 100644 index 0000000..561df27 --- /dev/null +++ b/.sqlx/query-1e339e959f8d2cdac13b3e2b452d2f718c0fd6cf6202d5c9139fb1afda123d29.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM tasks WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "1e339e959f8d2cdac13b3e2b452d2f718c0fd6cf6202d5c9139fb1afda123d29" +} diff --git a/.sqlx/query-4f244b199b76c28e71bc9fcefe076be8dd99822c3688c90548db5331703599a7.json b/.sqlx/query-4f244b199b76c28e71bc9fcefe076be8dd99822c3688c90548db5331703599a7.json new file mode 100644 index 0000000..9325f32 --- /dev/null +++ b/.sqlx/query-4f244b199b76c28e71bc9fcefe076be8dd99822c3688c90548db5331703599a7.json @@ -0,0 +1,56 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id, payload, queued_at, started_at, finished_at, retries, error_msg\n FROM tasks \n WHERE status = 'Queued'\n ORDER BY queued_at ASC\n FOR UPDATE SKIP LOCKED\n LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "payload", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "queued_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 3, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "finished_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "retries", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "error_msg", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false, + true, + true, + false, + true + ] + }, + "hash": "4f244b199b76c28e71bc9fcefe076be8dd99822c3688c90548db5331703599a7" +} diff --git a/.sqlx/query-926755a98d0ad62a460df7d1416af9951dad4dc06727d5367a727dfe303b5abd.json b/.sqlx/query-926755a98d0ad62a460df7d1416af9951dad4dc06727d5367a727dfe303b5abd.json new file mode 100644 index 0000000..06882ae --- /dev/null +++ b/.sqlx/query-926755a98d0ad62a460df7d1416af9951dad4dc06727d5367a727dfe303b5abd.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO tasks (id, payload, status, queued_at)\n VALUES ($1, $2, 'Queued', $3)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Jsonb", + "Timestamptz" + ] + }, + "nullable": [] + }, + "hash": "926755a98d0ad62a460df7d1416af9951dad4dc06727d5367a727dfe303b5abd" +} diff --git a/.sqlx/query-9797aa5f3d4d9be57c64856375d16990531cb1bfd55292b6f0fcb99a5efc2147.json b/.sqlx/query-9797aa5f3d4d9be57c64856375d16990531cb1bfd55292b6f0fcb99a5efc2147.json new file mode 100644 index 0000000..1dacea5 --- /dev/null +++ b/.sqlx/query-9797aa5f3d4d9be57c64856375d16990531cb1bfd55292b6f0fcb99a5efc2147.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE tasks \n SET status = 'InProgress', started_at = $1\n WHERE id = $2\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Timestamptz", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "9797aa5f3d4d9be57c64856375d16990531cb1bfd55292b6f0fcb99a5efc2147" +} diff --git a/.sqlx/query-e3e6dc7484e8e5602020b2c4f2445c3d0d149dccfae0e40f60dc907d968d1369.json b/.sqlx/query-e3e6dc7484e8e5602020b2c4f2445c3d0d149dccfae0e40f60dc907d968d1369.json new file mode 100644 index 0000000..6330b35 --- /dev/null +++ b/.sqlx/query-e3e6dc7484e8e5602020b2c4f2445c3d0d149dccfae0e40f60dc907d968d1369.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dlq (id, payload, error_msg) VALUES ($1, $2, $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Jsonb", + "Text" + ] + }, + "nullable": [] + }, + "hash": "e3e6dc7484e8e5602020b2c4f2445c3d0d149dccfae0e40f60dc907d968d1369" +} diff --git a/Cargo.toml b/Cargo.toml index 2d64030..183bf8a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ version = "0.4.5" edition = "2021" license = "MIT" +rust-version = "1.80" authors = ["oiwn"] repository = "https://github.com/oiwn/capp-rs" diff --git a/capp-queue/src/backend/postgres.rs b/capp-queue/src/backend/postgres.rs index 042093f..a0be36f 100644 --- a/capp-queue/src/backend/postgres.rs +++ b/capp-queue/src/backend/postgres.rs @@ -44,7 +44,7 @@ where pub async fn new(connection_string: &str) -> Result { let pool = PgPool::connect(connection_string) .await - .map_err(|e| TaskQueueError::PostgresError(e))?; + .map_err(TaskQueueError::PostgresError)?; Ok(Self { pool, @@ -236,8 +236,8 @@ where ) .bind(payload) .bind(status) // Now we're binding a proper Postgres enum type - .bind(task.started_at.map(|t| DateTime::::from(t))) - .bind(task.finished_at.map(|t| DateTime::::from(t))) + .bind(task.started_at.map(DateTime::::from)) + .bind(task.finished_at.map(DateTime::::from)) .bind(task.retries as i32) .bind(&task.error_msg) .bind(task.task_id.get()) diff --git a/capp-queue/tests/postgres_tests.rs b/capp-queue/tests/postgres_tests.rs index a2a7abc..56a2b89 100644 --- a/capp-queue/tests/postgres_tests.rs +++ b/capp-queue/tests/postgres_tests.rs @@ -1,4 +1,4 @@ -#[cfg(test)] +#[cfg(all(test, unix, feature = "postgres"))] mod tests { use capp_queue::{ JsonSerializer, PostgresTaskQueue, Task, TaskQueue, TaskQueueError, diff --git a/capp/Cargo.toml b/capp/Cargo.toml index c504256..0629d9d 100644 --- a/capp/Cargo.toml +++ b/capp/Cargo.toml @@ -61,7 +61,7 @@ rand = "0.8" md5 = "0.7" url = "2.5" base64 = "0.22" -tempfile = "3" +# tempfile = "3" [features] http = ["dep:reqwest", "capp-config/http"]