diff --git a/.gitignore b/.gitignore index acf3384..55af379 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ # archives for test_data/ *.tar.gz *.sqlite3 +*.sqlite *.dump *.bin test_data/archives/ diff --git a/Cargo.lock b/Cargo.lock index 099f06c..b23fcbe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -439,6 +439,17 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.4.0" @@ -973,6 +984,23 @@ dependencies = [ "libloading", ] +[[package]] +name = "clap" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "atty", + "bitflags 1.3.2", + "clap_derive 3.2.25", + "clap_lex 0.2.4", + "indexmap 1.9.3", + "once_cell", + "strsim 0.10.0", + "termcolor", + "textwrap", +] + [[package]] name = "clap" version = "4.5.37" @@ -980,7 +1008,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eccb054f56cbd38340b380d4a8e69ef1f02f1af43db2f0cc817a4774d80ae071" dependencies = [ "clap_builder", - "clap_derive", + "clap_derive 4.5.32", ] [[package]] @@ -991,8 +1019,21 @@ checksum = "efd9466fac8543255d3b1fcad4762c5e116ffe808c8a3043d4263cd4fd4862a2" dependencies = [ "anstream", "anstyle", - "clap_lex", - "strsim", + "clap_lex 0.7.4", + "strsim 0.11.1", +] + +[[package]] +name = "clap_derive" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" +dependencies = [ + "heck 0.4.1", + "proc-macro-error", + "proc-macro2 1.0.95", + "quote", + "syn 1.0.109", ] [[package]] @@ -1007,6 +1048,15 @@ dependencies = [ "syn 2.0.101", ] +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "clap_lex" version = "0.7.4" @@ -1451,7 +1501,7 @@ dependencies = [ "ident_case", "proc-macro2 1.0.95", "quote", - "strsim", + "strsim 0.11.1", "syn 2.0.101", ] @@ -1534,6 +1584,20 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "decaf377-fmd" +version = "1.3.1" +source = "git+https://github.com/penumbra-zone/penumbra?rev=e5afc365d00d48c2fe7d0136481b45ffafe6b953#e5afc365d00d48c2fe7d0136481b45ffafe6b953" +dependencies = [ + "ark-ff", + "ark-serialize", + "bitvec", + "blake2b_simd 1.0.3", + "decaf377", + "rand_core 0.6.4", + "thiserror 1.0.69", +] + [[package]] name = "decaf377-fmd" version = "1.3.2" @@ -1618,6 +1682,20 @@ dependencies = [ "zeroize_derive", ] +[[package]] +name = "decaf377-ka" +version = "1.3.1" +source = "git+https://github.com/penumbra-zone/penumbra?rev=e5afc365d00d48c2fe7d0136481b45ffafe6b953#e5afc365d00d48c2fe7d0136481b45ffafe6b953" +dependencies = [ + "ark-ff", + "decaf377", + "hex", + "rand_core 0.6.4", + "thiserror 1.0.69", + "zeroize", + "zeroize_derive", +] + [[package]] name = "decaf377-ka" version = "1.3.2" @@ -2369,6 +2447,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hermit-abi" version = "0.3.9" @@ -4051,7 +4138,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", ] @@ -4126,6 +4213,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + [[package]] name = "overload" version = "0.1.1" @@ -6088,7 +6181,7 @@ dependencies = [ "assert_cmd", "async-stream", "async-trait", - "clap", + "clap 4.5.37", "cnidarium 0.79.7", "cnidarium 0.80.13", "cnidarium 0.81.3", @@ -6127,6 +6220,7 @@ dependencies = [ "penumbra-sdk-transaction 2.0.0-alpha.11", "penumbra-transaction 0.80.13", "penumbra-transaction 0.81.3", + "picturesque", "prost 0.13.5", "reqwest 0.12.15", "serde_json", @@ -6474,6 +6568,45 @@ dependencies = [ "url", ] +[[package]] +name = "penumbra-sdk-asset" +version = "1.3.1" +source = "git+https://github.com/penumbra-zone/penumbra?rev=e5afc365d00d48c2fe7d0136481b45ffafe6b953#e5afc365d00d48c2fe7d0136481b45ffafe6b953" +dependencies = [ + "anyhow", + "ark-ff", + "ark-r1cs-std", + "ark-relations", + "ark-serialize", + "ark-std", + "base64 0.21.7", + "bech32", + "blake2b_simd 1.0.3", + "bytes", + "decaf377", + "decaf377-fmd 1.3.1", + "decaf377-rdsa", + "derivative", + "ethnum", + "getrandom 0.2.16", + "hex", + "ibig", + "num-bigint", + "once_cell", + "pbjson-types 0.7.0", + "penumbra-sdk-num 1.3.1", + "penumbra-sdk-proto 1.3.1", + "poseidon377", + "rand 0.8.5", + "rand_core 0.6.4", + "regex", + "serde", + "serde_with", + "sha2 0.10.9", + "thiserror 1.0.69", + "tracing", +] + [[package]] name = "penumbra-sdk-asset" version = "1.3.2" @@ -7621,6 +7754,50 @@ dependencies = [ "tracing", ] +[[package]] +name = "penumbra-sdk-keys" +version = "1.3.1" +source = "git+https://github.com/penumbra-zone/penumbra?rev=e5afc365d00d48c2fe7d0136481b45ffafe6b953#e5afc365d00d48c2fe7d0136481b45ffafe6b953" +dependencies = [ + "aes", + "anyhow", + "ark-ff", + "ark-r1cs-std", + "ark-relations", + "ark-serialize", + "ark-std", + "base64 0.21.7", + "bech32", + "bip32", + "blake2b_simd 1.0.3", + "bytes", + "chacha20poly1305", + "decaf377", + "decaf377-fmd 1.3.1", + "decaf377-ka 1.3.1", + "decaf377-rdsa", + "derivative", + "ethnum", + "f4jumble 0.1.1", + "hex", + "hmac", + "ibig", + "num-bigint", + "once_cell", + "pbkdf2", + "penumbra-sdk-asset 1.3.1", + "penumbra-sdk-proto 1.3.1", + "penumbra-sdk-tct 1.3.1", + "poseidon377", + "rand 0.8.5", + "rand_core 0.6.4", + "regex", + "serde", + "sha2 0.10.9", + "thiserror 1.0.69", + "tracing", +] + [[package]] name = "penumbra-sdk-keys" version = "1.3.2" @@ -7753,6 +7930,42 @@ dependencies = [ "tracing", ] +[[package]] +name = "penumbra-sdk-num" +version = "1.3.1" +source = "git+https://github.com/penumbra-zone/penumbra?rev=e5afc365d00d48c2fe7d0136481b45ffafe6b953#e5afc365d00d48c2fe7d0136481b45ffafe6b953" +dependencies = [ + "anyhow", + "ark-ff", + "ark-groth16", + "ark-r1cs-std", + "ark-relations", + "ark-serialize", + "ark-snark", + "ark-std", + "base64 0.21.7", + "bech32", + "blake2b_simd 1.0.3", + "bytes", + "decaf377", + "decaf377-fmd 1.3.1", + "decaf377-rdsa", + "derivative", + "ethnum", + "hex", + "ibig", + "num-bigint", + "once_cell", + "penumbra-sdk-proto 1.3.1", + "rand 0.8.5", + "rand_core 0.6.4", + "regex", + "serde", + "sha2 0.10.9", + "thiserror 1.0.69", + "tracing", +] + [[package]] name = "penumbra-sdk-num" version = "1.3.2" @@ -7936,6 +8149,33 @@ dependencies = [ "tracing", ] +[[package]] +name = "penumbra-sdk-proto" +version = "1.3.1" +source = "git+https://github.com/penumbra-zone/penumbra?rev=e5afc365d00d48c2fe7d0136481b45ffafe6b953#e5afc365d00d48c2fe7d0136481b45ffafe6b953" +dependencies = [ + "anyhow", + "async-trait", + "bech32", + "bytes", + "decaf377-fmd 1.3.1", + "decaf377-rdsa", + "futures", + "hex", + "ibc-proto 0.51.1", + "ibc-types 0.15.1", + "ics23 0.12.0", + "pbjson 0.7.0", + "pbjson-types 0.7.0", + "pin-project", + "prost 0.13.5", + "serde", + "serde_json", + "subtle-encoding", + "tendermint 0.40.3", + "tracing", +] + [[package]] name = "penumbra-sdk-proto" version = "1.3.2" @@ -8444,6 +8684,35 @@ dependencies = [ "tracing", ] +[[package]] +name = "penumbra-sdk-tct" +version = "1.3.1" +source = "git+https://github.com/penumbra-zone/penumbra?rev=e5afc365d00d48c2fe7d0136481b45ffafe6b953#e5afc365d00d48c2fe7d0136481b45ffafe6b953" +dependencies = [ + "ark-ed-on-bls12-377", + "ark-ff", + "ark-r1cs-std", + "ark-relations", + "ark-serialize", + "async-trait", + "blake2b_simd 1.0.3", + "decaf377", + "derivative", + "futures", + "getrandom 0.2.16", + "hash_hasher", + "hex", + "im", + "once_cell", + "parking_lot", + "penumbra-sdk-proto 1.3.1", + "poseidon377", + "rand 0.8.5", + "serde", + "thiserror 1.0.69", + "tracing", +] + [[package]] name = "penumbra-sdk-tct" version = "1.3.2" @@ -9544,6 +9813,20 @@ dependencies = [ "indexmap 2.9.0", ] +[[package]] +name = "picturesque" +version = "1.3.1" +source = "git+https://github.com/penumbra-zone/penumbra?rev=e5afc365d00d48c2fe7d0136481b45ffafe6b953#e5afc365d00d48c2fe7d0136481b45ffafe6b953" +dependencies = [ + "anyhow", + "clap 3.2.25", + "penumbra-sdk-keys 1.3.1", + "tokio", + "toml 0.7.8", + "tracing", + "tracing-subscriber 0.3.19", +] + [[package]] name = "pin-project" version = "1.1.10" @@ -9732,7 +10015,31 @@ version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" dependencies = [ - "toml_edit", + "toml_edit 0.22.26", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2 1.0.95", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2 1.0.95", + "quote", + "version_check", ] [[package]] @@ -11021,6 +11328,12 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "strsim" version = "0.11.1" @@ -11331,12 +11644,27 @@ dependencies = [ "walkdir", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "termtree" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" +[[package]] +name = "textwrap" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" + [[package]] name = "thiserror" version = "1.0.69" @@ -11560,6 +11888,18 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit 0.19.15", +] + [[package]] name = "toml" version = "0.8.22" @@ -11569,7 +11909,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit", + "toml_edit 0.22.26", ] [[package]] @@ -11581,6 +11921,19 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_edit" +version = "0.19.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +dependencies = [ + "indexmap 2.9.0", + "serde", + "serde_spanned", + "toml_datetime", + "winnow 0.5.40", +] + [[package]] name = "toml_edit" version = "0.22.26" @@ -11592,7 +11945,7 @@ dependencies = [ "serde_spanned", "toml_datetime", "toml_write", - "winnow", + "winnow 0.7.8", ] [[package]] @@ -12558,6 +12911,15 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" +[[package]] +name = "winnow" +version = "0.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" +dependencies = [ + "memchr", +] + [[package]] name = "winnow" version = "0.7.8" diff --git a/Cargo.toml b/Cargo.toml index 3917b94..349bafe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -110,6 +110,8 @@ opt-level = 3 assert_cmd = "2.0.16" escargot = "0.5.13" flate2 = "1.0.35" +# picturesque = { git = "https://github.com/penumbra-zone/penumbra", branch = "conorsch/picturesque" } +picturesque = { git = "https://github.com/penumbra-zone/penumbra", rev = "e5afc365d00d48c2fe7d0136481b45ffafe6b953" } reqwest = { version = "0.12.12", features = ["json", "stream"] } tar = "0.4.43" tokio-stream = "0.1.17" diff --git a/flake.lock b/flake.lock index 34743bf..10e9266 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "crane": { "locked": { - "lastModified": 1725409566, - "narHash": "sha256-PrtLmqhM6UtJP7v7IGyzjBFhbG4eOAHT6LPYOFmYfbk=", + "lastModified": 1744386647, + "narHash": "sha256-DXwQEJllxpYeVOiSlBhQuGjfvkoGHTtILLYO2FvcyzQ=", "owner": "ipetkov", "repo": "crane", - "rev": "7e4586bad4e3f8f97a9271def747cf58c4b68f3c", + "rev": "d02c1cdd7ec539699aa44e6ff912e15535969803", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 4ecb38e..f2b03bc 100644 --- a/flake.nix +++ b/flake.nix @@ -105,7 +105,6 @@ # Build the `penumbra-reindexer` binary penumbraReindexer = (craneLib.buildPackage { pname = "penumbra-reindexer"; - # what src = cleanSourceWith { src = if penumbraReindexerRelease == null then craneLib.path ./. else fetchFromGitHub { owner = "penumbra-zone"; @@ -144,7 +143,11 @@ homepage = "https://penumbra.zone"; license = [ licenses.mit licenses.asl20 ]; }; - }).overrideAttrs (_: { doCheck = false; }); # Disable tests to improve build times + + # Skip running `cargo check`, because doing so with a git dep in the dev-dependencies + # causes the nix build to fail. + doCheck = false; + }); # Container image for shipping the reindexer. diff --git a/justfile b/justfile index 3881e6a..7a6f011 100644 --- a/justfile +++ b/justfile @@ -1,3 +1,7 @@ +# Run network integration tests, specifically the 'penumbra-reindexer regen' ones. +integration-regen: + RUST_LOG=debug cargo nextest run --release --features network-integration --nocapture run_reindexer_regen_ + # Run cargo check, failing on warnings check: cargo check --all-targets --all-features @@ -13,6 +17,7 @@ test: # Run network integration tests. Requires a LOT of disk space and bandwidth! integration: + rm -rf /tmp/penumbra-reindexer-regen-1/ rm -rf test_data/ephemeral-storage/ cargo nextest run --release --features network-integration --nocapture @@ -34,3 +39,10 @@ container: # docker load < result # docker run -it localhost/penumbra-reindexer:0.5.0 bash # +# Serve the local postgres db, created via integration tests +db: + printf 'Use the following URL to connect to the db:\n\n\t%s\n\n' \ + "postgresql://?dbname=penumbra_raw&host=/tmp/penumbra-reindexer-regen-1/pgtown/postgres/sock" + postgres \ + -D /tmp/penumbra-reindexer-regen-1/pgtown/postgres/data/ \ + -k /tmp/penumbra-reindexer-regen-1/pgtown/postgres/sock/ diff --git a/src/penumbra.rs b/src/penumbra.rs index 238914d..720158d 100644 --- a/src/penumbra.rs +++ b/src/penumbra.rs @@ -591,6 +591,9 @@ impl Regenerator { last_block.map(|x| x.to_string()).unwrap_or("∞".to_string()) ); for height in first_block..=end { + if height % 1000 == 0 { + tracing::info!("reached height {}", height); + } let block: Block = self .archive .get_block(height) diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 2c14313..55eca22 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -7,11 +7,13 @@ use anyhow::Context; use assert_cmd::Command; use flate2::read::GzDecoder; use sha2::{Digest, Sha256}; +use sqlx::postgres::PgPool; use sqlx::sqlite::SqlitePool; use sqlx::{Error, FromRow, Row}; use std::io::Write; use std::path::Path; use std::path::PathBuf; +use tokio::task::JoinHandle; use tokio_stream::StreamExt; use url::Url; @@ -87,7 +89,7 @@ impl ReindexerTestRunner { } /// Sets up the integration test suite with required local archive data. - pub async fn prepare_local_workbench(&self, step: usize) -> anyhow::Result<()> { + pub async fn prepare_local_workbench_archive(&self, step: usize) -> anyhow::Result<()> { // If we're starting a reindex, then we should clear out the dirs. if step == 0 { if self.network_dir.exists() { @@ -109,6 +111,26 @@ impl ReindexerTestRunner { Ok(()) } + /// Prepares up the integration test suite for `regen` actions. + /// Removes the regen working dir, so that the pre-existing node0 directories + /// will be replayed. + pub async fn prepare_local_workbench_regen(&self, step: usize) -> anyhow::Result<()> { + if step == 0 { + if self.regen_working_dir().exists() { + tracing::debug!( + "removing working_dir {}", + &self.regen_working_dir().display() + ); + std::fs::remove_dir_all(self.regen_working_dir())?; + } + if self.pg_dir().exists() { + tracing::debug!("removing pg_dir {}", &self.pg_dir().display()); + std::fs::remove_dir_all(self.pg_dir())?; + } + } + Ok(()) + } + /// Prebuilt `penumbra-reindexer` command. pub async fn cmd(&self) -> anyhow::Result { tracing::debug!("building reindexer for tests"); @@ -143,11 +165,40 @@ impl ReindexerTestRunner { /// Query the sqlite3 database for any missing blocks, defined as `BlockGap`s, /// and fail if any are found. - pub async fn check_for_gaps(&self) -> anyhow::Result<()> { + pub async fn check_for_gaps_sqlite(&self) -> anyhow::Result<()> { // Connect to the database let pool = SqlitePool::connect(self.reindexer_db_filepath().to_str().unwrap()).await?; - let query = sqlx::query_as::<_, BlockGap>( + let sql = self.gaps_query(); + let query = sqlx::query_as::<_, BlockGap>(&sql); + let results = query.fetch_all(&pool).await?; + + // TODO: read fields to format an error message + assert!(results.is_empty(), "found missing blocks in the sqlite3 db"); + Ok(()) + } + + /// Query the postgres database for any missing blocks, defined as `BlockGap`s, + /// and fail if any are found. + pub async fn check_for_gaps_postgres(&self) -> anyhow::Result<()> { + // Connect to the database + let pool = PgPool::connect(self.pg_db_url().as_str()).await?; + + let sql = self.gaps_query(); + let query = sqlx::query_as::<_, BlockGap>(&sql); + let results = query.fetch_all(&pool).await?; + + // TODO: read fields to format an error message + assert!( + results.is_empty(), + "found missing blocks in the postgres db" + ); + Ok(()) + } + + /// Private function for generating SQL that checks for gaps within a database. + fn gaps_query(&self) -> String { + String::from( r#" WITH numbered_blocks AS ( SELECT height, @@ -158,19 +209,14 @@ impl ReindexerTestRunner { FROM numbered_blocks WHERE next_height - height > 1 "#, - ); - let results = query.fetch_all(&pool).await?; - - // TODO: read fields to format an error message - assert!(results.is_empty(), "found missing blocks in the sqlite3 db"); - Ok(()) + ) } /// Query the sqlite3 database for total number of known blocks. /// Fail if it doesn't match the expected number of blocks, or /// 1 less than the expected number. The tolerance is to acknowledge /// that the sqlite3 db can be 1 block behind the local node state. - pub async fn check_num_blocks(&self, expected: u64) -> anyhow::Result { + pub async fn check_num_blocks_sqlite(&self, expected: u64) -> anyhow::Result { // Connect to the database let pool = SqlitePool::connect(self.reindexer_db_filepath().to_str().unwrap()).await?; let query = sqlx::query("SELECT COUNT(*) FROM blocks"); @@ -178,8 +224,27 @@ impl ReindexerTestRunner { assert!( [expected, expected - 1].contains(&count), "archived blocks count looks wrong; expected: {}, found {}", + expected, + count, + ); + Ok(count) + } + + /// Query the postgres database for total number of known blocks. + /// Fail if it doesn't match the expected number of blocks, or + /// 1 less than the expected number. The tolerance is to acknowledge + /// that the postgres db can be 1 block behind the local node state. + pub async fn check_num_blocks_postgres(&self, expected: u64) -> anyhow::Result { + // Connect to the database + let pool = PgPool::connect(self.pg_db_url().as_str()).await?; + let query = sqlx::query("SELECT COUNT(*) FROM blocks"); + let count_raw: i64 = query.fetch_one(&pool).await?.get(0); + let count = count_raw as u64; + assert!( + [expected, expected - 1].contains(&count), + "regenerated blocks count looks wrong; expected: {}, found {}", + expected, count, - expected ); Ok(count) } @@ -190,7 +255,55 @@ impl ReindexerTestRunner { self.network_dir.join("node0") } - /// Run `reindexer-archive` against the [node_dir]. + /// Look up the data directory for postgresql, by appending `postgresql` + /// to the `node_dir`. Also creates the directory to ensure it exists. + pub fn pg_dir(&self) -> PathBuf { + // UDS paths must be absolute, but may not be longer than 107 bytes on Linux. + // Depending on where the git checkout for this repo lives, + // that's not very many dirs to work with. Let's create a symlink in a + // hardcoded dirname inside /tmp to ensure a short-enough path. + // + // THIS IS DANGEROUS: any user on the system that can write to that path can manipulate the + // socket. Also, the hardcoded path means that only one copy of these integration tests + // can be run per host, but that's a reasonable assumption. + // + // A more durable implementation would create a tempdir, and symlink an inner dir within + // that tempdir to the real postgres directory where it lives. That way, the socket path + // for psql can reference the shorter path to the tempdir, and stay under the char limit. + + // let p = PathBuf::from("/tmp/penumbra-reindexer-regen-1"); + let p = PathBuf::from("/tmp/penumbra-reindexer-regen-1").join("pgtown"); + std::fs::create_dir_all(&p).expect("failed to create pg dir"); + assert!( + p.display().to_string().as_bytes().len() <= 107, + "postgres data directory path is too long!" + ); + p + } + + /// Run a local-only PostgreSQL server, over a Unix domain socket, + /// so that `regen` operations can target a database. Returns a [JoinHandle] + /// on the spawned db process, which can be dropped to terminate the server. + /// Callers must ensure that no competing processes exist, based out of the same directory. + pub async fn run_postgres(&self) -> anyhow::Result>> { + let pg_dir = self.pg_dir(); + tracing::debug!("running postgres"); + let pg_handle = tokio::spawn(picturesque::postgres::run(pg_dir)); + tracing::debug!("sleeping a bit to let postgres server start"); + let _delay = tokio::time::sleep(std::time::Duration::from_secs(15)).await; + Ok(pg_handle) + } + + /// Return a connection URL for the PostgreSQL database used for `regen` commands, + /// formatted for use with `psql`. + pub fn pg_db_url(&self) -> String { + format!( + "postgresql://?dbname=penumbra_raw&host={}/postgres/sock", + self.pg_dir().display() + ) + } + + /// Run `penumbra-reindexer archive` against the [node_dir]. /// /// Will block until all available blocks have been archived, or else error. pub async fn archive(&self) -> anyhow::Result<()> { @@ -204,6 +317,37 @@ impl ReindexerTestRunner { .status()?; Ok(()) } + + // Private function for building a path to the `--working-dir` + // for the `regen` command. This is basically another copy of the pd rocksdb + // directory, used as a scratchpad while iterating through the sqlite3 db. + fn regen_working_dir(&self) -> PathBuf { + self.node_dir().join("regen-working-dir-1") + } + + /// Run `penumbra-reindexer regen` against the [node_dir]. + /// + /// Will block until all available blocks have been archived, or else error. + pub async fn regen(&self, stop_height: Option) -> anyhow::Result<()> { + let working_dir = self.regen_working_dir(); + let mut args: Vec = vec![ + "regen".to_owned(), + "--database-url".to_owned(), + self.pg_db_url(), + // The path to the sqlite3 db will be inferred from the `--home` arg, + // so we don't need to pass `--archive-file`. + "--home".to_owned(), + self.node_dir().display().to_string(), + "--working-dir".to_owned(), + working_dir.display().to_string(), + ]; + if let Some(h) = stop_height { + args.push("--stop-height".to_owned()); + args.push(h.to_string()); + } + let _result = self.cmd().await?.command().args(args).status()?; + Ok(()) + } } /// Set up [tracing_subscriber], so that tests can emit logging information. @@ -242,7 +386,7 @@ pub struct BlockGap { gap_end: i64, } -/// Ensure that we can query the sqlite3 and receive BlockGap results. +/// Ensure that we can query the sqlite3 db and receive BlockGap results. impl<'r> FromRow<'r, sqlx::sqlite::SqliteRow> for BlockGap { fn from_row(row: &'r sqlx::sqlite::SqliteRow) -> Result { Ok(BlockGap { @@ -252,8 +396,18 @@ impl<'r> FromRow<'r, sqlx::sqlite::SqliteRow> for BlockGap { } } +/// Ensure that we can query the postgres db and receive BlockGap results. +impl<'r> FromRow<'r, sqlx::postgres::PgRow> for BlockGap { + fn from_row(row: &'r sqlx::postgres::PgRow) -> Result { + Ok(BlockGap { + gap_start: row.try_get("start_block")?, // if column is named differently + gap_end: row.try_get("end_block")?, + }) + } +} + #[tracing::instrument] -/// Reusable function to handle running `penumbra-reindexer` archive +/// Reusable function to handle running `penumbra-reindexer archive` /// for a given network. The `step` value indicates which serial /// protocol compatibility period the `archive` run is in, indexed from 0 /// being the original network genesis. @@ -272,16 +426,52 @@ pub async fn run_reindexer_archive_step( network_dir: PathBuf::from(NETWORK_DIR).join(chain_id), }; - test_runner.prepare_local_workbench(step).await?; + test_runner.prepare_local_workbench_archive(step).await?; tracing::info!("running reindexer archive step {}", step); test_runner.archive().await?; - test_runner.check_for_gaps().await?; - test_runner.check_num_blocks(expected_blocks).await?; + test_runner.check_for_gaps_sqlite().await?; + test_runner.check_num_blocks_sqlite(expected_blocks).await?; test_runner.check_num_geneses(step).await?; Ok(()) } +#[tracing::instrument] +/// Reusable function to handle running `penumbra-reindexer regen` +/// for a given network. The `step` value indicates which serial +/// protocol compatibility period the `regen` run is in, indexed from 0 +/// being the original network genesis. +pub async fn run_reindexer_regen_step( + chain_id: &str, + step: usize, + stop_height: Option, +) -> anyhow::Result<()> { + // Set up logging + crate::common::init_tracing(); + + // Initialize testbed. + let test_runner = ReindexerTestRunner { + chain_id: chain_id.to_owned(), + // Append chain id to network dir to disambiguate local paths. + network_dir: PathBuf::from(NETWORK_DIR).join(chain_id), + }; + + // Run the workbench prep, specifically for regen operations. + test_runner.prepare_local_workbench_regen(step).await?; + + tracing::debug!("starting postgres server"); + let _pg = test_runner.run_postgres().await?; + + tracing::info!("running reindexer regen step {}", step); + test_runner.regen(stop_height).await?; + test_runner.check_for_gaps_postgres().await?; + if let Some(h) = stop_height { + tracing::info!(?h, "checking for number of blocks in pg db"); + test_runner.check_num_blocks_postgres(h).await?; + } + Ok(()) +} + /// `pd/rocksdb` and `cometbft/data` directories for a /// A complete set of node state archives, constituting /// node, representing each protocol version, segmented @@ -459,6 +649,30 @@ impl HistoricalArchiveSeries { }) } + /// List all sequential node state archives required + /// to reconstruct chain state for `penumbra-testnet-phobos-3`. + pub fn for_penumbra_testnet_phobos_3() -> anyhow::Result { + let chain_id = "penumbra-testnet-phobos-3".to_owned(); + let dest_dir = PathBuf::from(format!( + "{}/{}/{}", + env!("CARGO_MANIFEST_DIR"), + ARCHIVE_DIR, + chain_id, + )); + let archives: Vec = vec![HistoricalArchive { + download_url: "https://artifacts.plinfra.net/penumbra-testnet-phobos-3/penumbra-node-archive-height-368331.tar.gz".try_into()?, + checksum_sha256: "53b449e99f0663f1c46dcb50f61f53eae6c2892eb740d41e6d0ed068c3eb62fc" + .to_owned(), + chain_id: chain_id.clone(), + dest_dir: dest_dir.clone(), + }]; + + Ok(HistoricalArchiveSeries { + chain_id: chain_id.to_owned(), + archives, + }) + } + /// List all sequential node state archives required /// to reconstruct chain state for `penumbra-1`. pub fn for_penumbra_1() -> anyhow::Result { @@ -487,8 +701,15 @@ impl HistoricalArchiveSeries { }, HistoricalArchive { - download_url: "https://artifacts.plinfra.net/penumbra-1/penumbra-node-archive-height-4027443.tar.gz".try_into()?, - checksum_sha256: "cfb93391ae348275b221bb1811d59833b4bc2854c92c234fe266506b4a6b7c71".to_owned(), + download_url: "https://artifacts.plinfra.net/penumbra-1/penumbra-node-archive-height-4378762-pre-upgrade.tar.gz".try_into()?, + checksum_sha256: "9840c4d0c93a928412fc55faa6edfe69faa19aac662cc133d6a45c64d1e0062c".to_owned(), + chain_id: chain_id.clone(), + dest_dir: dest_dir.clone(), + }, + + HistoricalArchive { + download_url: "https://artifacts.plinfra.net/penumbra-1/penumbra-node-archive-height-4836782.tar.gz".try_into()?, + checksum_sha256: "ffce4cfc5d783f0fc06645c4049b7affb8207b70e68012c9b33b46d108cdf996".to_owned(), chain_id: chain_id.clone(), dest_dir: dest_dir.clone(), }, diff --git a/tests/penumbra_1.rs b/tests/penumbra_1.rs index 7519333..6b2fe8d 100644 --- a/tests/penumbra_1.rs +++ b/tests/penumbra_1.rs @@ -5,6 +5,7 @@ #[path = "common/mod.rs"] mod common; use crate::common::run_reindexer_archive_step; +use crate::common::run_reindexer_regen_step; /// The chain id for the network being reindexed. const PENUMBRA_CHAIN_ID: &str = "penumbra-1"; @@ -26,9 +27,49 @@ async fn run_reindexer_archive_step_2() -> anyhow::Result<()> { } #[tokio::test] -/// Run `penumbra-reindexer archive` from the second upgrade boundary to the present. +/// Run `penumbra-reindexer archive` from the second upgrade boundary to the third. async fn run_reindexer_archive_step_3() -> anyhow::Result<()> { - let expected_blocks = 4027443; + let expected_blocks = 4378762; run_reindexer_archive_step(PENUMBRA_CHAIN_ID, 2, expected_blocks).await?; Ok(()) } + +#[tokio::test] +/// Run `penumbra-reindexer archive` from the third upgrade boundary to the present. +async fn run_reindexer_archive_step_4() -> anyhow::Result<()> { + let expected_blocks = 4836782; + run_reindexer_archive_step(PENUMBRA_CHAIN_ID, 3, expected_blocks).await?; + Ok(()) +} + +#[tokio::test] +/// Run `penumbra-reindexer regen` from block 0 to the first upgrade boundary. +async fn run_reindexer_regen_step_1() -> anyhow::Result<()> { + let stop_height = Some(501974); + run_reindexer_regen_step(PENUMBRA_CHAIN_ID, 0, stop_height).await?; + Ok(()) +} + +#[tokio::test] +/// Run `penumbra-reindexer regen` from the first upgrade boundary to the second. +async fn run_reindexer_regen_step_2() -> anyhow::Result<()> { + let stop_height = Some(2611800); + run_reindexer_regen_step(PENUMBRA_CHAIN_ID, 1, stop_height).await?; + Ok(()) +} + +#[tokio::test] +/// Run `penumbra-reindexer regen` from the second upgrade boundary to the third. +async fn run_reindexer_regen_step_3() -> anyhow::Result<()> { + let stop_height = Some(4378762); + run_reindexer_regen_step(PENUMBRA_CHAIN_ID, 2, stop_height).await?; + Ok(()) +} + +#[tokio::test] +/// Run `penumbra-reindexer regen` from the third upgrade boundary to the present. +async fn run_reindexer_regen_step_4() -> anyhow::Result<()> { + let stop_height = None; + run_reindexer_regen_step(PENUMBRA_CHAIN_ID, 1, stop_height).await?; + Ok(()) +} diff --git a/tests/penumbra_testnet_phobos_2.rs b/tests/penumbra_testnet_phobos_2.rs index f1fbe81..2b8acc6 100644 --- a/tests/penumbra_testnet_phobos_2.rs +++ b/tests/penumbra_testnet_phobos_2.rs @@ -3,7 +3,7 @@ //! testnet, identified by chain id `penumbra-testnet-phobos-2`. #[path = "common/mod.rs"] mod common; -use crate::common::run_reindexer_archive_step; +use crate::common::{run_reindexer_archive_step, run_reindexer_regen_step}; /// The chain id for the network being reindexed. const PENUMBRA_CHAIN_ID: &str = "penumbra-testnet-phobos-2"; @@ -31,3 +31,26 @@ async fn run_reindexer_archive_step_3() -> anyhow::Result<()> { run_reindexer_archive_step(PENUMBRA_CHAIN_ID, 2, expected_blocks).await?; Ok(()) } + +#[tokio::test] +/// Run `penumbra-reindexer regen` from block 0 to the first upgrade boundary. +async fn run_reindexer_regen_step_1() -> anyhow::Result<()> { + let stop_height = Some(1459800); + run_reindexer_regen_step(PENUMBRA_CHAIN_ID, 0, stop_height).await?; + Ok(()) +} + +#[tokio::test] +/// Run `penumbra-reindexer regen` from the first upgrade boundary to the second. +async fn run_reindexer_regen_step_2() -> anyhow::Result<()> { + let stop_height = Some(2358329); + run_reindexer_regen_step(PENUMBRA_CHAIN_ID, 1, stop_height).await?; + Ok(()) +} +#[tokio::test] +/// Run `penumbra-reindexer regen` from the first upgrade boundary to the second. +async fn run_reindexer_regen_step_3() -> anyhow::Result<()> { + let stop_height = None; + run_reindexer_regen_step(PENUMBRA_CHAIN_ID, 2, stop_height).await?; + Ok(()) +} diff --git a/tests/penumbra_testnet_phobos_3.rs b/tests/penumbra_testnet_phobos_3.rs new file mode 100644 index 0000000..1aafa45 --- /dev/null +++ b/tests/penumbra_testnet_phobos_3.rs @@ -0,0 +1,25 @@ +#![cfg(feature = "network-integration")] +//! These integration tests operate on historical events for the public Penumbra Labs +//! testnet, identified by chain id `penumbra-testnet-phobos-3`. +#[path = "common/mod.rs"] +mod common; +use crate::common::{run_reindexer_archive_step, run_reindexer_regen_step}; + +/// The chain id for the network being reindexed. +const PENUMBRA_CHAIN_ID: &str = "penumbra-testnet-phobos-3"; + +#[tokio::test] +/// Run `penumbra-reindexer archive` from block 0 to present. +async fn run_reindexer_archive_step_1() -> anyhow::Result<()> { + let expected_blocks = 368331; + run_reindexer_archive_step(PENUMBRA_CHAIN_ID, 0, expected_blocks).await?; + Ok(()) +} + +#[tokio::test] +/// Run `penumbra-reindexer regen` from block 0 to the present. +async fn run_reindexer_regen_step_1() -> anyhow::Result<()> { + let stop_height = None; + run_reindexer_regen_step(PENUMBRA_CHAIN_ID, 0, stop_height).await?; + Ok(()) +} diff --git a/tools/regen.sh b/tools/regen.sh new file mode 100755 index 0000000..779c5e6 --- /dev/null +++ b/tools/regen.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -euo pipefail + + +archive_file="test_data/ephemeral-storage/network/penumbra-1/node0/reindexer_archive.bin" +working_dir="mainnet-lqt-reindex-2" + + +penumbra-reindexer regen \ + --database-url 'postgresql://penumbra:penumbra@127.0.0.1:5432/penumbra' \ + --archive-file "$archive_file" \ + --working-dir "$working_dir" \ + --stop-height 501974 + +penumbra-reindexer regen \ + --database-url 'postgresql://penumbra:penumbra@127.0.0.1:5432/penumbra' \ + --archive-file "$archive_file" \ + --working-dir "$working_dir" \ + --stop-height 2611799 + +penumbra-reindexer regen \ + --database-url 'postgresql://penumbra:penumbra@127.0.0.1:5432/penumbra' \ + --archive-file "$archive_file" \ + --working-dir "$working_dir" \ + --stop-height 4378761 + +penumbra-reindexer regen \ + --database-url 'postgresql://penumbra:penumbra@127.0.0.1:5432/penumbra' \ + --archive-file "$archive_file" \ + --working-dir "$working_dir"