From 69c1e6815d449af48b6bfbeac4ba2beaf5875a1f Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Thu, 21 Mar 2024 15:33:44 -0300 Subject: [PATCH 01/16] create a very basic `put_blocks` function for the memory wallet --- Cargo.lock | 7 +- Cargo.toml | 3 +- zcash_client_backend/src/data_api.rs | 4 +- .../src/data_api/mem_wallet.rs | 72 ++++++++++++++++--- zcash_client_backend/src/wallet.rs | 3 + 5 files changed, 75 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d056207c98..d0d905d66a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2105,7 +2105,7 @@ dependencies = [ [[package]] name = "sapling-crypto" version = "0.1.2" -source = "git+https://github.com/nuttycom/sapling-crypto?rev=827534329f3ff61f9212f7933ed1688fa04e987c#827534329f3ff61f9212f7933ed1688fa04e987c" +source = "git+https://github.com/zcash/sapling-crypto?rev=54fc7d3d9c8eeb8e6ef6f64619deed43b0b681a6#54fc7d3d9c8eeb8e6ef6f64619deed43b0b681a6" dependencies = [ "aes", "bellman", @@ -3306,3 +3306,8 @@ dependencies = [ "memuse", "subtle", ] + +[[patch.unused]] +name = "sapling-crypto" +version = "0.1.2" +source = "git+https://github.com/nuttycom/sapling-crypto?rev=827534329f3ff61f9212f7933ed1688fa04e987c#827534329f3ff61f9212f7933ed1688fa04e987c" diff --git a/Cargo.toml b/Cargo.toml index 3b91948b87..5c70fd56e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,7 +52,8 @@ bitvec = "1" blake2s_simd = "1" bls12_381 = "0.8" jubjub = "0.10" -sapling = { package = "sapling-crypto", version = "0.1.2" } +#sapling = { package = "sapling-crypto", version = "0.1.2" } +sapling = { package = "sapling-crypto", git = "https://github.com/zcash/sapling-crypto", rev = "54fc7d3d9c8eeb8e6ef6f64619deed43b0b681a6" } # - Orchard nonempty = "0.7" diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index f408347bc7..9763b764f3 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -992,7 +992,7 @@ pub struct ScannedBundles { } impl ScannedBundles { - pub(crate) fn new( + pub fn new( final_tree_size: u32, commitments: Vec<(NoteCommitment, Retention)>, nullifier_map: Vec<(TxId, u16, Vec)>, @@ -1053,7 +1053,7 @@ pub struct ScannedBlock { impl ScannedBlock { /// Constructs a new `ScannedBlock` - pub(crate) fn from_parts( + pub fn from_parts( block_height: BlockHeight, block_hash: BlockHash, block_time: u32, diff --git a/zcash_client_backend/src/data_api/mem_wallet.rs b/zcash_client_backend/src/data_api/mem_wallet.rs index c754fdf7bf..e662290787 100644 --- a/zcash_client_backend/src/data_api/mem_wallet.rs +++ b/zcash_client_backend/src/data_api/mem_wallet.rs @@ -6,10 +6,11 @@ use std::{ cmp::Ordering, collections::{BTreeMap, HashMap, HashSet}, convert::Infallible, - num::NonZeroU32, hash::Hash, + hash::Hash, + num::NonZeroU32, }; use zcash_keys::keys::{AddressGenerationError, DerivationError}; -use zip32::{fingerprint::SeedFingerprint, DiversifierIndex}; +use zip32::{fingerprint::SeedFingerprint, DiversifierIndex, Scope}; use zcash_primitives::{ block::BlockHash, @@ -25,7 +26,7 @@ use zcash_protocol::{ use crate::{ address::UnifiedAddress, keys::{UnifiedAddressRequest, UnifiedFullViewingKey, UnifiedSpendingKey}, - wallet::{NoteId, WalletTransparentOutput, WalletTx}, + wallet::{NoteId, WalletSpend, WalletTransparentOutput, WalletTx}, }; use super::{ @@ -100,10 +101,7 @@ pub struct MemoryWalletDb { } impl MemoryWalletDb { - pub fn new( - network: Network, - max_checkpoints: usize - ) -> Self { + pub fn new(network: Network, max_checkpoints: usize) -> Self { Self { network, accounts: BTreeMap::new(), @@ -122,6 +120,7 @@ impl MemoryWalletDb { #[derive(Debug)] pub enum Error { AccountUnknown(u32), + ViewingKeyNotFound(u32), MemoDecryption(memo::Error), KeyDerivation(DerivationError), AddressGeneration(AddressGenerationError), @@ -236,7 +235,10 @@ impl WalletRead for MemoryWalletDb { } fn chain_height(&self) -> Result, Self::Error> { - todo!() + match self.blocks.last_key_value() { + Some((last_key, _)) => Ok(Some(*last_key)), + None => Ok(None), + } } fn get_block_hash(&self, block_height: BlockHeight) -> Result, Self::Error> { @@ -379,12 +381,62 @@ impl WalletWrite for MemoryWalletDb { todo!() } + /// Adds a sequence of blocks to the data store. + /// + /// Assumes blocks will be here in order. fn put_blocks( &mut self, + // TODO: Figure out what to do with this field. _from_state: &super::chain::ChainState, - _blocks: Vec>, + blocks: Vec>, ) -> Result<(), Self::Error> { - todo!() + // TODO: + // - Make sure blocks are coming in order. + // - Make sure the first block in the sequence is tip + 1? + // - Add a check to make sure the blocks are not already in the data store. + for block in blocks.into_iter() { + let mut transactions = HashMap::new(); + for transaction in block.transactions().into_iter().cloned() { + let txid = transaction.txid(); + let account_id = 0; // TODO: Assuming the account is 0, handle this accordingly. + let ufvk = self + .accounts + .get(&account_id) + .ok_or(Error::AccountUnknown(0))? + .ufvk + .sapling() + .ok_or(Error::ViewingKeyNotFound(0))?; + let nk = ufvk.to_nk(Scope::External); + + let spent_nullifiers = transaction + .sapling_outputs() + .iter() + .map(|o| { + let nullifier = o.note().nf(&nk, o.note_commitment_tree_position().into()); + // TODO: Populate the bool field properly. + self.sapling_spends.entry(nullifier).or_insert((txid, true)); + nullifier + }) + .count(); + + // Is `self.tx_idx` field filled with all the transaction ids from the scanned blocks ? + self.tx_idx.insert(txid, block.block_height); + transactions.insert(txid, transaction); + } + + let memory_block = MemoryWalletBlock { + height: block.block_height, + hash: block.block_hash, + block_time: block.block_time, + transactions, + // TODO: Add memos + memos: HashMap::new(), + }; + + self.blocks.insert(block.block_height, memory_block); + } + + Ok(()) } /// Adds a transparent UTXO received by the wallet to the data store. diff --git a/zcash_client_backend/src/wallet.rs b/zcash_client_backend/src/wallet.rs index 133ed234f5..4b90efe1a6 100644 --- a/zcash_client_backend/src/wallet.rs +++ b/zcash_client_backend/src/wallet.rs @@ -98,6 +98,7 @@ impl Recipient> { /// The shielded subset of a [`Transaction`]'s data that is relevant to a particular wallet. /// /// [`Transaction`]: zcash_primitives::transaction::Transaction +#[derive(Clone)] pub struct WalletTx { txid: TxId, block_index: usize, @@ -227,6 +228,7 @@ impl transparent_fees::InputView for WalletTransparentOutput { } /// A reference to a spent note belonging to the wallet within a transaction. +#[derive(Clone)] pub struct WalletSpend { index: usize, nf: Nf, @@ -266,6 +268,7 @@ pub type WalletSaplingSpend = WalletSpend = WalletSpend; /// An output that was successfully decrypted in the process of wallet scanning. +#[derive(Clone)] pub struct WalletOutput { index: usize, ephemeral_key: EphemeralKeyBytes, From 9c0c6b079a6d4554c241a0479ad440b2a0d9f235 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Fri, 22 Mar 2024 11:15:13 -0300 Subject: [PATCH 02/16] append sapling noie commitments to database --- zcash_client_backend/src/data_api/mem_wallet.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/zcash_client_backend/src/data_api/mem_wallet.rs b/zcash_client_backend/src/data_api/mem_wallet.rs index e662290787..72b9d28605 100644 --- a/zcash_client_backend/src/data_api/mem_wallet.rs +++ b/zcash_client_backend/src/data_api/mem_wallet.rs @@ -419,6 +419,8 @@ impl WalletWrite for MemoryWalletDb { }) .count(); + // TODO: Add orchard nullifiers to the orchard spends. + // Is `self.tx_idx` field filled with all the transaction ids from the scanned blocks ? self.tx_idx.insert(txid, block.block_height); transactions.insert(txid, transaction); @@ -434,6 +436,18 @@ impl WalletWrite for MemoryWalletDb { }; self.blocks.insert(block.block_height, memory_block); + + // Add the sapling commitments to the sapling tree. + let sapling_block_commitments = block.into_commitments().sapling; + sapling_block_commitments.iter().map(|(node, height)| { + self.sapling_tree.append(*node, *height); + }); + + // TODO: Add orchard commitments to the orchard tree. + + // TODO: Received notes need to be made available for note selection & balance calculation + + // TODO: Spent notes need to be made unavailable for note selection & balance calculation } Ok(()) From 4d3c441c9aca486f2ee80c5c3f66a7cad7418e36 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 2 Apr 2024 14:29:39 -0300 Subject: [PATCH 03/16] remove non needed pubs --- zcash_client_backend/src/data_api.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 9763b764f3..f408347bc7 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -992,7 +992,7 @@ pub struct ScannedBundles { } impl ScannedBundles { - pub fn new( + pub(crate) fn new( final_tree_size: u32, commitments: Vec<(NoteCommitment, Retention)>, nullifier_map: Vec<(TxId, u16, Vec)>, @@ -1053,7 +1053,7 @@ pub struct ScannedBlock { impl ScannedBlock { /// Constructs a new `ScannedBlock` - pub fn from_parts( + pub(crate) fn from_parts( block_height: BlockHeight, block_hash: BlockHash, block_time: u32, From 7f746212e1b13e79c4ed41b1707d84c9816a9bef Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 2 Apr 2024 14:29:58 -0300 Subject: [PATCH 04/16] remove chain_height code --- zcash_client_backend/src/data_api/mem_wallet.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/zcash_client_backend/src/data_api/mem_wallet.rs b/zcash_client_backend/src/data_api/mem_wallet.rs index 72b9d28605..c6d7f6c3e0 100644 --- a/zcash_client_backend/src/data_api/mem_wallet.rs +++ b/zcash_client_backend/src/data_api/mem_wallet.rs @@ -235,10 +235,7 @@ impl WalletRead for MemoryWalletDb { } fn chain_height(&self) -> Result, Self::Error> { - match self.blocks.last_key_value() { - Some((last_key, _)) => Ok(Some(*last_key)), - None => Ok(None), - } + todo!() } fn get_block_hash(&self, block_height: BlockHeight) -> Result, Self::Error> { From 0393a69af5725f7750f5f37305a838f9e3445274 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 2 Apr 2024 15:57:32 -0300 Subject: [PATCH 05/16] add orchard spends --- .../src/data_api/mem_wallet.rs | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/zcash_client_backend/src/data_api/mem_wallet.rs b/zcash_client_backend/src/data_api/mem_wallet.rs index c6d7f6c3e0..e49c9e6315 100644 --- a/zcash_client_backend/src/data_api/mem_wallet.rs +++ b/zcash_client_backend/src/data_api/mem_wallet.rs @@ -405,20 +405,24 @@ impl WalletWrite for MemoryWalletDb { .ok_or(Error::ViewingKeyNotFound(0))?; let nk = ufvk.to_nk(Scope::External); - let spent_nullifiers = transaction - .sapling_outputs() - .iter() - .map(|o| { - let nullifier = o.note().nf(&nk, o.note_commitment_tree_position().into()); - // TODO: Populate the bool field properly. - self.sapling_spends.entry(nullifier).or_insert((txid, true)); - nullifier - }) - .count(); - - // TODO: Add orchard nullifiers to the orchard spends. - - // Is `self.tx_idx` field filled with all the transaction ids from the scanned blocks ? + // Insert the Sapling nullifiers of the spent notes into the `sapling_spends` map. + transaction.sapling_outputs().iter().map(|o| { + let nullifier = o.note().nf(&nk, o.note_commitment_tree_position().into()); + // TODO: Populate the bool field properly. + self.sapling_spends.entry(nullifier).or_insert((txid, true)); + }); + + #[cfg(feature = "orchard")] + // Insert the Orchard nullifiers of the spent notes into the `orchard_spends` map. + transaction.orchard_outputs().iter().map(|o| { + if let Some(nullifier) = o.nf() { + self.orchard_spends + .entry(*nullifier) + .or_insert((txid, true)); + } + }); + + // TODO: Is `self.tx_idx` field filled with all the transaction ids from the scanned blocks ? self.tx_idx.insert(txid, block.block_height); transactions.insert(txid, transaction); } From a4323c2d60d6696ad867bed63183206a1a874c47 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 2 Apr 2024 16:31:55 -0300 Subject: [PATCH 06/16] add sapling and orchard trees --- .../src/data_api/mem_wallet.rs | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/zcash_client_backend/src/data_api/mem_wallet.rs b/zcash_client_backend/src/data_api/mem_wallet.rs index e49c9e6315..fea7af5708 100644 --- a/zcash_client_backend/src/data_api/mem_wallet.rs +++ b/zcash_client_backend/src/data_api/mem_wallet.rs @@ -1,5 +1,5 @@ #![allow(unused)] -use incrementalmerkletree::Address; +use incrementalmerkletree::{Address, Retention}; use secrecy::{ExposeSecret, SecretVec}; use shardtree::{error::ShardTreeError, store::memory::MemoryShardStore, ShardTree}; use std::{ @@ -383,8 +383,7 @@ impl WalletWrite for MemoryWalletDb { /// Assumes blocks will be here in order. fn put_blocks( &mut self, - // TODO: Figure out what to do with this field. - _from_state: &super::chain::ChainState, + from_state: &super::chain::ChainState, blocks: Vec>, ) -> Result<(), Self::Error> { // TODO: @@ -422,6 +421,25 @@ impl WalletWrite for MemoryWalletDb { } }); + // Add frontier to the sapling tree + self.sapling_tree.insert_frontier( + from_state.final_sapling_tree().clone(), + Retention::Checkpoint { + id: from_state.block_height(), + is_marked: false, + }, + ); + + #[cfg(feature = "orchard")] + // Add frontier to the orchard tree + self.orchard_tree.insert_frontier( + from_state.final_orchard_tree().clone(), + Retention::Checkpoint { + id: from_state.block_height(), + is_marked: false, + }, + ); + // TODO: Is `self.tx_idx` field filled with all the transaction ids from the scanned blocks ? self.tx_idx.insert(txid, block.block_height); transactions.insert(txid, transaction); From a3b4de0459e2ee443385b7a062910644db6ac400 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 2 Apr 2024 16:40:13 -0300 Subject: [PATCH 07/16] use sapling-crypto v0.1.3 --- Cargo.lock | 5 +++-- Cargo.toml | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d0d905d66a..dfa1e9df95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2104,8 +2104,9 @@ dependencies = [ [[package]] name = "sapling-crypto" -version = "0.1.2" -source = "git+https://github.com/zcash/sapling-crypto?rev=54fc7d3d9c8eeb8e6ef6f64619deed43b0b681a6#54fc7d3d9c8eeb8e6ef6f64619deed43b0b681a6" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02f4270033afcb0c74c5c7d59c73cfd1040367f67f224fe7ed9a919ae618f1b7" dependencies = [ "aes", "bellman", diff --git a/Cargo.toml b/Cargo.toml index 5c70fd56e7..d12328bc67 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,8 +52,7 @@ bitvec = "1" blake2s_simd = "1" bls12_381 = "0.8" jubjub = "0.10" -#sapling = { package = "sapling-crypto", version = "0.1.2" } -sapling = { package = "sapling-crypto", git = "https://github.com/zcash/sapling-crypto", rev = "54fc7d3d9c8eeb8e6ef6f64619deed43b0b681a6" } +sapling = { package = "sapling-crypto", version = "0.1.3" } # - Orchard nonempty = "0.7" From 370a4aa7e522faca1b3bc56ae9c7087fa7a6ac91 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 2 Apr 2024 17:29:41 -0300 Subject: [PATCH 08/16] populate orchard note commitment trees --- zcash_client_backend/src/data_api/mem_wallet.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/zcash_client_backend/src/data_api/mem_wallet.rs b/zcash_client_backend/src/data_api/mem_wallet.rs index fea7af5708..47a8a5db8a 100644 --- a/zcash_client_backend/src/data_api/mem_wallet.rs +++ b/zcash_client_backend/src/data_api/mem_wallet.rs @@ -456,13 +456,21 @@ impl WalletWrite for MemoryWalletDb { self.blocks.insert(block.block_height, memory_block); - // Add the sapling commitments to the sapling tree. - let sapling_block_commitments = block.into_commitments().sapling; + // Add the Sapling commitments to the sapling tree. + let block_commitments = block.into_commitments(); + let sapling_block_commitments = block_commitments.sapling; sapling_block_commitments.iter().map(|(node, height)| { self.sapling_tree.append(*node, *height); }); - // TODO: Add orchard commitments to the orchard tree. + #[cfg(feature = "orchard")] + { + // Add the Orchard commitments to the sapling tree. + let orchard_block_commitments = block_commitments.orchard; + orchard_block_commitments.iter().map(|(node, height)| { + self.orchard_tree.append(*node, *height); + }); + } // TODO: Received notes need to be made available for note selection & balance calculation From b1a0fcc6972f4c72bb4f34f74a4aa4b06e346970 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 2 Apr 2024 20:09:11 -0300 Subject: [PATCH 09/16] use batch_insert --- .../src/data_api/mem_wallet.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/zcash_client_backend/src/data_api/mem_wallet.rs b/zcash_client_backend/src/data_api/mem_wallet.rs index 47a8a5db8a..3a317ecde5 100644 --- a/zcash_client_backend/src/data_api/mem_wallet.rs +++ b/zcash_client_backend/src/data_api/mem_wallet.rs @@ -458,18 +458,16 @@ impl WalletWrite for MemoryWalletDb { // Add the Sapling commitments to the sapling tree. let block_commitments = block.into_commitments(); - let sapling_block_commitments = block_commitments.sapling; - sapling_block_commitments.iter().map(|(node, height)| { - self.sapling_tree.append(*node, *height); - }); + if let Ok(Some(pos)) = self.sapling_tree.max_leaf_position(0) { + self.sapling_tree + .batch_insert(pos, block_commitments.sapling.into_iter()); + } #[cfg(feature = "orchard")] - { - // Add the Orchard commitments to the sapling tree. - let orchard_block_commitments = block_commitments.orchard; - orchard_block_commitments.iter().map(|(node, height)| { - self.orchard_tree.append(*node, *height); - }); + // Add the Orchard commitments to the orchard tree. + if let Ok(Some(pos)) = self.orchard_tree.max_leaf_position(0) { + self.orchard_tree + .batch_insert(pos, block_commitments.orchard.into_iter()); } // TODO: Received notes need to be made available for note selection & balance calculation From 23a2ae8fc697381ed24054c2e0e4a57a724da978 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 3 Apr 2024 11:54:53 -0300 Subject: [PATCH 10/16] add the memos --- .../src/data_api/mem_wallet.rs | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/zcash_client_backend/src/data_api/mem_wallet.rs b/zcash_client_backend/src/data_api/mem_wallet.rs index 3a317ecde5..09841cdf2f 100644 --- a/zcash_client_backend/src/data_api/mem_wallet.rs +++ b/zcash_client_backend/src/data_api/mem_wallet.rs @@ -21,6 +21,7 @@ use zcash_primitives::{ use zcash_protocol::{ memo::{self, Memo, MemoBytes}, value::Zatoshis, + ShieldedProtocol::{Orchard, Sapling}, }; use crate::{ @@ -392,6 +393,7 @@ impl WalletWrite for MemoryWalletDb { // - Add a check to make sure the blocks are not already in the data store. for block in blocks.into_iter() { let mut transactions = HashMap::new(); + let mut memos = HashMap::new(); for transaction in block.transactions().into_iter().cloned() { let txid = transaction.txid(); let account_id = 0; // TODO: Assuming the account is 0, handle this accordingly. @@ -404,21 +406,41 @@ impl WalletWrite for MemoryWalletDb { .ok_or(Error::ViewingKeyNotFound(0))?; let nk = ufvk.to_nk(Scope::External); - // Insert the Sapling nullifiers of the spent notes into the `sapling_spends` map. transaction.sapling_outputs().iter().map(|o| { + // Insert the Sapling nullifiers of the spent notes into the `sapling_spends` map. let nullifier = o.note().nf(&nk, o.note_commitment_tree_position().into()); // TODO: Populate the bool field properly. self.sapling_spends.entry(nullifier).or_insert((txid, true)); + + // Insert the memo into the `memos` map. + let note_id = NoteId::new( + txid, + Sapling, + u16::try_from(o.index()).expect("output indices are representable as u16"), + ); + if let Ok(Some(memo)) = self.get_memo(note_id) { + memos.insert(note_id, memo.encode()); + } }); #[cfg(feature = "orchard")] - // Insert the Orchard nullifiers of the spent notes into the `orchard_spends` map. transaction.orchard_outputs().iter().map(|o| { + // Insert the Orchard nullifiers of the spent notes into the `orchard_spends` map. if let Some(nullifier) = o.nf() { self.orchard_spends .entry(*nullifier) .or_insert((txid, true)); } + + // Insert the memo into the `memos` map. + let note_id = NoteId::new( + txid, + Orchard, + u16::try_from(o.index()).expect("output indices are representable as u16"), + ); + if let Ok(Some(memo)) = self.get_memo(note_id) { + memos.insert(note_id, memo.encode()); + } }); // Add frontier to the sapling tree @@ -450,8 +472,7 @@ impl WalletWrite for MemoryWalletDb { hash: block.block_hash, block_time: block.block_time, transactions, - // TODO: Add memos - memos: HashMap::new(), + memos, }; self.blocks.insert(block.block_height, memory_block); From 2f28306c6183453c533b8971ae12960a389f755e Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 3 Apr 2024 14:05:53 -0300 Subject: [PATCH 11/16] mark spent notes --- .../src/data_api/mem_wallet.rs | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/zcash_client_backend/src/data_api/mem_wallet.rs b/zcash_client_backend/src/data_api/mem_wallet.rs index 09841cdf2f..4212d32e40 100644 --- a/zcash_client_backend/src/data_api/mem_wallet.rs +++ b/zcash_client_backend/src/data_api/mem_wallet.rs @@ -409,8 +409,9 @@ impl WalletWrite for MemoryWalletDb { transaction.sapling_outputs().iter().map(|o| { // Insert the Sapling nullifiers of the spent notes into the `sapling_spends` map. let nullifier = o.note().nf(&nk, o.note_commitment_tree_position().into()); - // TODO: Populate the bool field properly. - self.sapling_spends.entry(nullifier).or_insert((txid, true)); + self.sapling_spends + .entry(nullifier) + .or_insert((txid, false)); // Insert the memo into the `memos` map. let note_id = NoteId::new( @@ -429,7 +430,7 @@ impl WalletWrite for MemoryWalletDb { if let Some(nullifier) = o.nf() { self.orchard_spends .entry(*nullifier) - .or_insert((txid, true)); + .or_insert((txid, false)); } // Insert the memo into the `memos` map. @@ -462,6 +463,23 @@ impl WalletWrite for MemoryWalletDb { }, ); + // Mark the Sapling nullifiers of the spent notes as spent in the `sapling_spends` map. + transaction.sapling_spends().iter().map(|s| { + let nullifier = s.nf(); + if let Some((txid, spent)) = self.sapling_spends.get_mut(&nullifier) { + *spent = true; + } + }); + + #[cfg(feature = "orchard")] + // Mark the Orchard nullifiers of the spent notes as spent in the `orchard_spends` map. + transaction.orchard_spends().iter().map(|s| { + let nullifier = s.nf(); + if let Some((txid, spent)) = self.orchard_spends.get_mut(&nullifier) { + *spent = true; + } + }); + // TODO: Is `self.tx_idx` field filled with all the transaction ids from the scanned blocks ? self.tx_idx.insert(txid, block.block_height); transactions.insert(txid, transaction); @@ -490,10 +508,6 @@ impl WalletWrite for MemoryWalletDb { self.orchard_tree .batch_insert(pos, block_commitments.orchard.into_iter()); } - - // TODO: Received notes need to be made available for note selection & balance calculation - - // TODO: Spent notes need to be made unavailable for note selection & balance calculation } Ok(()) From 588156f2632f9a91be8f1bd67f856882780db112 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 3 Apr 2024 14:37:12 -0300 Subject: [PATCH 12/16] fix note commitment tree starting position --- zcash_client_backend/src/data_api/mem_wallet.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zcash_client_backend/src/data_api/mem_wallet.rs b/zcash_client_backend/src/data_api/mem_wallet.rs index 4212d32e40..a0fcb16f2b 100644 --- a/zcash_client_backend/src/data_api/mem_wallet.rs +++ b/zcash_client_backend/src/data_api/mem_wallet.rs @@ -497,16 +497,16 @@ impl WalletWrite for MemoryWalletDb { // Add the Sapling commitments to the sapling tree. let block_commitments = block.into_commitments(); - if let Ok(Some(pos)) = self.sapling_tree.max_leaf_position(0) { + if let Some(value) = from_state.final_sapling_tree().value() { self.sapling_tree - .batch_insert(pos, block_commitments.sapling.into_iter()); + .batch_insert(value.position(), block_commitments.sapling.into_iter()); } #[cfg(feature = "orchard")] // Add the Orchard commitments to the orchard tree. - if let Ok(Some(pos)) = self.orchard_tree.max_leaf_position(0) { + if let Some(value) = from_state.final_orchard_tree().value() { self.orchard_tree - .batch_insert(pos, block_commitments.orchard.into_iter()); + .batch_insert(value.position(), block_commitments.orchard.into_iter()); } } From f77dfc1ee33120ec916122ac9bf12e2bade36785 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 9 Apr 2024 14:17:41 -0300 Subject: [PATCH 13/16] use all accounts in the wallet --- .../src/data_api/mem_wallet.rs | 178 +++++++++--------- 1 file changed, 92 insertions(+), 86 deletions(-) diff --git a/zcash_client_backend/src/data_api/mem_wallet.rs b/zcash_client_backend/src/data_api/mem_wallet.rs index a0fcb16f2b..c7c2c4e513 100644 --- a/zcash_client_backend/src/data_api/mem_wallet.rs +++ b/zcash_client_backend/src/data_api/mem_wallet.rs @@ -1,5 +1,6 @@ #![allow(unused)] use incrementalmerkletree::{Address, Retention}; +use sapling::NullifierDerivingKey; use secrecy::{ExposeSecret, SecretVec}; use shardtree::{error::ShardTreeError, store::memory::MemoryShardStore, ShardTree}; use std::{ @@ -31,7 +32,7 @@ use crate::{ }; use super::{ - chain::CommitmentTreeRoot, scanning::ScanRange, AccountBirthday, BlockMetadata, + chain::CommitmentTreeRoot, scanning::ScanRange, Account, AccountBirthday, BlockMetadata, DecryptedTransaction, NullifierQuery, ScannedBlock, SentTransaction, WalletCommitmentTrees, WalletRead, WalletSummary, WalletWrite, SAPLING_SHARD_HEIGHT, }; @@ -394,95 +395,100 @@ impl WalletWrite for MemoryWalletDb { for block in blocks.into_iter() { let mut transactions = HashMap::new(); let mut memos = HashMap::new(); - for transaction in block.transactions().into_iter().cloned() { + for transaction in block.transactions().iter() { let txid = transaction.txid(); - let account_id = 0; // TODO: Assuming the account is 0, handle this accordingly. - let ufvk = self - .accounts - .get(&account_id) - .ok_or(Error::AccountUnknown(0))? - .ufvk - .sapling() - .ok_or(Error::ViewingKeyNotFound(0))?; - let nk = ufvk.to_nk(Scope::External); - - transaction.sapling_outputs().iter().map(|o| { - // Insert the Sapling nullifiers of the spent notes into the `sapling_spends` map. - let nullifier = o.note().nf(&nk, o.note_commitment_tree_position().into()); - self.sapling_spends - .entry(nullifier) - .or_insert((txid, false)); - - // Insert the memo into the `memos` map. - let note_id = NoteId::new( - txid, - Sapling, - u16::try_from(o.index()).expect("output indices are representable as u16"), - ); - if let Ok(Some(memo)) = self.get_memo(note_id) { - memos.insert(note_id, memo.encode()); - } - }); - - #[cfg(feature = "orchard")] - transaction.orchard_outputs().iter().map(|o| { - // Insert the Orchard nullifiers of the spent notes into the `orchard_spends` map. - if let Some(nullifier) = o.nf() { - self.orchard_spends - .entry(*nullifier) + for account_id in self.get_account_ids()? { + let ufvk = self + .get_account(account_id)? + .ok_or(Error::AccountUnknown(account_id))? + .ufvk() + .ok_or(Error::ViewingKeyNotFound(account_id))? + .clone(); + let dfvk = ufvk + .sapling() + .ok_or(Error::ViewingKeyNotFound(account_id))?; + let nk = dfvk.to_nk(Scope::External); + + transaction.sapling_outputs().iter().map(|o| { + // Insert the Sapling nullifiers of the spent notes into the `sapling_spends` map. + let nullifier = o.note().nf(&nk, o.note_commitment_tree_position().into()); + self.sapling_spends + .entry(nullifier) .or_insert((txid, false)); - } - // Insert the memo into the `memos` map. - let note_id = NoteId::new( - txid, - Orchard, - u16::try_from(o.index()).expect("output indices are representable as u16"), + // Insert the memo into the `memos` map. + let note_id = NoteId::new( + txid, + Sapling, + u16::try_from(o.index()) + .expect("output indices are representable as u16"), + ); + if let Ok(Some(memo)) = self.get_memo(note_id) { + memos.insert(note_id, memo.encode()); + } + }); + + #[cfg(feature = "orchard")] + transaction.orchard_outputs().iter().map(|o| { + // Insert the Orchard nullifiers of the spent notes into the `orchard_spends` map. + if let Some(nullifier) = o.nf() { + self.orchard_spends + .entry(*nullifier) + .or_insert((txid, false)); + } + + // Insert the memo into the `memos` map. + let note_id = NoteId::new( + txid, + Orchard, + u16::try_from(o.index()) + .expect("output indices are representable as u16"), + ); + if let Ok(Some(memo)) = self.get_memo(note_id) { + memos.insert(note_id, memo.encode()); + } + }); + + // Add frontier to the sapling tree + self.sapling_tree.insert_frontier( + from_state.final_sapling_tree().clone(), + Retention::Checkpoint { + id: from_state.block_height(), + is_marked: false, + }, ); - if let Ok(Some(memo)) = self.get_memo(note_id) { - memos.insert(note_id, memo.encode()); - } - }); - - // Add frontier to the sapling tree - self.sapling_tree.insert_frontier( - from_state.final_sapling_tree().clone(), - Retention::Checkpoint { - id: from_state.block_height(), - is_marked: false, - }, - ); - - #[cfg(feature = "orchard")] - // Add frontier to the orchard tree - self.orchard_tree.insert_frontier( - from_state.final_orchard_tree().clone(), - Retention::Checkpoint { - id: from_state.block_height(), - is_marked: false, - }, - ); - - // Mark the Sapling nullifiers of the spent notes as spent in the `sapling_spends` map. - transaction.sapling_spends().iter().map(|s| { - let nullifier = s.nf(); - if let Some((txid, spent)) = self.sapling_spends.get_mut(&nullifier) { - *spent = true; - } - }); - - #[cfg(feature = "orchard")] - // Mark the Orchard nullifiers of the spent notes as spent in the `orchard_spends` map. - transaction.orchard_spends().iter().map(|s| { - let nullifier = s.nf(); - if let Some((txid, spent)) = self.orchard_spends.get_mut(&nullifier) { - *spent = true; - } - }); - - // TODO: Is `self.tx_idx` field filled with all the transaction ids from the scanned blocks ? - self.tx_idx.insert(txid, block.block_height); - transactions.insert(txid, transaction); + + #[cfg(feature = "orchard")] + // Add frontier to the orchard tree + self.orchard_tree.insert_frontier( + from_state.final_orchard_tree().clone(), + Retention::Checkpoint { + id: from_state.block_height(), + is_marked: false, + }, + ); + + // Mark the Sapling nullifiers of the spent notes as spent in the `sapling_spends` map. + transaction.sapling_spends().iter().map(|s| { + let nullifier = s.nf(); + if let Some((txid, spent)) = self.sapling_spends.get_mut(&nullifier) { + *spent = true; + } + }); + + #[cfg(feature = "orchard")] + // Mark the Orchard nullifiers of the spent notes as spent in the `orchard_spends` map. + transaction.orchard_spends().iter().map(|s| { + let nullifier = s.nf(); + if let Some((txid, spent)) = self.orchard_spends.get_mut(&nullifier) { + *spent = true; + } + }); + + // TODO: Is `self.tx_idx` field filled with all the transaction ids from the scanned blocks ? + self.tx_idx.insert(txid, block.block_height); + transactions.insert(txid, transaction.clone()); + } } let memory_block = MemoryWalletBlock { From 147c033f6aa9a1d5f98206b62ebcb227ab0495e8 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 9 Apr 2024 16:33:13 -0300 Subject: [PATCH 14/16] remove todo comment --- zcash_client_backend/src/data_api/mem_wallet.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/zcash_client_backend/src/data_api/mem_wallet.rs b/zcash_client_backend/src/data_api/mem_wallet.rs index c7c2c4e513..4d8d259a3d 100644 --- a/zcash_client_backend/src/data_api/mem_wallet.rs +++ b/zcash_client_backend/src/data_api/mem_wallet.rs @@ -485,7 +485,6 @@ impl WalletWrite for MemoryWalletDb { } }); - // TODO: Is `self.tx_idx` field filled with all the transaction ids from the scanned blocks ? self.tx_idx.insert(txid, block.block_height); transactions.insert(txid, transaction.clone()); } From cbb49b4686bd62a3180d85a39d62849e361065f0 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Thu, 11 Apr 2024 18:17:05 -0300 Subject: [PATCH 15/16] fix incorrect start note commitment position Co-authored-by: Kris Nuttycombe --- .../src/data_api/mem_wallet.rs | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/zcash_client_backend/src/data_api/mem_wallet.rs b/zcash_client_backend/src/data_api/mem_wallet.rs index 4d8d259a3d..aae43272c2 100644 --- a/zcash_client_backend/src/data_api/mem_wallet.rs +++ b/zcash_client_backend/src/data_api/mem_wallet.rs @@ -502,16 +502,22 @@ impl WalletWrite for MemoryWalletDb { // Add the Sapling commitments to the sapling tree. let block_commitments = block.into_commitments(); - if let Some(value) = from_state.final_sapling_tree().value() { - self.sapling_tree - .batch_insert(value.position(), block_commitments.sapling.into_iter()); - } + let start_position = from_state + .final_sapling_tree() + .value() + .map_or(0.into(), |t| t.position() + 1); + self.sapling_tree + .batch_insert(start_position, block_commitments.sapling.into_iter()); #[cfg(feature = "orchard")] - // Add the Orchard commitments to the orchard tree. - if let Some(value) = from_state.final_orchard_tree().value() { + { + // Add the Orchard commitments to the orchard tree. + let start_position = from_state + .final_orchard_tree() + .value() + .map_or(0.into(), |t| t.position() + 1); self.orchard_tree - .batch_insert(value.position(), block_commitments.orchard.into_iter()); + .batch_insert(start_position, block_commitments.orchard.into_iter()); } } From c9a236e90ef16cebdcebcba2566613f42da9236f Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 16 Apr 2024 17:07:02 -0300 Subject: [PATCH 16/16] clippy --- zcash_client_backend/src/data_api/mem_wallet.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zcash_client_backend/src/data_api/mem_wallet.rs b/zcash_client_backend/src/data_api/mem_wallet.rs index aae43272c2..d7d4a5295d 100644 --- a/zcash_client_backend/src/data_api/mem_wallet.rs +++ b/zcash_client_backend/src/data_api/mem_wallet.rs @@ -471,7 +471,7 @@ impl WalletWrite for MemoryWalletDb { // Mark the Sapling nullifiers of the spent notes as spent in the `sapling_spends` map. transaction.sapling_spends().iter().map(|s| { let nullifier = s.nf(); - if let Some((txid, spent)) = self.sapling_spends.get_mut(&nullifier) { + if let Some((txid, spent)) = self.sapling_spends.get_mut(nullifier) { *spent = true; } }); @@ -480,7 +480,7 @@ impl WalletWrite for MemoryWalletDb { // Mark the Orchard nullifiers of the spent notes as spent in the `orchard_spends` map. transaction.orchard_spends().iter().map(|s| { let nullifier = s.nf(); - if let Some((txid, spent)) = self.orchard_spends.get_mut(&nullifier) { + if let Some((txid, spent)) = self.orchard_spends.get_mut(nullifier) { *spent = true; } });