From 3c03fd0cceed66a44996176517d345f88923edf3 Mon Sep 17 00:00:00 2001 From: imotai Date: Tue, 13 Jun 2023 23:13:53 +0800 Subject: [PATCH 1/4] feat: add doc store --- Cargo.toml | 1 + src/proto/proto/db3_database_v2.proto | 108 ++++---------------------- src/proto/proto/db3_mutation_v2.proto | 2 +- src/storage/Cargo.toml | 2 + src/storage/src/db_store_v2.rs | 4 +- src/storage/src/lib.rs | 1 + 6 files changed, 20 insertions(+), 98 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a1aef40a0..95c2af42c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,4 +19,5 @@ ethers = {git="https://github.com/imotai/ethers-rs", rev="d526191b7972e8cf4412fe tonic = {git="https://github.com/hyperium/tonic", rev="ae7580160431cd25c1eecda4c85014ef6ce8d93f"} tonic-web = {git="https://github.com/hyperium/tonic", rev="ae7580160431cd25c1eecda4c85014ef6ce8d93f"} arweave-rs = {git="https://github.com/imotai/arweave-rs", rev="7ac5027305db833e4d9b62c967309d5df8fba4f2"} +ejdb2rs = {git="https://github.com/imotai/ejdb2rs", rev="c937a085eda043bd854a0ef4d3e7bb63650610d1"} serde_json= "1.0" diff --git a/src/proto/proto/db3_database_v2.proto b/src/proto/proto/db3_database_v2.proto index dd8e015e3..6e42e2047 100644 --- a/src/proto/proto/db3_database_v2.proto +++ b/src/proto/proto/db3_database_v2.proto @@ -54,10 +54,22 @@ message EventTable { message Collection { bytes id = 1; string name = 2; - repeated Index index_list = 3; + repeated Index index_fields = 3; bytes sender = 4; } +enum IndexType { + UniqueKey = 0; + StringKey = 1; + Int64Key = 2; + DoubleKey = 3; +} + +message Index { + string path = 1; + IndexType index_type = 2; +} + message Document { bytes id = 1; bytes doc = 2; @@ -65,100 +77,6 @@ message Document { bytes tx_id = 4; } -message Index { - // A field in an index. - // The field_path describes which field is indexed, the value_mode describes - // how the field value is indexed. - message IndexField { - // The supported orderings. - enum Order { - // The ordering is unspecified. Not a valid option. - ORDER_UNSPECIFIED = 0; - - // The field is ordered by ascending field value. - ASCENDING = 1; - - // The field is ordered by descending field value. - DESCENDING = 2; - } - - // The supported array value configurations. - enum ArrayConfig { - // The index does not support additional array queries. - ARRAY_CONFIG_UNSPECIFIED = 0; - - // The index supports array containment queries. - CONTAINS = 1; - } - - // Can be __name__. - // For single field indexes, this must match the name of the field or may - // be omitted. - string field_path = 1; - - // How the field value is indexed. - oneof value_mode { - // Indicates that this field supports ordering by the specified order or - // comparing using =, !=, <, <=, >, >=. - Order order = 2; - - // Indicates that this field supports operations on `array_value`s. - ArrayConfig array_config = 3; - } - } - - // The state of an index. During index creation, an index will be in the - // `CREATING` state. If the index is created successfully, it will transition - // to the `READY` state. If the index creation encounters a problem, the index - // will transition to the `NEEDS_REPAIR` state. - enum State { - // The state is unspecified. - STATE_UNSPECIFIED = 0; - - // The index is being created. - // There is an active long-running operation for the index. - // The index is updated when writing a document. - // Some index data may exist. - CREATING = 1; - - // The index is ready to be used. - // The index is updated when writing a document. - // The index is fully populated from all stored documents it applies to. - READY = 2; - - // The index was being created, but something went wrong. - // There is no active long-running operation for the index, - // and the most recently finished long-running operation failed. - // The index is not updated when writing a document. - // Some index data may exist. - // Use the google.longrunning.Operations API to determine why the operation - // that last attempted to create this index failed, then re-create the - // index. - NEEDS_REPAIR = 3; - } - - // Output only. A server defined name for this index. - // The form of this name for composite indexes will be: - // `projects/{project_id}/databases/{database_id}/collectionGroups/{collection_id}/indexes/{composite_index_id}` - // For single field indexes, this field will be empty. - string name = 1; - // Id of the index filed in the collection - uint32 id = 2; - - // The fields supported by this index. - // - // For composite indexes, this is always 2 or more fields. - // The last field entry is always for the field path `__name__`. If, on - // creation, `__name__` was not specified as the last field, it will be added - // automatically with the same direction as that of the last field defined. If - // the final field in a composite index is not directional, the `__name__` - // will be ordered ASCENDING (unless explicitly specified). - // - // For single field indexes, this will always be exactly one entry with a - // field path equal to the field path of the associated field. - repeated IndexField fields = 3; -} - // A Firestore query. message StructuredQuery { // A filter. diff --git a/src/proto/proto/db3_mutation_v2.proto b/src/proto/proto/db3_mutation_v2.proto index 4b303f139..926a71556 100644 --- a/src/proto/proto/db3_mutation_v2.proto +++ b/src/proto/proto/db3_mutation_v2.proto @@ -27,7 +27,7 @@ message DocumentDatabaseMutation { } message CollectionMutation { - repeated db3_database_v2_proto.Index index = 1; + repeated db3_database_v2_proto.Index index_fields = 1; string collection_name = 2; } diff --git a/src/storage/Cargo.toml b/src/storage/Cargo.toml index ea4a52ab2..c2312d224 100644 --- a/src/storage/Cargo.toml +++ b/src/storage/Cargo.toml @@ -35,3 +35,5 @@ timer = "0.2.0" arweave-rs = {workspace=true} url = "2.4.0" http = "0.2" +moka = "0.11.2" +ejdb2rs = {workspace=true} diff --git a/src/storage/src/db_store_v2.rs b/src/storage/src/db_store_v2.rs index 22cc07a1a..74f2cf2ea 100644 --- a/src/storage/src/db_store_v2.rs +++ b/src/storage/src/db_store_v2.rs @@ -219,7 +219,7 @@ impl DBStoreV2 { let col = Collection { id: id.as_ref().to_vec(), name: collection.collection_name.to_string(), - index_list: collection.index.to_vec(), + index_fields: collection.index_fields.to_vec(), sender: sender.as_ref().to_vec(), }; let mut buf = BytesMut::with_capacity(1024); @@ -338,7 +338,7 @@ mod tests { assert!(false); } let collection = CollectionMutation { - index: vec![], + index_fields: vec![], collection_name: "col1".to_string(), }; let result = diff --git a/src/storage/src/lib.rs b/src/storage/src/lib.rs index 6caf8e518..80c6512e1 100644 --- a/src/storage/src/lib.rs +++ b/src/storage/src/lib.rs @@ -26,6 +26,7 @@ pub mod db_owner_key; pub mod db_owner_key_v2; pub mod db_store; pub mod db_store_v2; +pub mod doc_store; pub mod key; pub mod mutation_store; pub mod state_store; From 366bbb48839e83c7b345cc50e5937ff3357ed3e0 Mon Sep 17 00:00:00 2001 From: imotai Date: Sat, 12 Aug 2023 11:40:07 +0800 Subject: [PATCH 2/4] fix: improve the add doc error --- sdk/src/index.ts | 8 +++++++- sdk/src/store/document_v2.ts | 12 +++++++++++- sdk/tests/client_v2.test.ts | 4 ++-- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/sdk/src/index.ts b/sdk/src/index.ts index 904cd043f..b2f161adb 100644 --- a/sdk/src/index.ts +++ b/sdk/src/index.ts @@ -31,7 +31,13 @@ export type { MutationResult, QueryResult, } from './store/types' -export { addDoc, updateDoc, deleteDoc, queryDoc, getDoc } from './store/document_v2' +export { + addDoc, + updateDoc, + deleteDoc, + queryDoc, + getDoc, +} from './store/document_v2' export { SystemConfig, SystemStatus, Version } from './proto/db3_base' export { diff --git a/sdk/src/store/document_v2.ts b/sdk/src/store/document_v2.ts index 9c9baef61..b4e84990b 100644 --- a/sdk/src/store/document_v2.ts +++ b/sdk/src/store/document_v2.ts @@ -225,6 +225,13 @@ export async function updateDoc( } } +/** + * Add a document to the collection. + * + * @param col The collection to add the document to. + * @param doc The document to add. + * @returns The ID of the newly added document. + */ export async function addDoc(col: Collection, doc: DocumentData) { const documentMutation: DocumentMutation = { collectionName: col.name, @@ -250,6 +257,7 @@ export async function addDoc(col: Collection, doc: DocumentData) { payload, col.db.client.nonce.toString() ) + if (response.code == 0 && response.items.length > 0) { col.db.client.nonce += 1 return { @@ -259,6 +267,8 @@ export async function addDoc(col: Collection, doc: DocumentData) { id: response.items[0].value, } } else { - throw new Error('fail to create collection') + throw new Error( + 'fail to addDoc, maybe you can syncAccountNonce to resolve the problem' + ) } } diff --git a/sdk/tests/client_v2.test.ts b/sdk/tests/client_v2.test.ts index 0452fe85e..894bb85fe 100644 --- a/sdk/tests/client_v2.test.ts +++ b/sdk/tests/client_v2.test.ts @@ -36,7 +36,7 @@ import { deleteDoc, updateDoc, queryDoc, - getDoc + getDoc, } from '../src/store/document_v2' import { createFromPrivateKey, @@ -383,7 +383,7 @@ describe('test db3.js client module', () => { try { const doc = await getDoc(collection, 1000000000000) except(1).toBe(0) - } catch(e) {} + } catch (e) {} } { From 36dc0e763895c9c91224a21e4ccf8559cbc60545 Mon Sep 17 00:00:00 2001 From: imotai Date: Sun, 3 Sep 2023 12:58:39 +0800 Subject: [PATCH 3/4] fix: use github runner --- .github/workflows/ci.yml | 4 ++-- Cargo.toml | 1 - src/node/src/rollup_executor.rs | 7 +++++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e79f2d61..09b29060c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,8 +31,8 @@ jobs: coverage: name: test - #runs-on: ubuntu-latest - runs-on: [self-hosted, linux, x64] + runs-on: ubuntu-latest + #runs-on: [self-hosted, linux, x64] steps: - name: Checkout repository uses: actions/checkout@v3 diff --git a/Cargo.toml b/Cargo.toml index 36cb07a07..fca698eff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,6 @@ members = [ "src/sdk", "src/event" ] - [workspace.dependencies] fastcrypto = {git = "https://github.com/MystenLabs/fastcrypto", rev = "306465d4fe04f6c26359d885f3b0a548b661de40"} ethers = {git="https://github.com/imotai/ethers-rs", rev="d526191b7972e8cf4412fee8b71cbf42e0ce7995"} diff --git a/src/node/src/rollup_executor.rs b/src/node/src/rollup_executor.rs index 2956eeaf9..597e0f7bf 100644 --- a/src/node/src/rollup_executor.rs +++ b/src/node/src/rollup_executor.rs @@ -270,12 +270,15 @@ impl RollupExecutor { return Ok(()); } let now = Instant::now(); + info!( "the next rollup start block {} and the newest block {current_block}", last_end_block ); + self.pending_start_block .store(last_end_block, Ordering::Relaxed); + self.pending_end_block .store(current_block, Ordering::Relaxed); let mutations = self @@ -305,6 +308,7 @@ impl RollupExecutor { self.pending_data_size.store(0, Ordering::Relaxed); self.pending_mutations.store(0, Ordering::Relaxed); } + let (id, reward, num_rows, size) = ar_toolbox .compress_and_upload_record_batch( tx, @@ -318,6 +322,7 @@ impl RollupExecutor { let (evm_cost, tx_hash) = meta_store .update_rollup_step(id.as_str(), network_id) .await?; + let tx_str = format!("0x{}", hex::encode(tx_hash.as_bytes())); info!("the process rollup done with num mutations {num_rows}, raw data size {memory_size}, compress data size {size} and processed time {} id {} ar cost {} and evm tx {} and cost {}", now.elapsed().as_secs(), @@ -325,6 +330,7 @@ impl RollupExecutor { tx_str.as_str(), evm_cost.as_u64() ); + let record = RollupRecord { end_block: current_block, raw_data_size: memory_size as u64, @@ -338,6 +344,7 @@ impl RollupExecutor { evm_tx: tx_str, evm_cost: evm_cost.as_u64(), }; + self.storage .add_rollup_record(&record) .map_err(|e| DB3Error::RollupError(format!("{e}")))?; From ff95ca44fda07bda5a1998ce489ffb7c6844be46 Mon Sep 17 00:00:00 2001 From: imotai Date: Sun, 3 Sep 2023 13:30:55 +0800 Subject: [PATCH 4/4] fix: add protobuf --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09b29060c..5ea9979e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,8 +51,8 @@ jobs: - name: Setup Build env run: | ROOT_DIR=`pwd` - #sudo apt-get install protobuf-compiler -y - #yarn global add arlocal_db3 + sudo apt-get install protobuf-compiler -y + yarn global add arlocal_db3 cd ${ROOT_DIR}/metadata && yarn install cd ${ROOT_DIR}/metadata && npx hardhat test test -e ${ROOT_DIR}/metadata/artifacts/contracts/DB3MetaStore.sol/DB3MetaStore.json && cp -f ${ROOT_DIR}/metadata/artifacts/contracts/DB3MetaStore.sol/DB3MetaStore.json ${ROOT_DIR}/abi/