From 042a9b1b579fed4259f7b03672934bc0ea73a4e0 Mon Sep 17 00:00:00 2001 From: phamphong9981 Date: Wed, 21 Feb 2024 16:56:48 +0700 Subject: [PATCH 1/7] feat: crawl from state file --- ci/config.json.ci | 3 +- config.json | 5 +- .../crawl-genesis/crawl_genesis.service.ts | 76 ++++++++++--------- 3 files changed, 47 insertions(+), 37 deletions(-) diff --git a/ci/config.json.ci b/ci/config.json.ci index 0c0e652c5..f2951ff6e 100644 --- a/ci/config.json.ci +++ b/ci/config.json.ci @@ -119,7 +119,8 @@ "proposalsPerBatch": 3000, "codesPerBatch": 5000, "smartContractsPerBatch": 5000, - "feeGrantsPerBatch": 5000 + "feeGrantsPerBatch": 5000, + "fromGenesis": true }, "handleVoteTx": { "key": "handleVoteTx", diff --git a/config.json b/config.json index fa240814c..c64f965bf 100644 --- a/config.json +++ b/config.json @@ -1,5 +1,5 @@ { - "chainId": "aura-testnet", + "chainId": "aura-testnet-2", "chainName": "aura", "networkPrefixAddress": "aura", "consensusPrefixAddress": "valcons", @@ -113,7 +113,8 @@ "proposalsPerBatch": 3000, "codesPerBatch": 5000, "smartContractsPerBatch": 5000, - "feeGrantsPerBatch": 5000 + "feeGrantsPerBatch": 5000, + "fromGenesis": true }, "handleVoteTx": { "key": "handleVoteTx", diff --git a/src/services/crawl-genesis/crawl_genesis.service.ts b/src/services/crawl-genesis/crawl_genesis.service.ts index 9a3465984..7e3dcb11b 100644 --- a/src/services/crawl-genesis/crawl_genesis.service.ts +++ b/src/services/crawl-genesis/crawl_genesis.service.ts @@ -41,6 +41,7 @@ import config from '../../../config.json' assert { type: 'json' }; import knex from '../../common/utils/db_connection'; import { ALLOWANCE_TYPE, FEEGRANT_STATUS } from '../feegrant/feegrant.service'; +const {fromGenesis} = config.crawlGenesis; @Service({ name: SERVICE.V1.CrawlGenesisService.key, version: 1, @@ -80,45 +81,51 @@ export default class CrawlGenesisService extends BullableService { return; } - if (!fs.existsSync('genesis.json')) fs.appendFileSync('genesis.json', ''); - try { - const genesis = await this._httpBatchClient.execute( - createJsonRpcRequest('genesis') - ); + if (fromGenesis) { + if (!fs.existsSync('genesis.json')) fs.appendFileSync('genesis.json', ''); + try { + const genesis = await this._httpBatchClient.execute( + createJsonRpcRequest('genesis') + ); - fs.appendFileSync('genesis.json', JSON.stringify(genesis.result.genesis)); - } catch (error: any) { - if (JSON.parse(error.message).code !== -32603) { - this.logger.error(error); - return; - } + fs.appendFileSync( + 'genesis.json', + JSON.stringify(genesis.result.genesis) + ); + } catch (error: any) { + if (JSON.parse(error.message).code !== -32603) { + this.logger.error(error); + return; + } - let index = 0; - let done = false; - while (!done) { - try { - this.logger.info(`Query genesis_chunked at page ${index}`); - const resultChunk = await this._httpBatchClient.execute( - createJsonRpcRequest('genesis_chunked', { - chunk: index.toString(), - }) - ); + let index = 0; + let done = false; + while (!done) { + try { + this.logger.info(`Query genesis_chunked at page ${index}`); + const resultChunk = await this._httpBatchClient.execute( + createJsonRpcRequest('genesis_chunked', { + chunk: index.toString(), + }) + ); - fs.appendFileSync( - 'genesis.json', - fromUtf8(fromBase64(resultChunk.result.data)) - ); - index += 1; - } catch (err) { - if (JSON.parse(error.message).code !== -32603) { - this.logger.error(error); - return; - } + fs.appendFileSync( + 'genesis.json', + fromUtf8(fromBase64(resultChunk.result.data)) + ); + index += 1; + } catch (err) { + if (JSON.parse(error.message).code !== -32603) { + this.logger.error(error); + return; + } - done = true; + done = true; + } } } - } + } else if (!fs.existsSync('state.json')) + throw new Error('Not found state json file'); // fs.renameSync('genesis.txt', 'genesis.json'); @@ -885,8 +892,9 @@ export default class CrawlGenesisService extends BullableService { data: string, filter?: (data: any) => void ): Promise { + const file = fromGenesis ? 'genesis.json' : 'state.json'; const pipeline = Chain.chain([ - fs.createReadStream('genesis.json'), + fs.createReadStream(file), Pick.withParser({ filter: data }), StreamArr.streamArray(), ]); From 6364c9d2ceb082f4c5e30c8b7b0090292532ecca Mon Sep 17 00:00:00 2001 From: phamphong9981 Date: Wed, 21 Feb 2024 16:59:37 +0700 Subject: [PATCH 2/7] feat: crawl from state file --- config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.json b/config.json index c64f965bf..2d94a13e8 100644 --- a/config.json +++ b/config.json @@ -1,5 +1,5 @@ { - "chainId": "aura-testnet-2", + "chainId": "aura-testnet", "chainName": "aura", "networkPrefixAddress": "aura", "consensusPrefixAddress": "valcons", From 0e183c6ac8440d36052f886c9b35dada28cffc2a Mon Sep 17 00:00:00 2001 From: phamphong9981 Date: Thu, 22 Feb 2024 09:51:24 +0700 Subject: [PATCH 3/7] feat: crawl from state file --- ci/config.json.ci | 2 +- config.json | 2 +- src/services/crawl-genesis/crawl_genesis.service.ts | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/config.json.ci b/ci/config.json.ci index f2951ff6e..dbf3a38d7 100644 --- a/ci/config.json.ci +++ b/ci/config.json.ci @@ -120,7 +120,7 @@ "codesPerBatch": 5000, "smartContractsPerBatch": 5000, "feeGrantsPerBatch": 5000, - "fromGenesis": true + "stateFile": false }, "handleVoteTx": { "key": "handleVoteTx", diff --git a/config.json b/config.json index 2d94a13e8..4a561c745 100644 --- a/config.json +++ b/config.json @@ -114,7 +114,7 @@ "codesPerBatch": 5000, "smartContractsPerBatch": 5000, "feeGrantsPerBatch": 5000, - "fromGenesis": true + "stateFile": false }, "handleVoteTx": { "key": "handleVoteTx", diff --git a/src/services/crawl-genesis/crawl_genesis.service.ts b/src/services/crawl-genesis/crawl_genesis.service.ts index 7e3dcb11b..539d05155 100644 --- a/src/services/crawl-genesis/crawl_genesis.service.ts +++ b/src/services/crawl-genesis/crawl_genesis.service.ts @@ -41,7 +41,7 @@ import config from '../../../config.json' assert { type: 'json' }; import knex from '../../common/utils/db_connection'; import { ALLOWANCE_TYPE, FEEGRANT_STATUS } from '../feegrant/feegrant.service'; -const {fromGenesis} = config.crawlGenesis; +const { stateFile } = config.crawlGenesis; @Service({ name: SERVICE.V1.CrawlGenesisService.key, version: 1, @@ -81,7 +81,7 @@ export default class CrawlGenesisService extends BullableService { return; } - if (fromGenesis) { + if (!stateFile) { if (!fs.existsSync('genesis.json')) fs.appendFileSync('genesis.json', ''); try { const genesis = await this._httpBatchClient.execute( @@ -125,7 +125,7 @@ export default class CrawlGenesisService extends BullableService { } } } else if (!fs.existsSync('state.json')) - throw new Error('Not found state json file'); + throw new Error('Not found state json file'); // fs.renameSync('genesis.txt', 'genesis.json'); @@ -892,7 +892,7 @@ export default class CrawlGenesisService extends BullableService { data: string, filter?: (data: any) => void ): Promise { - const file = fromGenesis ? 'genesis.json' : 'state.json'; + const file = stateFile ? 'state.json' : 'genesis.json'; const pipeline = Chain.chain([ fs.createReadStream(file), Pick.withParser({ filter: data }), From 35be2096f061e04bfe0b58329f614ad4c3b57d30 Mon Sep 17 00:00:00 2001 From: phamphong9981 Date: Thu, 22 Feb 2024 16:00:21 +0700 Subject: [PATCH 4/7] refactor: code --- ci/config.json.ci | 2 +- config.json | 2 +- .../crawl-genesis/crawl_genesis.service.ts | 144 ++++++++++-------- 3 files changed, 81 insertions(+), 67 deletions(-) diff --git a/ci/config.json.ci b/ci/config.json.ci index dbf3a38d7..e75588ade 100644 --- a/ci/config.json.ci +++ b/ci/config.json.ci @@ -120,7 +120,7 @@ "codesPerBatch": 5000, "smartContractsPerBatch": 5000, "feeGrantsPerBatch": 5000, - "stateFile": false + "stateFile": "https://s3-sgn10.fptcloud.com/punkga-dev/sei-49630285.json" }, "handleVoteTx": { "key": "handleVoteTx", diff --git a/config.json b/config.json index 4a561c745..9177e0a78 100644 --- a/config.json +++ b/config.json @@ -114,7 +114,7 @@ "codesPerBatch": 5000, "smartContractsPerBatch": 5000, "feeGrantsPerBatch": 5000, - "stateFile": false + "stateFile": "https://s3-sgn10.fptcloud.com/punkga-dev/sei-49630285.json" }, "handleVoteTx": { "key": "handleVoteTx", diff --git a/src/services/crawl-genesis/crawl_genesis.service.ts b/src/services/crawl-genesis/crawl_genesis.service.ts index 539d05155..ec109252c 100644 --- a/src/services/crawl-genesis/crawl_genesis.service.ts +++ b/src/services/crawl-genesis/crawl_genesis.service.ts @@ -1,44 +1,47 @@ /* eslint-disable no-param-reassign */ /* eslint-disable no-await-in-loop */ -import { Service } from '@ourparentcenter/moleculer-decorators-extended'; -import { ServiceBroker } from 'moleculer'; +import { ibc } from '@aura-nw/aurajs'; +import { QueryDenomTraceRequest } from '@aura-nw/aurajs/types/codegen/ibc/applications/transfer/v1/query'; +import { fromBase64, fromUtf8, toHex } from '@cosmjs/encoding'; +import { JsonRpcSuccessResponse } from '@cosmjs/json-rpc'; import { HttpBatchClient } from '@cosmjs/tendermint-rpc'; import { createJsonRpcRequest } from '@cosmjs/tendermint-rpc/build/jsonrpc'; -import { fromBase64, fromUtf8, toHex } from '@cosmjs/encoding'; +import { Service } from '@ourparentcenter/moleculer-decorators-extended'; +import axios from 'axios'; import fs from 'fs'; import _ from 'lodash'; +import { ServiceBroker } from 'moleculer'; +import { Stream } from 'stream'; import Chain from 'stream-chain'; import Pick from 'stream-json/filters/Pick'; import StreamArr from 'stream-json/streamers/StreamArray'; -import { QueryDenomTraceRequest } from '@aura-nw/aurajs/types/codegen/ibc/applications/transfer/v1/query'; -import { ibc } from '@aura-nw/aurajs'; -import { JsonRpcSuccessResponse } from '@cosmjs/json-rpc'; -import Utils from '../../common/utils/utils'; -import { - Account, - BlockCheckpoint, - Code, - Proposal, - SmartContract, - Validator, - Feegrant, - IbcClient, - IbcConnection, - IbcChannel, -} from '../../models'; +import { promisify } from 'util'; +import config from '../../../config.json' assert { type: 'json' }; import BullableService, { QueueHandler } from '../../base/bullable.service'; import { ABCI_QUERY_PATH, AccountType, BULL_JOB_NAME, - getHttpBatchClient, ICoin, MSG_TYPE, REDIS_KEY, SERVICE, + getHttpBatchClient, } from '../../common'; -import config from '../../../config.json' assert { type: 'json' }; import knex from '../../common/utils/db_connection'; +import Utils from '../../common/utils/utils'; +import { + Account, + BlockCheckpoint, + Code, + Feegrant, + IbcChannel, + IbcClient, + IbcConnection, + Proposal, + SmartContract, + Validator, +} from '../../models'; import { ALLOWANCE_TYPE, FEEGRANT_STATUS } from '../feegrant/feegrant.service'; const { stateFile } = config.crawlGenesis; @@ -82,48 +85,7 @@ export default class CrawlGenesisService extends BullableService { } if (!stateFile) { - if (!fs.existsSync('genesis.json')) fs.appendFileSync('genesis.json', ''); - try { - const genesis = await this._httpBatchClient.execute( - createJsonRpcRequest('genesis') - ); - - fs.appendFileSync( - 'genesis.json', - JSON.stringify(genesis.result.genesis) - ); - } catch (error: any) { - if (JSON.parse(error.message).code !== -32603) { - this.logger.error(error); - return; - } - - let index = 0; - let done = false; - while (!done) { - try { - this.logger.info(`Query genesis_chunked at page ${index}`); - const resultChunk = await this._httpBatchClient.execute( - createJsonRpcRequest('genesis_chunked', { - chunk: index.toString(), - }) - ); - - fs.appendFileSync( - 'genesis.json', - fromUtf8(fromBase64(resultChunk.result.data)) - ); - index += 1; - } catch (err) { - if (JSON.parse(error.message).code !== -32603) { - this.logger.error(error); - return; - } - - done = true; - } - } - } + await this.getGenesisFile(); } else if (!fs.existsSync('state.json')) throw new Error('Not found state json file'); @@ -163,6 +125,59 @@ export default class CrawlGenesisService extends BullableService { }); } + async getGenesisFile() { + if (!fs.existsSync('genesis.json')) fs.appendFileSync('genesis.json', ''); + try { + const genesis = await this._httpBatchClient.execute( + createJsonRpcRequest('genesis') + ); + + fs.appendFileSync('genesis.json', JSON.stringify(genesis.result.genesis)); + } catch (error: any) { + if (JSON.parse(error.message).code !== -32603) { + this.logger.error(error); + return; + } + + let index = 0; + let done = false; + while (!done) { + try { + this.logger.info(`Query genesis_chunked at page ${index}`); + const resultChunk = await this._httpBatchClient.execute( + createJsonRpcRequest('genesis_chunked', { + chunk: index.toString(), + }) + ); + + fs.appendFileSync( + 'genesis.json', + fromUtf8(fromBase64(resultChunk.result.data)) + ); + index += 1; + } catch (err) { + if (JSON.parse(error.message).code !== -32603) { + this.logger.error(error); + return; + } + + done = true; + } + } + } + } + + async getStateFile() { + const writer = fs.createWriteStream('genesis.json'); + const response = await axios({ + url: stateFile, + method: 'GET', + responseType: 'stream', + }); + response.data.pipe(writer); + return promisify(Stream.Stream.finished)(writer); + } + @QueueHandler({ queueName: BULL_JOB_NAME.CRAWL_GENESIS_ACCOUNT, jobName: BULL_JOB_NAME.CRAWL_GENESIS_ACCOUNT, @@ -892,9 +907,8 @@ export default class CrawlGenesisService extends BullableService { data: string, filter?: (data: any) => void ): Promise { - const file = stateFile ? 'state.json' : 'genesis.json'; const pipeline = Chain.chain([ - fs.createReadStream(file), + fs.createReadStream('genesis.json'), Pick.withParser({ filter: data }), StreamArr.streamArray(), ]); From ca0cde3297f0ad5f16885ac9ab116c33b68a885c Mon Sep 17 00:00:00 2001 From: phamphong9981 Date: Thu, 22 Feb 2024 16:06:37 +0700 Subject: [PATCH 5/7] refactor: code --- ci/config.json.ci | 2 +- config.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/config.json.ci b/ci/config.json.ci index e75588ade..b58329575 100644 --- a/ci/config.json.ci +++ b/ci/config.json.ci @@ -120,7 +120,7 @@ "codesPerBatch": 5000, "smartContractsPerBatch": 5000, "feeGrantsPerBatch": 5000, - "stateFile": "https://s3-sgn10.fptcloud.com/punkga-dev/sei-49630285.json" + "stateFile": "" }, "handleVoteTx": { "key": "handleVoteTx", diff --git a/config.json b/config.json index 9177e0a78..388a0a039 100644 --- a/config.json +++ b/config.json @@ -114,7 +114,7 @@ "codesPerBatch": 5000, "smartContractsPerBatch": 5000, "feeGrantsPerBatch": 5000, - "stateFile": "https://s3-sgn10.fptcloud.com/punkga-dev/sei-49630285.json" + "stateFile": "" }, "handleVoteTx": { "key": "handleVoteTx", From 3bb1550e76a6ca7fde80e4e51ec99283bb7628aa Mon Sep 17 00:00:00 2001 From: phamphong9981 Date: Fri, 23 Feb 2024 15:35:05 +0700 Subject: [PATCH 6/7] refactor: code --- src/services/crawl-genesis/crawl_genesis.service.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/services/crawl-genesis/crawl_genesis.service.ts b/src/services/crawl-genesis/crawl_genesis.service.ts index ec109252c..cf8222c25 100644 --- a/src/services/crawl-genesis/crawl_genesis.service.ts +++ b/src/services/crawl-genesis/crawl_genesis.service.ts @@ -86,8 +86,9 @@ export default class CrawlGenesisService extends BullableService { if (!stateFile) { await this.getGenesisFile(); - } else if (!fs.existsSync('state.json')) - throw new Error('Not found state json file'); + } else { + await this.getStateFile(); + } // fs.renameSync('genesis.txt', 'genesis.json'); From 8c98ad87dd3c29b117d7406da6c6ca787c77e68c Mon Sep 17 00:00:00 2001 From: fibonacci998 Date: Thu, 2 May 2024 14:49:37 +0700 Subject: [PATCH 7/7] feat: add copy state file from directory --- ci/config.json.ci | 3 +- config.json | 3 +- .../crawl-genesis/crawl_genesis.service.ts | 39 ++++++++++++------- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/ci/config.json.ci b/ci/config.json.ci index b58329575..5cd8bb62e 100644 --- a/ci/config.json.ci +++ b/ci/config.json.ci @@ -120,7 +120,8 @@ "codesPerBatch": 5000, "smartContractsPerBatch": 5000, "feeGrantsPerBatch": 5000, - "stateFile": "" + "stateFileURL": "", + "stateFileDirectory": "" }, "handleVoteTx": { "key": "handleVoteTx", diff --git a/config.json b/config.json index 388a0a039..55a37cd2e 100644 --- a/config.json +++ b/config.json @@ -114,7 +114,8 @@ "codesPerBatch": 5000, "smartContractsPerBatch": 5000, "feeGrantsPerBatch": 5000, - "stateFile": "" + "stateFileURL": "", + "stateFileDirectory": "" }, "handleVoteTx": { "key": "handleVoteTx", diff --git a/src/services/crawl-genesis/crawl_genesis.service.ts b/src/services/crawl-genesis/crawl_genesis.service.ts index cf8222c25..01f058cb9 100644 --- a/src/services/crawl-genesis/crawl_genesis.service.ts +++ b/src/services/crawl-genesis/crawl_genesis.service.ts @@ -44,7 +44,7 @@ import { } from '../../models'; import { ALLOWANCE_TYPE, FEEGRANT_STATUS } from '../feegrant/feegrant.service'; -const { stateFile } = config.crawlGenesis; +const { stateFileURL, stateFileDirectory } = config.crawlGenesis; @Service({ name: SERVICE.V1.CrawlGenesisService.key, version: 1, @@ -84,14 +84,15 @@ export default class CrawlGenesisService extends BullableService { return; } - if (!stateFile) { - await this.getGenesisFile(); - } else { - await this.getStateFile(); + // if not found file genesis.json, then need fetch from rpc or volume + if (!fs.existsSync('genesis.json')) { + if (stateFileURL || stateFileDirectory) { + await this.getStateFile(); + } else { + await this.getGenesisFile(); + } } - // fs.renameSync('genesis.txt', 'genesis.json'); - let updateBlkCheck: BlockCheckpoint; if (genesisBlkCheck) { updateBlkCheck = genesisBlkCheck; @@ -169,14 +170,22 @@ export default class CrawlGenesisService extends BullableService { } async getStateFile() { - const writer = fs.createWriteStream('genesis.json'); - const response = await axios({ - url: stateFile, - method: 'GET', - responseType: 'stream', - }); - response.data.pipe(writer); - return promisify(Stream.Stream.finished)(writer); + if (stateFileURL) { + const writer = fs.createWriteStream('genesis.json'); + const response = await axios({ + url: stateFileURL, + method: 'GET', + responseType: 'stream', + }); + response.data.pipe(writer); + return promisify(Stream.Stream.finished)(writer); + } + if (stateFileDirectory) { + return fs.copyFileSync(stateFileDirectory, 'genesis.json'); + } + throw Error( + 'At least stateFileUrl or stateFileDirectory must be configured' + ); } @QueueHandler({