From 7f1ea0890ab1898fa60b6c8d23c3b0517289a53e Mon Sep 17 00:00:00 2001 From: NagyZoltanPeter <113987313+NagyZoltanPeter@users.noreply.github.com> Date: Thu, 3 Oct 2024 13:10:48 +0200 Subject: [PATCH 1/5] Bump nim-chornicles, nim-libp2p, nimcrypto Bump nim-web3, nim-eth and deps - on_chain/group_manager.nim adaption Added status-im/nim-minilru submodule required by latest nim-eth Added nim-quic and nim-ngtcp2 as new deps of nim-libp2p Fixing tests. Changes: modified: .gitmodules modified: tests/waku_discv5/utils.nim modified: tests/waku_enr/utils.nim modified: tests/waku_rln_relay/test_rln_group_manager_onchain.nim modified: tests/waku_rln_relay/utils.nim modified: tests/waku_rln_relay/utils_onchain.nim modified: vendor/nim-chronicles modified: vendor/nim-eth modified: vendor/nim-http-utils modified: vendor/nim-json-rpc modified: vendor/nim-json-serialization modified: vendor/nim-libp2p modified: vendor/nim-metrics new file: vendor/nim-minilru modified: vendor/nim-nat-traversal new file: vendor/nim-ngtcp2 modified: vendor/nim-presto new file: vendor/nim-quic modified: vendor/nim-secp256k1 modified: vendor/nim-serialization modified: vendor/nim-stew modified: vendor/nim-taskpools modified: vendor/nim-testutils modified: vendor/nim-toml-serialization modified: vendor/nim-unicodedb modified: vendor/nim-unittest2 modified: vendor/nim-web3 modified: vendor/nim-websock modified: vendor/nim-zlib modified: vendor/nimcrypto modified: waku.nimble modified: waku/common/enr/builder.nim modified: waku/common/enr/typed_record.nim modified: waku/common/utils/nat.nim modified: waku/discovery/waku_discv5.nim modified: waku/waku_rln_relay/conversion_utils.nim modified: waku/waku_rln_relay/group_manager/on_chain/group_manager.nim modified: waku/waku_rln_relay/rln/wrappers.nim modified: waku/waku_rln_relay/rln_relay.nim --- .gitmodules | 15 +++++ tests/waku_discv5/utils.nim | 1 + tests/waku_enr/utils.nim | 1 + .../test_rln_group_manager_onchain.nim | 4 +- tests/waku_rln_relay/utils.nim | 17 +++--- tests/waku_rln_relay/utils_onchain.nim | 31 +++++++--- vendor/nim-chronicles | 2 +- vendor/nim-eth | 2 +- vendor/nim-http-utils | 2 +- vendor/nim-json-rpc | 2 +- vendor/nim-json-serialization | 2 +- vendor/nim-libp2p | 2 +- vendor/nim-metrics | 2 +- vendor/nim-minilru | 1 + vendor/nim-nat-traversal | 2 +- vendor/nim-ngtcp2 | 1 + vendor/nim-presto | 2 +- vendor/nim-quic | 1 + vendor/nim-secp256k1 | 2 +- vendor/nim-serialization | 2 +- vendor/nim-stew | 2 +- vendor/nim-taskpools | 2 +- vendor/nim-testutils | 2 +- vendor/nim-toml-serialization | 2 +- vendor/nim-unicodedb | 2 +- vendor/nim-unittest2 | 2 +- vendor/nim-web3 | 2 +- vendor/nim-websock | 2 +- vendor/nim-zlib | 2 +- vendor/nimcrypto | 2 +- waku.nimble | 4 +- waku/common/enr/builder.nim | 6 +- waku/common/enr/typed_record.nim | 15 ++++- waku/common/utils/nat.nim | 4 +- waku/discovery/waku_discv5.nim | 8 +-- waku/waku_rln_relay/conversion_utils.nim | 5 ++ .../group_manager/on_chain/group_manager.nim | 61 ++++++++++--------- waku/waku_rln_relay/rln/wrappers.nim | 2 +- waku/waku_rln_relay/rln_relay.nim | 4 +- 39 files changed, 142 insertions(+), 81 deletions(-) create mode 160000 vendor/nim-minilru create mode 160000 vendor/nim-ngtcp2 create mode 160000 vendor/nim-quic diff --git a/.gitmodules b/.gitmodules index 5650701b4c..eb05c758dc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -179,3 +179,18 @@ branch = master path = vendor/nph url = https://github.com/arnetheduck/nph.git +[submodule "vendor/nim-minilru"] + path = vendor/nim-minilru + url = https://github.com/status-im/nim-minilru.git + ignore = untracked + branch = master +[submodule "vendor/nim-quic"] + path = vendor/nim-quic + url = https://github.com/status-im/nim-quic.git + ignore = untracked + branch = master +[submodule "vendor/nim-ngtcp2"] + path = vendor/nim-ngtcp2 + url = https://github.com/vacp2p/nim-ngtcp2.git + ignore = untracked + branch = master diff --git a/tests/waku_discv5/utils.nim b/tests/waku_discv5/utils.nim index 10026155c7..679d206ea3 100644 --- a/tests/waku_discv5/utils.nim +++ b/tests/waku_discv5/utils.nim @@ -1,4 +1,5 @@ import + std/options, stew/results, stew/shims/net, chronos, diff --git a/tests/waku_enr/utils.nim b/tests/waku_enr/utils.nim index 5cc36f607e..6dd017add9 100644 --- a/tests/waku_enr/utils.nim +++ b/tests/waku_enr/utils.nim @@ -1,4 +1,5 @@ import + std/options, sequtils, stew/results, stew/shims/net, diff --git a/tests/waku_rln_relay/test_rln_group_manager_onchain.nim b/tests/waku_rln_relay/test_rln_group_manager_onchain.nim index 5ef6913f78..3e6980db63 100644 --- a/tests/waku_rln_relay/test_rln_group_manager_onchain.nim +++ b/tests/waku_rln_relay/test_rln_group_manager_onchain.nim @@ -42,14 +42,14 @@ suite "Onchain group manager": manager.ethRpc.isSome() manager.wakuRlnContract.isSome() manager.initialized - manager.rlnContractDeployedBlockNumber > 0 + manager.rlnContractDeployedBlockNumber > 0.Quantity manager.rlnRelayMaxMessageLimit == 100 await manager.stop() asyncTest "should error on initialization when chainId does not match": let manager = await setup() - manager.chainId = CHAIN_ID + 1 + manager.chainId = utils_onchain.CHAIN_ID + 1 (await manager.init()).isErrOr: raiseAssert "Expected error when chainId does not match" diff --git a/tests/waku_rln_relay/utils.nim b/tests/waku_rln_relay/utils.nim index 548414fb7d..2086acb88e 100644 --- a/tests/waku_rln_relay/utils.nim +++ b/tests/waku_rln_relay/utils.nim @@ -1,4 +1,4 @@ -import web3, chronos, options, stint +import web3, chronos, options, stint, stew/byteutils proc deployContract*( web3: Web3, code: string, gasPrice = 0, contractInput = "" @@ -6,13 +6,13 @@ proc deployContract*( # the contract input is the encoded version of contract constructor's input # use nim-web3/encoding.nim module to find the appropriate encoding procedure for different argument types # e.g., consider the following contract constructor in solidity - # constructor(uint256 x, uint256 y) + # constructor(uint256 x, uint256 y) # # the contractInput can be calculated as follows # let # x = 1.u256 # y = 5.u256 - # contractInput = encode(x).data & encode(y).data + # contractInput = encode(x).data & encode(y).data # Note that the order of encoded inputs should match the order of the constructor inputs let provider = web3.provider let accounts = await provider.eth_accounts() @@ -20,12 +20,13 @@ proc deployContract*( var code = code if code[1] notin {'x', 'X'}: code = "0x" & code - var tr: EthSend - tr.source = web3.defaultAccount - tr.data = code & contractInput - tr.gas = Quantity(3000000000000).some + var tr: TransactionArgs + tr.`from` = Opt.some(web3.defaultAccount) + let sData = code & contractInput + tr.data = Opt.some(sData.toBytes()) + tr.gas = Opt.some(Quantity(3000000000000)) if gasPrice != 0: - tr.gasPrice = some(gasPrice) + tr.gasPrice = Opt.some(gasPrice.Quantity) let r = await web3.send(tr) return await web3.getMinedTransactionReceipt(r) diff --git a/tests/waku_rln_relay/utils_onchain.nim b/tests/waku_rln_relay/utils_onchain.nim index 272ddffa69..0c89f10c10 100644 --- a/tests/waku_rln_relay/utils_onchain.nim +++ b/tests/waku_rln_relay/utils_onchain.nim @@ -10,6 +10,8 @@ import chronicles, stint, web3, + web3/conversions, + web3/eth_api_types, json, libp2p/crypto/crypto, eth/keys @@ -27,6 +29,18 @@ import const CHAIN_ID* = 1337 +template skip0xPrefix(hexStr: string): int = + ## Returns the index of the first meaningful char in `hexStr` by skipping + ## "0x" prefix + if hexStr.len > 1 and hexStr[0] == '0' and hexStr[1] in {'x', 'X'}: 2 else: 0 + +func strip0xPrefix(s: string): string = + let prefixLen = skip0xPrefix(s) + if prefixLen != 0: + s[prefixLen .. ^1] + else: + s + proc generateCredentials*(rlnInstance: ptr RLN): IdentityCredential = let credRes = membershipKeyGen(rlnInstance) return credRes.get() @@ -85,12 +99,13 @@ proc uploadRLNContract*(ethClientAddress: string): Future[Address] {.async.} = debug "Address of the deployed rlnv2 contract: ", wakuRlnContractAddress # need to send concat: impl & init_bytes - let contractInput = encode(wakuRlnContractAddress).data & Erc1967ProxyContractInput + let contractInput = + byteutils.toHex(encode(wakuRlnContractAddress)) & Erc1967ProxyContractInput debug "contractInput", contractInput let proxyReceipt = await web3.deployContract(Erc1967Proxy, contractInput = contractInput) - debug "proxy receipt", proxyReceipt + debug "proxy receipt", contractAddress = proxyReceipt.contractAddress.get() let proxyAddress = proxyReceipt.contractAddress.get() let newBalance = await web3.provider.eth_getBalance(web3.defaultAccount, "latest") @@ -106,17 +121,17 @@ proc createEthAccount*( ): Future[(keys.PrivateKey, Address)] {.async.} = let web3 = await newWeb3(EthClient) let accounts = await web3.provider.eth_accounts() - let gasPrice = int(await web3.provider.eth_gasPrice()) + let gasPrice = Quantity(await web3.provider.eth_gasPrice()) web3.defaultAccount = accounts[0] let pk = keys.PrivateKey.random(rng[]) let acc = Address(toCanonicalAddress(pk.toPublicKey())) - var tx: EthSend - tx.source = accounts[0] - tx.value = some(ethToWei(ethAmount)) - tx.to = some(acc) - tx.gasPrice = some(gasPrice) + var tx: TransactionArgs + tx.`from` = Opt.some(accounts[0]) + tx.value = Opt.some(ethToWei(ethAmount)) + tx.to = Opt.some(acc) + tx.gasPrice = Opt.some(gasPrice) # Send ethAmount to acc discard await web3.send(tx) diff --git a/vendor/nim-chronicles b/vendor/nim-chronicles index a28bb9781c..4524912fca 160000 --- a/vendor/nim-chronicles +++ b/vendor/nim-chronicles @@ -1 +1 @@ -Subproject commit a28bb9781ce74e725796c307ad05083e646872be +Subproject commit 4524912fcacfd3965e32a9fe23c9bb12d48177ff diff --git a/vendor/nim-eth b/vendor/nim-eth index bb5cb6a4d0..84664b0fc0 160000 --- a/vendor/nim-eth +++ b/vendor/nim-eth @@ -1 +1 @@ -Subproject commit bb5cb6a4d0dbb592b395ab41cda3fd3f1fdc443e +Subproject commit 84664b0fc0b9128b4316afb85274271389aab00f diff --git a/vendor/nim-http-utils b/vendor/nim-http-utils index 8b88ad6dd9..e8fc71aee1 160000 --- a/vendor/nim-http-utils +++ b/vendor/nim-http-utils @@ -1 +1 @@ -Subproject commit 8b88ad6dd9a6326c29f82067800c483d9410d873 +Subproject commit e8fc71aee15203a852f6321e4dd8d87517502847 diff --git a/vendor/nim-json-rpc b/vendor/nim-json-rpc index 33360528d2..be16a6528a 160000 --- a/vendor/nim-json-rpc +++ b/vendor/nim-json-rpc @@ -1 +1 @@ -Subproject commit 33360528d208aaa70a6ed914619fe76321760d54 +Subproject commit be16a6528acfcfdaab381c4900240e454f7cdc63 diff --git a/vendor/nim-json-serialization b/vendor/nim-json-serialization index 8a4ed98bbd..ab1a061756 160000 --- a/vendor/nim-json-serialization +++ b/vendor/nim-json-serialization @@ -1 +1 @@ -Subproject commit 8a4ed98bbd0a9479df15af2fa31da38a586ea6d5 +Subproject commit ab1a061756bb6fc2e0f98cb57852f2bb0c6f9772 diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p index 18a2e79ce2..09fe199b6b 160000 --- a/vendor/nim-libp2p +++ b/vendor/nim-libp2p @@ -1 +1 @@ -Subproject commit 18a2e79ce209d2f21d8c5db7a41659cc39dbe0b4 +Subproject commit 09fe199b6bee6c3d0cc0e8750e67bbc0d52da138 diff --git a/vendor/nim-metrics b/vendor/nim-metrics index 29bb7ba63c..f1f8869578 160000 --- a/vendor/nim-metrics +++ b/vendor/nim-metrics @@ -1 +1 @@ -Subproject commit 29bb7ba63cd884770169891687595348a70cf166 +Subproject commit f1f886957831e9f2f16ffa728abb4bf44bfd5b98 diff --git a/vendor/nim-minilru b/vendor/nim-minilru new file mode 160000 index 0000000000..2682cffa87 --- /dev/null +++ b/vendor/nim-minilru @@ -0,0 +1 @@ +Subproject commit 2682cffa8733f3b61751c65a963941315e887bac diff --git a/vendor/nim-nat-traversal b/vendor/nim-nat-traversal index a3aa0c5f9d..05e76accbc 160000 --- a/vendor/nim-nat-traversal +++ b/vendor/nim-nat-traversal @@ -1 +1 @@ -Subproject commit a3aa0c5f9d2a2870f1fd0f7a613d4fe025c84ab7 +Subproject commit 05e76accbc7811273fadc23d8886ac1a2f83bb81 diff --git a/vendor/nim-ngtcp2 b/vendor/nim-ngtcp2 new file mode 160000 index 0000000000..6834f4756b --- /dev/null +++ b/vendor/nim-ngtcp2 @@ -0,0 +1 @@ +Subproject commit 6834f4756b6af58356ac9c4fef3d71db3c3ae5fe diff --git a/vendor/nim-presto b/vendor/nim-presto index 2190421e09..5d5dc51bac 160000 --- a/vendor/nim-presto +++ b/vendor/nim-presto @@ -1 +1 @@ -Subproject commit 2190421e09938696cd95d54b1f4753446c84c7a2 +Subproject commit 5d5dc51bac4aafb26c03d2d813a47e80093bd0c7 diff --git a/vendor/nim-quic b/vendor/nim-quic new file mode 160000 index 0000000000..ddcb31ffb7 --- /dev/null +++ b/vendor/nim-quic @@ -0,0 +1 @@ +Subproject commit ddcb31ffb74b5460ab37fd13547eca90594248bc diff --git a/vendor/nim-secp256k1 b/vendor/nim-secp256k1 index 4470f49bcd..641902d492 160000 --- a/vendor/nim-secp256k1 +++ b/vendor/nim-secp256k1 @@ -1 +1 @@ -Subproject commit 4470f49bcd6bcbfb59f0eeb67315ca9ddac0bdc0 +Subproject commit 641902d492aff3910b0240217fa0b28b5471baad diff --git a/vendor/nim-serialization b/vendor/nim-serialization index 298a9554a8..7950ac1e8c 160000 --- a/vendor/nim-serialization +++ b/vendor/nim-serialization @@ -1 +1 @@ -Subproject commit 298a9554a885b2df59737bb3461aac8d0d339724 +Subproject commit 7950ac1e8c9f0019c798f97ab1fe1d9367cc2e6d diff --git a/vendor/nim-stew b/vendor/nim-stew index d4634c5405..b7b5969557 160000 --- a/vendor/nim-stew +++ b/vendor/nim-stew @@ -1 +1 @@ -Subproject commit d4634c5405ac188e7050d348332edb6c3b09a527 +Subproject commit b7b596955726a65788561d2d69978223d4484c95 diff --git a/vendor/nim-taskpools b/vendor/nim-taskpools index d4c43137c0..66585e2e96 160000 --- a/vendor/nim-taskpools +++ b/vendor/nim-taskpools @@ -1 +1 @@ -Subproject commit d4c43137c0590cb47f893a66ca8cb027fa6c217e +Subproject commit 66585e2e960b7695e48ea60377fb3aeac96406e8 diff --git a/vendor/nim-testutils b/vendor/nim-testutils index ae476c6731..c36724c469 160000 --- a/vendor/nim-testutils +++ b/vendor/nim-testutils @@ -1 +1 @@ -Subproject commit ae476c67314ac4b294d21040315e0f716189a70e +Subproject commit c36724c469b657435f40fb7d365ad14640341606 diff --git a/vendor/nim-toml-serialization b/vendor/nim-toml-serialization index cb1fc73f35..53ae081cc9 160000 --- a/vendor/nim-toml-serialization +++ b/vendor/nim-toml-serialization @@ -1 +1 @@ -Subproject commit cb1fc73f3519fed5f3a8fbfa90afc9a96d5f5f5c +Subproject commit 53ae081cc9a3e61095cab49862658a46b00eacf7 diff --git a/vendor/nim-unicodedb b/vendor/nim-unicodedb index 8c8959d84c..15c5e25e2a 160000 --- a/vendor/nim-unicodedb +++ b/vendor/nim-unicodedb @@ -1 +1 @@ -Subproject commit 8c8959d84c12ecda6ea14c67bd68675b1936f8cf +Subproject commit 15c5e25e2a49a924bc97647481ff50125bba2c76 diff --git a/vendor/nim-unittest2 b/vendor/nim-unittest2 index e96f321503..11f7cff928 160000 --- a/vendor/nim-unittest2 +++ b/vendor/nim-unittest2 @@ -1 +1 @@ -Subproject commit e96f3215030cbfa13abc2f5827069b6f8ba87e38 +Subproject commit 11f7cff9280933067fba1c736eac9d05b9bfca79 diff --git a/vendor/nim-web3 b/vendor/nim-web3 index 428b931e7c..62a0005b09 160000 --- a/vendor/nim-web3 +++ b/vendor/nim-web3 @@ -1 +1 @@ -Subproject commit 428b931e7c4f1284b4272bc2c11fca2bd70991cd +Subproject commit 62a0005b0907a64090827d4e5d691682587f5b2a diff --git a/vendor/nim-websock b/vendor/nim-websock index 63bcc2902d..179f81deda 160000 --- a/vendor/nim-websock +++ b/vendor/nim-websock @@ -1 +1 @@ -Subproject commit 63bcc2902d884c63101e144555ad99421734a70a +Subproject commit 179f81dedaddb5ba8d02534ccc8b7a8335981f49 diff --git a/vendor/nim-zlib b/vendor/nim-zlib index 45b06fca15..02311a3562 160000 --- a/vendor/nim-zlib +++ b/vendor/nim-zlib @@ -1 +1 @@ -Subproject commit 45b06fca15ce0f09586067d950da30c10227865a +Subproject commit 02311a35623964a3ef37da8cd896ed95be06e6da diff --git a/vendor/nimcrypto b/vendor/nimcrypto index 71bca15508..a0b65f2a0d 160000 --- a/vendor/nimcrypto +++ b/vendor/nimcrypto @@ -1 +1 @@ -Subproject commit 71bca15508e2c0548f32b42a69bcfb1ccd9ab9ff +Subproject commit a0b65f2a0dba987c75eadf03f811fb5508fd3372 diff --git a/waku.nimble b/waku.nimble index fccb6f6b11..48a8bb1486 100644 --- a/waku.nimble +++ b/waku.nimble @@ -23,7 +23,9 @@ requires "nim >= 2.0.8", "web3", "presto", "regex", - "db_connector" + "db_connector", + "minilru", + "quic" ### Helper functions proc buildModule(filePath, params = "", lang = "c"): bool = diff --git a/waku/common/enr/builder.nim b/waku/common/enr/builder.nim index f4b3b472fc..49e9064135 100644 --- a/waku/common/enr/builder.nim +++ b/waku/common/enr/builder.nim @@ -41,9 +41,9 @@ proc build*(builder: EnrBuilder): EnrResult[enr.Record] = enr.Record.init( seqNum = builder.seqNumber, pk = builder.privateKey, - ip = none(IpAddress), - tcpPort = none(Port), - udpPort = none(Port), + ip = Opt.none(IpAddress), + tcpPort = Opt.none(Port), + udpPort = Opt.none(Port), extraFields = builder.fields, ) diff --git a/waku/common/enr/typed_record.nim b/waku/common/enr/typed_record.nim index 23ff6cc896..261e56d9a3 100644 --- a/waku/common/enr/typed_record.nim +++ b/waku/common/enr/typed_record.nim @@ -4,6 +4,19 @@ import std/options, results, eth/keys as eth_keys, libp2p/crypto/crypto as libp2 import eth/p2p/discoveryv5/enr except TypedRecord, toTypedRecord +## Since enr changed to result.Opt[T] from Option[T] for intercompatibility introduce a conversion between +func toOpt*[T](o: Option[T]): Opt[T] = + if o.isSome(): + return Opt.some(o.get()) + else: + return Opt.none(T) + +func toOption*[T](o: Opt[T]): Option[T] = + if o.isSome(): + return some(o.get()) + else: + return none(T) + ## ENR typed record # Record identity scheme @@ -31,7 +44,7 @@ proc init(T: type TypedRecord, record: Record): T = TypedRecord(raw: record) proc tryGet*(record: TypedRecord, field: string, T: type): Option[T] = - record.raw.tryGet(field, T) + return record.raw.tryGet(field, T).toOption() func toTyped*(record: Record): EnrResult[TypedRecord] = let tr = TypedRecord.init(record) diff --git a/waku/common/utils/nat.nim b/waku/common/utils/nat.nim index 893e8d8a82..355f63aec3 100644 --- a/waku/common/utils/nat.nim +++ b/waku/common/utils/nat.nim @@ -44,7 +44,7 @@ proc setupNat*( endpoint.ip = some(extIp.get()) # RedirectPorts in considered a gcsafety violation # because it obtains the address of a non-gcsafe proc? - var extPorts: Option[(Port, Port)] + var extPorts: Opt[(Port, Port)] try: extPorts = ( {.gcsafe.}: @@ -55,7 +55,7 @@ proc setupNat*( except CatchableError: # TODO: nat.nim Error: can raise an unlisted exception: Exception. Isolate here for now. error "unable to determine external ports" - extPorts = none((Port, Port)) + extPorts = Opt.none((Port, Port)) if extPorts.isSome(): let (extTcpPort, extUdpPort) = extPorts.get() diff --git a/waku/discovery/waku_discv5.nim b/waku/discovery/waku_discv5.nim index 5841b56859..badbe9c027 100644 --- a/waku/discovery/waku_discv5.nim +++ b/waku/discovery/waku_discv5.nim @@ -87,10 +87,10 @@ proc new*( privKey = conf.privateKey, bootstrapRecords = conf.bootstrapRecords, enrAutoUpdate = conf.autoupdateRecord, - previousRecord = record, - enrIp = none(IpAddress), - enrTcpPort = none(Port), - enrUdpPort = none(Port), + previousRecord = record.toOpt(), + enrIp = Opt.none(IpAddress), + enrTcpPort = Opt.none(Port), + enrUdpPort = Opt.none(Port), ) let shardPredOp = diff --git a/waku/waku_rln_relay/conversion_utils.nim b/waku/waku_rln_relay/conversion_utils.nim index d8be773d69..e710fea62b 100644 --- a/waku/waku_rln_relay/conversion_utils.nim +++ b/waku/waku_rln_relay/conversion_utils.nim @@ -128,3 +128,8 @@ proc fromEpoch*(epoch: Epoch): uint64 = ## decodes bytes of `epoch` (in little-endian) to uint64 let t = fromBytesLE(uint64, array[32, byte](epoch)) return t + +func `+`*(a, b: Quantity): Quantity {.borrow.} + +func u256*(n: Quantity): UInt256 {.inline.} = + n.uint64.stuint(256) diff --git a/waku/waku_rln_relay/group_manager/on_chain/group_manager.nim b/waku/waku_rln_relay/group_manager/on_chain/group_manager.nim index 1fada3a956..5415595fc7 100644 --- a/waku/waku_rln_relay/group_manager/on_chain/group_manager.nim +++ b/waku/waku_rln_relay/group_manager/on_chain/group_manager.nim @@ -3,10 +3,11 @@ import os, web3, - web3/ethtypes, + web3/eth_api_types, + web3/primitives, eth/keys as keys, chronicles, - nimcrypto/keccak, + nimcrypto/keccak as keccak, stint, json, std/tables, @@ -30,11 +31,11 @@ logScope: # using the when predicate does not work within the contract macro, hence need to dupe contract(WakuRlnContract): # this serves as an entrypoint into the rln membership set - proc register(idCommitment: UInt256, userMessageLimit: UInt32) + proc register(idCommitment: UInt256, userMessageLimit: StUint[32]) # Initializes the implementation contract (only used in unit tests) proc initialize(maxMessageLimit: UInt256) # this event is raised when a new member is registered - proc MemberRegistered(rateCommitment: UInt256, index: Uint32) {.event.} + proc MemberRegistered(rateCommitment: UInt256, index: StUint[32]) {.event.} # this function denotes existence of a given user proc memberExists(idCommitment: Uint256): UInt256 {.view.} @@ -100,7 +101,7 @@ proc setMetadata*( try: let metadataSetRes = g.rlnInstance.setMetadata( RlnMetadata( - lastProcessedBlock: normalizedBlock, + lastProcessedBlock: normalizedBlock.uint64, chainId: g.chainId, contractAddress: g.ethContractAddress, validRoots: g.validRoots.toSeq(), @@ -190,15 +191,15 @@ method register*( g.registrationTxHash = some(txHash) # the receipt topic holds the hash of signature of the raised events # TODO: make this robust. search within the event list for the event - debug "ts receipt", tsReceipt + # debug "ts receipt", tsReceipt let firstTopic = tsReceipt.logs[0].topics[0] # the hash of the signature of MemberRegistered(uint256,uint32) event is equal to the following hex value if firstTopic != - cast[FixedBytes[32]](keccak256.digest("MemberRegistered(uint256,uint32)").data): + cast[FixedBytes[32]](keccak.keccak256.digest("MemberRegistered(uint256,uint32)").data): raise newException(ValueError, "unexpected event signature") # the arguments of the raised event i.e., MemberRegistered are encoded inside the data field - # data = rateCommitment encoded as 256 bits || index encoded as 32 bits + # data = rateCommitment encoded as 256 bits || index encoded as 32 bits let arguments = tsReceipt.logs[0].data debug "tx log data", arguments = arguments let @@ -231,11 +232,10 @@ proc parseEvent( ## returns an error if it cannot parse the `data` parameter var rateCommitment: UInt256 var index: UInt256 - var data: string - # Remove the 0x prefix + var data: seq[byte] try: - data = strip0xPrefix(log["data"].getStr()) - except CatchableError: + data = hexToSeqByte(log["data"].getStr()) + except ValueError: return err( "failed to parse the data field of the MemberRegistered event: " & getCurrentExceptionMsg() @@ -243,9 +243,9 @@ proc parseEvent( var offset = 0 try: # Parse the rateCommitment - offset += decode(data, offset, rateCommitment) + offset += decode(data, 0, offset, rateCommitment) # Parse the index - offset += decode(data, offset, index) + offset += decode(data, 0, offset, index) return ok( Membership( rateCommitment: rateCommitment.toRateCommitment(), @@ -291,13 +291,18 @@ proc getRawEvents( let ethRpc = g.ethRpc.get() let wakuRlnContract = g.wakuRlnContract.get() - var events: JsonNode - g.retryWrapper(events, "Failed to get the events"): + var eventStrs: seq[JsonString] + g.retryWrapper(eventStrs, "Failed to get the events"): await wakuRlnContract.getJsonLogs( MemberRegistered, - fromBlock = some(fromBlock.blockId()), - toBlock = some(toBlock.blockId()), + fromBlock = Opt.some(fromBlock.blockId()), + toBlock = Opt.some(toBlock.blockId()), ) + + var events: JsonNode + if events.len == 0: + for eventStr in eventStrs: + events.add(parseJson($eventStr)) return events proc getBlockTable( @@ -314,7 +319,7 @@ proc getBlockTable( return blockTable for event in events: - let blockNumber = parseHexInt(event["blockNumber"].getStr()).uint + let blockNumber = parseHexInt(event["blockNumber"].getStr()).Quantity let removed = event["removed"].getBool() let parsedEventRes = parseEvent(MemberRegistered, event) if parsedEventRes.isErr(): @@ -450,13 +455,13 @@ proc startOnchainSync( let ethRpc = g.ethRpc.get() # static block chunk size - let blockChunkSize = 2_000 + let blockChunkSize = 2_000.BlockNumber # delay between rpc calls to not overload the rate limit let rpcDelay = 200.milliseconds # max number of futures to run concurrently let maxFutures = 10 - var fromBlock = + var fromBlock: BlockNumber = if g.latestProcessedBlock > g.rlnContractDeployedBlockNumber: info "syncing from last processed block", blockNumber = g.latestProcessedBlock g.latestProcessedBlock + 1 @@ -479,11 +484,11 @@ proc startOnchainSync( if fromBlock >= currentLatestBlock: break - if fromBlock + blockChunkSize.uint > currentLatestBlock.uint: + if fromBlock + blockChunkSize > currentLatestBlock: g.retryWrapper(currentLatestBlock, "Failed to get the latest block number"): cast[BlockNumber](await ethRpc.provider.eth_blockNumber()) - let toBlock = min(fromBlock + BlockNumber(blockChunkSize), currentLatestBlock) + let toBlock = min(fromBlock + blockChunkSize, currentLatestBlock) debug "fetching events", fromBlock = fromBlock, toBlock = toBlock await sleepAsync(rpcDelay) futs.add(g.getAndHandleEvents(fromBlock, toBlock)) @@ -551,7 +556,7 @@ method init*(g: OnchainGroupManager): Future[GroupManagerResult[void]] {.async.} let pk = g.ethPrivateKey.get() let parsedPk = keys.PrivateKey.fromHex(pk).valueOr: return err("failed to parse the private key" & ": " & $error) - ethRpc.privateKey = some(parsedPk) + ethRpc.privateKey = Opt.some(parsedPk) ethRpc.defaultAccount = ethRpc.privateKey.get().toPublicKey().toCanonicalAddress().Address @@ -605,7 +610,7 @@ method init*(g: OnchainGroupManager): Future[GroupManagerResult[void]] {.async.} if metadata.contractAddress != g.ethContractAddress.toLower(): return err("persisted data: contract address mismatch") - g.latestProcessedBlock = metadata.lastProcessedBlock + g.latestProcessedBlock = metadata.lastProcessedBlock.BlockNumber g.validRoots = metadata.validRoots.toDeque() var deployedBlockNumber: Uint256 @@ -661,10 +666,10 @@ method stop*(g: OnchainGroupManager): Future[void] {.async, gcsafe.} = proc isSyncing*(g: OnchainGroupManager): Future[bool] {.async, gcsafe.} = let ethRpc = g.ethRpc.get() - var syncing: JsonNode + var syncing: SyncingStatus g.retryWrapper(syncing, "Failed to get the syncing status"): await ethRpc.provider.eth_syncing() - return syncing.getBool() + return syncing.syncing method isReady*(g: OnchainGroupManager): Future[bool] {.async.} = initializedGuard(g) @@ -677,7 +682,7 @@ method isReady*(g: OnchainGroupManager): Future[bool] {.async.} = cast[BlockNumber](await g.ethRpc.get().provider.eth_blockNumber()) # the node is still able to process messages if it is behind the latest block by a factor of the valid roots - if u256(g.latestProcessedBlock) < (u256(currentBlock) - u256(g.validRoots.len)): + if u256(g.latestProcessedBlock.uint64) < (u256(currentBlock) - u256(g.validRoots.len)): return false return not (await g.isSyncing()) diff --git a/waku/waku_rln_relay/rln/wrappers.nim b/waku/waku_rln_relay/rln/wrappers.nim index 3d2b62e21c..24682dda63 100644 --- a/waku/waku_rln_relay/rln/wrappers.nim +++ b/waku/waku_rln_relay/rln/wrappers.nim @@ -87,7 +87,7 @@ proc createRLNInstanceLocal( ## generates an instance of RLN ## An RLN instance supports both zkSNARKs logics and Merkle tree data structure and operations ## d indicates the depth of Merkle tree - ## tree_path indicates the path of the Merkle tree + ## tree_path indicates the path of the Merkle tree ## Returns an error if the instance creation fails let rln_config = RlnConfig( diff --git a/waku/waku_rln_relay/rln_relay.nim b/waku/waku_rln_relay/rln_relay.nim index 0c86f0746c..535cee4a2d 100644 --- a/waku/waku_rln_relay/rln_relay.nim +++ b/waku/waku_rln_relay/rln_relay.nim @@ -8,7 +8,7 @@ import stint, web3, json, - web3/ethtypes, + web3/eth_api_types, eth/keys, libp2p/protocols/pubsub/rpc/messages, libp2p/protocols/pubsub/pubsub, @@ -277,7 +277,7 @@ proc validateMessageAndUpdateLog*( if proofMetadataRes.isErr(): return MessageValidationResult.Invalid - # insert the message to the log (never errors) only if the + # insert the message to the log (never errors) only if the # message is valid. if isValidMessage == MessageValidationResult.Valid: discard rlnPeer.updateLog(msgProof.epoch, proofMetadataRes.get()) From a5189d05664d8a93b9753b98271a685ad0ec6029 Mon Sep 17 00:00:00 2001 From: NagyZoltanPeter <113987313+NagyZoltanPeter@users.noreply.github.com> Date: Wed, 9 Oct 2024 23:18:44 +0200 Subject: [PATCH 2/5] Eliminate C compilation issue with chat2bridge due to an overcomplicating import from json_rpc instead of using std/json --- apps/chat2bridge/chat2bridge.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/chat2bridge/chat2bridge.nim b/apps/chat2bridge/chat2bridge.nim index 24533821d2..ce6152f30e 100644 --- a/apps/chat2bridge/chat2bridge.nim +++ b/apps/chat2bridge/chat2bridge.nim @@ -1,7 +1,7 @@ {.push raises: [].} import - std/[tables, times, strutils, hashes, sequtils], + std/[tables, times, strutils, hashes, sequtils, json], chronos, confutils, chronicles, @@ -11,7 +11,6 @@ import metrics/chronos_httpserver, stew/byteutils, eth/net/nat, - json_rpc/rpcserver, # Matterbridge client imports # Waku v2 imports libp2p/crypto/crypto, From 0a91ab6eb5a6e804eb27081cc85d772e4e8f5a40 Mon Sep 17 00:00:00 2001 From: NagyZoltanPeter <113987313+NagyZoltanPeter@users.noreply.github.com> Date: Mon, 21 Oct 2024 15:33:50 +0200 Subject: [PATCH 3/5] Adapt ENR Record handling to new interface of nim-eth --- tests/common/test_enr_builder.nim | 2 +- waku/common/enr/builder.nim | 54 ++++++++++++------------------- waku/common/enr/typed_record.nim | 10 ++++-- waku/factory/external_config.nim | 2 +- 4 files changed, 31 insertions(+), 37 deletions(-) diff --git a/tests/common/test_enr_builder.nim b/tests/common/test_enr_builder.nim index 6320c64032..b95828bb4d 100644 --- a/tests/common/test_enr_builder.nim +++ b/tests/common/test_enr_builder.nim @@ -120,7 +120,7 @@ suite "nim-eth ENR - Ext: IP address and TCP/UDP ports": @(record.secp256k1.get()) == expectedPubKey record.ip == none(array[4, byte]) record.tcp == none(uint16) - record.udp == none(uint16) record.ip6 == some(enrIpAddr.address_v6) record.tcp6 == none(uint16) record.udp6 == some(enrUdpPort.uint16) + record.udp == some(enrUdpPort.uint16) diff --git a/waku/common/enr/builder.nim b/waku/common/enr/builder.nim index 49e9064135..3c5455d52f 100644 --- a/waku/common/enr/builder.nim +++ b/waku/common/enr/builder.nim @@ -7,15 +7,27 @@ import eth/p2p/discoveryv5/enr, libp2p/crypto/crypto as libp2p_crypto +import ./typed_record + ## Builder type EnrBuilder* = object seqNumber: uint64 privateKey: eth_keys.PrivateKey + ipAddress: Opt[IpAddress] + tcpPort: Opt[Port] + udpPort: Opt[Port] fields: seq[FieldPair] proc init*(T: type EnrBuilder, key: eth_keys.PrivateKey, seqNum: uint64 = 1): T = - EnrBuilder(seqNumber: seqNum, privateKey: key, fields: newSeq[FieldPair]()) + EnrBuilder( + seqNumber: seqNum, + privateKey: key, + ipAddress: Opt.none(IpAddress), + tcpPort: Opt.none(Port), + udpPort: Opt.none(Port), + fields: newSeq[FieldPair](), + ) proc init*(T: type EnrBuilder, key: libp2p_crypto.PrivateKey, seqNum: uint64 = 1): T = # TODO: Inconvenient runtime assertion. Move this assertion to compile time @@ -41,9 +53,9 @@ proc build*(builder: EnrBuilder): EnrResult[enr.Record] = enr.Record.init( seqNum = builder.seqNumber, pk = builder.privateKey, - ip = Opt.none(IpAddress), - tcpPort = Opt.none(Port), - udpPort = Opt.none(Port), + ip = builder.ipAddress, + tcpPort = builder.tcpPort, + udpPort = builder.udpPort, extraFields = builder.fields, ) @@ -52,38 +64,14 @@ proc build*(builder: EnrBuilder): EnrResult[enr.Record] = proc addAddressAndPorts( builder: var EnrBuilder, ip: IpAddress, tcpPort, udpPort: Option[Port] ) = - # Based on: https://github.com/status-im/nim-eth/blob/4b22fcd/eth/p2p/discoveryv5/enr.nim#L166 - let isV6 = ip.family == IPv6 - - let ipField = - if isV6: - toFieldPair("ip6", ip.address_v6) - else: - toFieldPair("ip", ip.address_v4) - builder.addFieldPair(ipField) - - if tcpPort.isSome(): - let - tcpPortFieldKey = if isV6: "tcp6" else: "tcp" - tcpPortFieldValue = tcpPort.get() - builder.addFieldPair(tcpPortFieldKey, tcpPortFieldValue.uint16) - - if udpPort.isSome(): - let - udpPortFieldKey = if isV6: "udp6" else: "udp" - udpPortFieldValue = udpPort.get() - builder.addFieldPair(udpPortFieldKey, udpPortFieldValue.uint16) + builder.ipAddress = Opt.some(ip) + builder.tcpPort = tcpPort.toOpt() + builder.udpPort = udpPort.toOpt() proc addPorts(builder: var EnrBuilder, tcp, udp: Option[Port]) = # Based on: https://github.com/status-im/nim-eth/blob/4b22fcd/eth/p2p/discoveryv5/enr.nim#L166 - - if tcp.isSome(): - let tcpPort = tcp.get() - builder.addFieldPair("tcp", tcpPort.uint16) - - if udp.isSome(): - let udpPort = udp.get() - builder.addFieldPair("udp", udpPort.uint16) + builder.tcpPort = tcp.toOpt() + builder.udpPort = udp.toOpt() proc withIpAddressAndPorts*( builder: var EnrBuilder, diff --git a/waku/common/enr/typed_record.nim b/waku/common/enr/typed_record.nim index 261e56d9a3..d0b055ac45 100644 --- a/waku/common/enr/typed_record.nim +++ b/waku/common/enr/typed_record.nim @@ -84,10 +84,16 @@ func tcp*(record: TypedRecord): Option[uint16] = record.tryGet("tcp", uint16) func tcp6*(record: TypedRecord): Option[uint16] = - record.tryGet("tcp6", uint16) + let port = record.tryGet("tcp6", uint16) + if port.isNone(): + return record.tcp() + return port func udp*(record: TypedRecord): Option[uint16] = record.tryGet("udp", uint16) func udp6*(record: TypedRecord): Option[uint16] = - record.tryGet("udp6", uint16) + let port = record.tryGet("udp6", uint16) + if port.isNone(): + return record.udp() + return port diff --git a/waku/factory/external_config.nim b/waku/factory/external_config.nim index 2446352c10..904dee509c 100644 --- a/waku/factory/external_config.nim +++ b/waku/factory/external_config.nim @@ -160,7 +160,7 @@ type WakuNodeConf* = object .}: uint16 agentString* {. - defaultValue: "nwaku-" & git_version, + defaultValue: "nwaku-" & external_config.git_version, desc: "Node agent string which is used as identifier in network", name: "agent-string" .}: string From d35f15738f5949b731d15cc527e0d2c7bc312b01 Mon Sep 17 00:00:00 2001 From: NagyZoltanPeter <113987313+NagyZoltanPeter@users.noreply.github.com> Date: Mon, 28 Oct 2024 11:27:34 +0100 Subject: [PATCH 4/5] Fix chrash in group_manager on_chain --- tests/waku_rln_relay/utils.nim | 2 +- .../group_manager/on_chain/group_manager.nim | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/waku_rln_relay/utils.nim b/tests/waku_rln_relay/utils.nim index 2086acb88e..46ab8392c6 100644 --- a/tests/waku_rln_relay/utils.nim +++ b/tests/waku_rln_relay/utils.nim @@ -23,7 +23,7 @@ proc deployContract*( var tr: TransactionArgs tr.`from` = Opt.some(web3.defaultAccount) let sData = code & contractInput - tr.data = Opt.some(sData.toBytes()) + tr.data = Opt.some(hexToSeqByte(sData)) tr.gas = Opt.some(Quantity(3000000000000)) if gasPrice != 0: tr.gasPrice = Opt.some(gasPrice.Quantity) diff --git a/waku/waku_rln_relay/group_manager/on_chain/group_manager.nim b/waku/waku_rln_relay/group_manager/on_chain/group_manager.nim index 5415595fc7..b47302f60d 100644 --- a/waku/waku_rln_relay/group_manager/on_chain/group_manager.nim +++ b/waku/waku_rln_relay/group_manager/on_chain/group_manager.nim @@ -191,7 +191,7 @@ method register*( g.registrationTxHash = some(txHash) # the receipt topic holds the hash of signature of the raised events # TODO: make this robust. search within the event list for the event - # debug "ts receipt", tsReceipt + debug "ts receipt", receipt = tsReceipt[] let firstTopic = tsReceipt.logs[0].topics[0] # the hash of the signature of MemberRegistered(uint256,uint32) event is equal to the following hex value if firstTopic != @@ -299,10 +299,9 @@ proc getRawEvents( toBlock = Opt.some(toBlock.blockId()), ) - var events: JsonNode - if events.len == 0: - for eventStr in eventStrs: - events.add(parseJson($eventStr)) + var events = newJArray() + for eventStr in eventStrs: + events.add(parseJson($eventStr)) return events proc getBlockTable( From b5df2ce30d431fecc9c03c628f455a01b01b5c78 Mon Sep 17 00:00:00 2001 From: NagyZoltanPeter <113987313+NagyZoltanPeter@users.noreply.github.com> Date: Wed, 30 Oct 2024 06:26:50 +0100 Subject: [PATCH 5/5] Fix signature of register and MemberRegister to UInt256, check transaction success in register --- .../test_rln_group_manager_onchain.nim | 18 +++++++++++++++++- .../group_manager/on_chain/group_manager.nim | 18 +++++++++++------- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/tests/waku_rln_relay/test_rln_group_manager_onchain.nim b/tests/waku_rln_relay/test_rln_group_manager_onchain.nim index 3e6980db63..cd22556832 100644 --- a/tests/waku_rln_relay/test_rln_group_manager_onchain.nim +++ b/tests/waku_rln_relay/test_rln_group_manager_onchain.nim @@ -29,9 +29,23 @@ import ./utils_onchain, ./utils +proc logAnvilOutput(runAnvil: Process) = + # NOTICE: only for temporary debugging purposes + # the readLine will block forever.... + var cmdline: string + while true: + try: + if runAnvil.outputstream.readLine(cmdline): + echo "Anvil log - " & cmdLine + else: + break + except Exception, CatchableError: + break + suite "Onchain group manager": # We run Anvil - let runAnvil {.used.} = runAnvil() + var runAnvil {.threadvar, used.}: Process + runAnvil = runAnvil() asyncTest "should initialize successfully": let manager = await setup() @@ -143,6 +157,7 @@ suite "Onchain group manager": asyncTest "startGroupSync: should sync to the state of the group": let manager = await setup() + let credentials = generateCredentials(manager.rlnInstance) let rateCommitment = getRateCommitment(credentials, UserMessageLimit(1)).valueOr: raiseAssert $error @@ -170,6 +185,7 @@ suite "Onchain group manager": (await manager.startGroupSync()).isOkOr: raiseAssert $error except Exception, CatchableError: + # logAnvilOutput(runAnvil) assert false, "exception raised: " & getCurrentExceptionMsg() await fut diff --git a/waku/waku_rln_relay/group_manager/on_chain/group_manager.nim b/waku/waku_rln_relay/group_manager/on_chain/group_manager.nim index b47302f60d..309267bb9d 100644 --- a/waku/waku_rln_relay/group_manager/on_chain/group_manager.nim +++ b/waku/waku_rln_relay/group_manager/on_chain/group_manager.nim @@ -31,11 +31,11 @@ logScope: # using the when predicate does not work within the contract macro, hence need to dupe contract(WakuRlnContract): # this serves as an entrypoint into the rln membership set - proc register(idCommitment: UInt256, userMessageLimit: StUint[32]) + proc register(idCommitment: UInt256, userMessageLimit: UInt256) # Initializes the implementation contract (only used in unit tests) proc initialize(maxMessageLimit: UInt256) # this event is raised when a new member is registered - proc MemberRegistered(rateCommitment: UInt256, index: StUint[32]) {.event.} + proc MemberRegistered(rateCommitment: UInt256, index: UInt256) {.event.} # this function denotes existence of a given user proc memberExists(idCommitment: Uint256): UInt256 {.view.} @@ -179,7 +179,7 @@ method register*( idCommitment = idCommitment, userMessageLimit = userMessageLimit var txHash: TxHash g.retryWrapper(txHash, "Failed to register the member"): - await wakuRlnContract.register(idCommitment, userMessageLimit.stuint(32)).send( + await wakuRlnContract.register(idCommitment, userMessageLimit.stuint(256)).send( gasPrice = gasPrice ) @@ -192,11 +192,15 @@ method register*( # the receipt topic holds the hash of signature of the raised events # TODO: make this robust. search within the event list for the event debug "ts receipt", receipt = tsReceipt[] + + if tsReceipt.status.isNone() or tsReceipt.status.get() != 1.Quantity: + raise newException(ValueError, "register: transaction failed") + let firstTopic = tsReceipt.logs[0].topics[0] # the hash of the signature of MemberRegistered(uint256,uint32) event is equal to the following hex value if firstTopic != - cast[FixedBytes[32]](keccak.keccak256.digest("MemberRegistered(uint256,uint32)").data): - raise newException(ValueError, "unexpected event signature") + cast[FixedBytes[32]](keccak.keccak256.digest("MemberRegistered(uint256,uint256)").data): + raise newException(ValueError, "register: unexpected event signature") # the arguments of the raised event i.e., MemberRegistered are encoded inside the data field # data = rateCommitment encoded as 256 bits || index encoded as 32 bits @@ -301,7 +305,7 @@ proc getRawEvents( var events = newJArray() for eventStr in eventStrs: - events.add(parseJson($eventStr)) + events.add(parseJson(eventStr.string)) return events proc getBlockTable( @@ -318,7 +322,7 @@ proc getBlockTable( return blockTable for event in events: - let blockNumber = parseHexInt(event["blockNumber"].getStr()).Quantity + let blockNumber = parseHexInt(event["blockNumber"].getStr()).BlockNumber let removed = event["removed"].getBool() let parsedEventRes = parseEvent(MemberRegistered, event) if parsedEventRes.isErr():