diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index fce5b1c8..81a62184 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -45,6 +45,8 @@ jobs: run: npm run test:inheritance - name: Check pragma consistency between files run: npm run test:pragma + - name: Check procedurally generated contracts are up-to-date + run: npm run test:generation coverage: runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 258554f8..c9ae9e5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## XX-XX-XXXX + +- `EnumerableSetExtended` and `EnumerableMapExtended`: Extensions of the `EnumerableSet` and `EnumerableMap` libraries with more types, including non-value types. + ## 03-04-2025 - `PaymasterERC20`: Extension of `PaymasterCore` that sponsors user operations against payment in ERC-20 tokens. diff --git a/contracts/utils/README.adoc b/contracts/utils/README.adoc index c82a83cf..4454470a 100644 --- a/contracts/utils/README.adoc +++ b/contracts/utils/README.adoc @@ -9,6 +9,7 @@ Miscellaneous contracts and libraries containing utility functions you can use t * {ERC7739}: An abstract contract to validate signatures following the rehashing scheme from `ERC7739Utils`. * {ERC7739Utils}: Utilities library that implements a defensive rehashing mechanism to prevent replayability of smart contract signatures based on ERC-7739. * {SignerECDSA}, {SignerP256}, {SignerRSA}: Implementations of an {AbstractSigner} with specific signature validation algorithms. + * {EnumerableSetExtended} and {EnumerableMapExtended}: Extensions of the `EnumerableSet` and `EnumerableMap` libraries with more types, including non-value types. * {Masks}: Library to handle `bytes32` masks. == Cryptography @@ -27,6 +28,12 @@ Miscellaneous contracts and libraries containing utility functions you can use t {{SignerRSA}} +== Structs + +{{EnumerableSetExtended}} + +{{EnumerableMapExtended}} + == Libraries {{Masks}} diff --git a/contracts/utils/structs/EnumerableMapExtended.sol b/contracts/utils/structs/EnumerableMapExtended.sol new file mode 100644 index 00000000..b568dec1 --- /dev/null +++ b/contracts/utils/structs/EnumerableMapExtended.sol @@ -0,0 +1,281 @@ +// SPDX-License-Identifier: MIT +// This file was procedurally generated from scripts/generate/templates/EnumerableMapExtended.js. + +pragma solidity ^0.8.20; + +import {EnumerableSet} from "@openzeppelin/contracts/utils/structs/EnumerableSet.sol"; +import {EnumerableSetExtended} from "./EnumerableSetExtended.sol"; + +/** + * @dev Library for managing an enumerable variant of Solidity's + * https://solidity.readthedocs.io/en/latest/types.html#mapping-types[`mapping`] + * type for non-value types as keys. + * + * Maps have the following properties: + * + * - Entries are added, removed, and checked for existence in constant time + * (O(1)). + * - Entries are enumerated in O(n). No guarantees are made on the ordering. + * - Map can be cleared (all entries removed) in O(n). + * + * ```solidity + * contract Example { + * // Add the library methods + * using EnumerableMapExtended for EnumerableMapExtended.BytesToUintMap; + * + * // Declare a set state variable + * EnumerableMapExtended.BytesToUintMap private myMap; + * } + * ``` + * + * The following map types are supported: + * + * - `bytes -> uint256` (`BytesToUintMap`) + * - `string -> string` (`StringToStringMap`) + * + * [WARNING] + * ==== + * Trying to delete such a structure from storage will likely result in data corruption, rendering the structure + * unusable. + * See https://github.com/ethereum/solidity/pull/11843[ethereum/solidity#11843] for more info. + * + * In order to clean an EnumerableMap, you can either remove all elements one by one or create a fresh instance using an + * array of EnumerableMap. + * ==== + * + * NOTE: Extensions of openzeppelin/contracts/utils/struct/EnumerableMap.sol. + */ +library EnumerableMapExtended { + using EnumerableSet for *; + using EnumerableSetExtended for *; + + /** + * @dev Query for a nonexistent map key. + */ + error EnumerableMapNonexistentBytesKey(bytes key); + + struct BytesToUintMap { + // Storage of keys + EnumerableSetExtended.BytesSet _keys; + mapping(bytes key => uint256) _values; + } + + /** + * @dev Adds a key-value pair to a map, or updates the value for an existing + * key. O(1). + * + * Returns true if the key was added to the map, that is if it was not + * already present. + */ + function set(BytesToUintMap storage map, bytes memory key, uint256 value) internal returns (bool) { + map._values[key] = value; + return map._keys.add(key); + } + + /** + * @dev Removes a key-value pair from a map. O(1). + * + * Returns true if the key was removed from the map, that is if it was present. + */ + function remove(BytesToUintMap storage map, bytes memory key) internal returns (bool) { + delete map._values[key]; + return map._keys.remove(key); + } + + /** + * @dev Removes all the entries from a map. O(n). + * + * WARNING: Developers should keep in mind that this function has an unbounded cost and using it may render the + * function uncallable if the map grows to the point where clearing it consumes too much gas to fit in a block. + */ + function clear(BytesToUintMap storage map) internal { + uint256 len = length(map); + for (uint256 i = 0; i < len; ++i) { + delete map._values[map._keys.at(i)]; + } + map._keys.clear(); + } + + /** + * @dev Returns true if the key is in the map. O(1). + */ + function contains(BytesToUintMap storage map, bytes memory key) internal view returns (bool) { + return map._keys.contains(key); + } + + /** + * @dev Returns the number of key-value pairs in the map. O(1). + */ + function length(BytesToUintMap storage map) internal view returns (uint256) { + return map._keys.length(); + } + + /** + * @dev Returns the key-value pair stored at position `index` in the map. O(1). + * + * Note that there are no guarantees on the ordering of entries inside the + * array, and it may change when more entries are added or removed. + * + * Requirements: + * + * - `index` must be strictly less than {length}. + */ + function at(BytesToUintMap storage map, uint256 index) internal view returns (bytes memory key, uint256 value) { + key = map._keys.at(index); + value = map._values[key]; + } + + /** + * @dev Tries to returns the value associated with `key`. O(1). + * Does not revert if `key` is not in the map. + */ + function tryGet(BytesToUintMap storage map, bytes memory key) internal view returns (bool exists, uint256 value) { + value = map._values[key]; + exists = value != uint256(0) || contains(map, key); + } + + /** + * @dev Returns the value associated with `key`. O(1). + * + * Requirements: + * + * - `key` must be in the map. + */ + function get(BytesToUintMap storage map, bytes memory key) internal view returns (uint256 value) { + bool exists; + (exists, value) = tryGet(map, key); + if (!exists) { + revert EnumerableMapNonexistentBytesKey(key); + } + } + + /** + * @dev Return the an array containing all the keys + * + * WARNING: This operation will copy the entire storage to memory, which can be quite expensive. This is designed + * to mostly be used by view accessors that are queried without any gas fees. Developers should keep in mind that + * this function has an unbounded cost, and using it as part of a state-changing function may render the function + * uncallable if the map grows to a point where copying to memory consumes too much gas to fit in a block. + */ + function keys(BytesToUintMap storage map) internal view returns (bytes[] memory) { + return map._keys.values(); + } + + /** + * @dev Query for a nonexistent map key. + */ + error EnumerableMapNonexistentStringKey(string key); + + struct StringToStringMap { + // Storage of keys + EnumerableSetExtended.StringSet _keys; + mapping(string key => string) _values; + } + + /** + * @dev Adds a key-value pair to a map, or updates the value for an existing + * key. O(1). + * + * Returns true if the key was added to the map, that is if it was not + * already present. + */ + function set(StringToStringMap storage map, string memory key, string memory value) internal returns (bool) { + map._values[key] = value; + return map._keys.add(key); + } + + /** + * @dev Removes a key-value pair from a map. O(1). + * + * Returns true if the key was removed from the map, that is if it was present. + */ + function remove(StringToStringMap storage map, string memory key) internal returns (bool) { + delete map._values[key]; + return map._keys.remove(key); + } + + /** + * @dev Removes all the entries from a map. O(n). + * + * WARNING: Developers should keep in mind that this function has an unbounded cost and using it may render the + * function uncallable if the map grows to the point where clearing it consumes too much gas to fit in a block. + */ + function clear(StringToStringMap storage map) internal { + uint256 len = length(map); + for (uint256 i = 0; i < len; ++i) { + delete map._values[map._keys.at(i)]; + } + map._keys.clear(); + } + + /** + * @dev Returns true if the key is in the map. O(1). + */ + function contains(StringToStringMap storage map, string memory key) internal view returns (bool) { + return map._keys.contains(key); + } + + /** + * @dev Returns the number of key-value pairs in the map. O(1). + */ + function length(StringToStringMap storage map) internal view returns (uint256) { + return map._keys.length(); + } + + /** + * @dev Returns the key-value pair stored at position `index` in the map. O(1). + * + * Note that there are no guarantees on the ordering of entries inside the + * array, and it may change when more entries are added or removed. + * + * Requirements: + * + * - `index` must be strictly less than {length}. + */ + function at( + StringToStringMap storage map, + uint256 index + ) internal view returns (string memory key, string memory value) { + key = map._keys.at(index); + value = map._values[key]; + } + + /** + * @dev Tries to returns the value associated with `key`. O(1). + * Does not revert if `key` is not in the map. + */ + function tryGet( + StringToStringMap storage map, + string memory key + ) internal view returns (bool exists, string memory value) { + value = map._values[key]; + exists = bytes(value).length != 0 || contains(map, key); + } + + /** + * @dev Returns the value associated with `key`. O(1). + * + * Requirements: + * + * - `key` must be in the map. + */ + function get(StringToStringMap storage map, string memory key) internal view returns (string memory value) { + bool exists; + (exists, value) = tryGet(map, key); + if (!exists) { + revert EnumerableMapNonexistentStringKey(key); + } + } + + /** + * @dev Return the an array containing all the keys + * + * WARNING: This operation will copy the entire storage to memory, which can be quite expensive. This is designed + * to mostly be used by view accessors that are queried without any gas fees. Developers should keep in mind that + * this function has an unbounded cost, and using it as part of a state-changing function may render the function + * uncallable if the map grows to a point where copying to memory consumes too much gas to fit in a block. + */ + function keys(StringToStringMap storage map) internal view returns (string[] memory) { + return map._keys.values(); + } +} diff --git a/contracts/utils/structs/EnumerableSetExtended.sol b/contracts/utils/structs/EnumerableSetExtended.sol new file mode 100644 index 00000000..cc38406b --- /dev/null +++ b/contracts/utils/structs/EnumerableSetExtended.sol @@ -0,0 +1,415 @@ +// SPDX-License-Identifier: MIT +// This file was procedurally generated from scripts/generate/templates/EnumerableSetExtended.js. + +pragma solidity ^0.8.20; + +import {Arrays} from "@openzeppelin/contracts/utils/Arrays.sol"; +import {Hashes} from "@openzeppelin/contracts/utils/cryptography/Hashes.sol"; + +/** + * @dev Library for managing + * https://en.wikipedia.org/wiki/Set_(abstract_data_type)[sets] of non-value + * types. + * + * Sets have the following properties: + * + * - Elements are added, removed, and checked for existence in constant time + * (O(1)). + * - Elements are enumerated in O(n). No guarantees are made on the ordering. + * - Set can be cleared (all elements removed) in O(n). + * + * ```solidity + * contract Example { + * // Add the library methods + * using EnumerableSetExtended for EnumerableSetExtended.StringSet; + * + * // Declare a set state variable + * EnumerableSetExtended.StringSet private mySet; + * } + * ``` + * + * Sets of type `string` (`StringSet`), `bytes` (`BytesSet`) and + * `bytes32[2]` (`Bytes32x2Set`) are supported. + * + * [WARNING] + * ==== + * Trying to delete such a structure from storage will likely result in data corruption, rendering the structure + * unusable. + * See https://github.com/ethereum/solidity/pull/11843[ethereum/solidity#11843] for more info. + * + * In order to clean an EnumerableSet, you can either remove all elements one by one or create a fresh instance using an + * array of EnumerableSet. + * ==== + * + * NOTE: This is an extension of openzeppelin/contracts/utils/struct/EnumerableSet.sol. + */ +library EnumerableSetExtended { + struct StringSet { + // Storage of set values + string[] _values; + // Position is the index of the value in the `values` array plus 1. + // Position 0 is used to mean a value is not in the set. + mapping(string value => uint256) _positions; + } + + /** + * @dev Add a value to a set. O(1). + * + * Returns true if the value was added to the set, that is if it was not + * already present. + */ + function add(StringSet storage self, string memory value) internal returns (bool) { + if (!contains(self, value)) { + self._values.push(value); + // The value is stored at length-1, but we add 1 to all indexes + // and use 0 as a sentinel value + self._positions[value] = self._values.length; + return true; + } else { + return false; + } + } + + /** + * @dev Removes a value from a set. O(1). + * + * Returns true if the value was removed from the set, that is if it was + * present. + */ + function remove(StringSet storage self, string memory value) internal returns (bool) { + // We cache the value's position to prevent multiple reads from the same storage slot + uint256 position = self._positions[value]; + + if (position != 0) { + // Equivalent to contains(self, value) + // To delete an element from the _values array in O(1), we swap the element to delete with the last one in + // the array, and then remove the last element (sometimes called as 'swap and pop'). + // This modifies the order of the array, as noted in {at}. + + uint256 valueIndex = position - 1; + uint256 lastIndex = self._values.length - 1; + + if (valueIndex != lastIndex) { + string memory lastValue = self._values[lastIndex]; + + // Move the lastValue to the index where the value to delete is + self._values[valueIndex] = lastValue; + // Update the tracked position of the lastValue (that was just moved) + self._positions[lastValue] = position; + } + + // Delete the slot where the moved value was stored + self._values.pop(); + + // Delete the tracked position for the deleted slot + delete self._positions[value]; + + return true; + } else { + return false; + } + } + + /** + * @dev Removes all the values from a set. O(n). + * + * WARNING: Developers should keep in mind that this function has an unbounded cost and using it may render the + * function uncallable if the set grows to the point where clearing it consumes too much gas to fit in a block. + */ + function clear(StringSet storage set) internal { + uint256 len = length(set); + for (uint256 i = 0; i < len; ++i) { + delete set._positions[set._values[i]]; + } + Arrays.unsafeSetLength(set._values, 0); + } + + /** + * @dev Returns true if the value is in the set. O(1). + */ + function contains(StringSet storage self, string memory value) internal view returns (bool) { + return self._positions[value] != 0; + } + + /** + * @dev Returns the number of values on the set. O(1). + */ + function length(StringSet storage self) internal view returns (uint256) { + return self._values.length; + } + + /** + * @dev Returns the value stored at position `index` in the set. O(1). + * + * Note that there are no guarantees on the ordering of values inside the + * array, and it may change when more values are added or removed. + * + * Requirements: + * + * - `index` must be strictly less than {length}. + */ + function at(StringSet storage self, uint256 index) internal view returns (string memory) { + return self._values[index]; + } + + /** + * @dev Return the entire set in an array + * + * WARNING: This operation will copy the entire storage to memory, which can be quite expensive. This is designed + * to mostly be used by view accessors that are queried without any gas fees. Developers should keep in mind that + * this function has an unbounded cost, and using it as part of a state-changing function may render the function + * uncallable if the set grows to a point where copying to memory consumes too much gas to fit in a block. + */ + function values(StringSet storage self) internal view returns (string[] memory) { + return self._values; + } + + struct BytesSet { + // Storage of set values + bytes[] _values; + // Position is the index of the value in the `values` array plus 1. + // Position 0 is used to mean a value is not in the set. + mapping(bytes value => uint256) _positions; + } + + /** + * @dev Add a value to a set. O(1). + * + * Returns true if the value was added to the set, that is if it was not + * already present. + */ + function add(BytesSet storage self, bytes memory value) internal returns (bool) { + if (!contains(self, value)) { + self._values.push(value); + // The value is stored at length-1, but we add 1 to all indexes + // and use 0 as a sentinel value + self._positions[value] = self._values.length; + return true; + } else { + return false; + } + } + + /** + * @dev Removes a value from a set. O(1). + * + * Returns true if the value was removed from the set, that is if it was + * present. + */ + function remove(BytesSet storage self, bytes memory value) internal returns (bool) { + // We cache the value's position to prevent multiple reads from the same storage slot + uint256 position = self._positions[value]; + + if (position != 0) { + // Equivalent to contains(self, value) + // To delete an element from the _values array in O(1), we swap the element to delete with the last one in + // the array, and then remove the last element (sometimes called as 'swap and pop'). + // This modifies the order of the array, as noted in {at}. + + uint256 valueIndex = position - 1; + uint256 lastIndex = self._values.length - 1; + + if (valueIndex != lastIndex) { + bytes memory lastValue = self._values[lastIndex]; + + // Move the lastValue to the index where the value to delete is + self._values[valueIndex] = lastValue; + // Update the tracked position of the lastValue (that was just moved) + self._positions[lastValue] = position; + } + + // Delete the slot where the moved value was stored + self._values.pop(); + + // Delete the tracked position for the deleted slot + delete self._positions[value]; + + return true; + } else { + return false; + } + } + + /** + * @dev Removes all the values from a set. O(n). + * + * WARNING: Developers should keep in mind that this function has an unbounded cost and using it may render the + * function uncallable if the set grows to the point where clearing it consumes too much gas to fit in a block. + */ + function clear(BytesSet storage set) internal { + uint256 len = length(set); + for (uint256 i = 0; i < len; ++i) { + delete set._positions[set._values[i]]; + } + Arrays.unsafeSetLength(set._values, 0); + } + + /** + * @dev Returns true if the value is in the set. O(1). + */ + function contains(BytesSet storage self, bytes memory value) internal view returns (bool) { + return self._positions[value] != 0; + } + + /** + * @dev Returns the number of values on the set. O(1). + */ + function length(BytesSet storage self) internal view returns (uint256) { + return self._values.length; + } + + /** + * @dev Returns the value stored at position `index` in the set. O(1). + * + * Note that there are no guarantees on the ordering of values inside the + * array, and it may change when more values are added or removed. + * + * Requirements: + * + * - `index` must be strictly less than {length}. + */ + function at(BytesSet storage self, uint256 index) internal view returns (bytes memory) { + return self._values[index]; + } + + /** + * @dev Return the entire set in an array + * + * WARNING: This operation will copy the entire storage to memory, which can be quite expensive. This is designed + * to mostly be used by view accessors that are queried without any gas fees. Developers should keep in mind that + * this function has an unbounded cost, and using it as part of a state-changing function may render the function + * uncallable if the set grows to a point where copying to memory consumes too much gas to fit in a block. + */ + function values(BytesSet storage self) internal view returns (bytes[] memory) { + return self._values; + } + + struct Bytes32x2Set { + // Storage of set values + bytes32[2][] _values; + // Position is the index of the value in the `values` array plus 1. + // Position 0 is used to mean a value is not in the set. + mapping(bytes32 valueHash => uint256) _positions; + } + + /** + * @dev Add a value to a set. O(1). + * + * Returns true if the value was added to the set, that is if it was not + * already present. + */ + function add(Bytes32x2Set storage self, bytes32[2] memory value) internal returns (bool) { + if (!contains(self, value)) { + self._values.push(value); + // The value is stored at length-1, but we add 1 to all indexes + // and use 0 as a sentinel value + self._positions[_hash(value)] = self._values.length; + return true; + } else { + return false; + } + } + + /** + * @dev Removes a value from a set. O(1). + * + * Returns true if the value was removed from the set, that is if it was + * present. + */ + function remove(Bytes32x2Set storage self, bytes32[2] memory value) internal returns (bool) { + // We cache the value's position to prevent multiple reads from the same storage slot + bytes32 valueHash = _hash(value); + uint256 position = self._positions[valueHash]; + + if (position != 0) { + // Equivalent to contains(self, value) + // To delete an element from the _values array in O(1), we swap the element to delete with the last one in + // the array, and then remove the last element (sometimes called as 'swap and pop'). + // This modifies the order of the array, as noted in {at}. + + uint256 valueIndex = position - 1; + uint256 lastIndex = self._values.length - 1; + + if (valueIndex != lastIndex) { + bytes32[2] memory lastValue = self._values[lastIndex]; + + // Move the lastValue to the index where the value to delete is + self._values[valueIndex] = lastValue; + // Update the tracked position of the lastValue (that was just moved) + self._positions[_hash(lastValue)] = position; + } + + // Delete the slot where the moved value was stored + self._values.pop(); + + // Delete the tracked position for the deleted slot + delete self._positions[valueHash]; + + return true; + } else { + return false; + } + } + + /** + * @dev Removes all the values from a set. O(n). + * + * WARNING: Developers should keep in mind that this function has an unbounded cost and using it may render the + * function uncallable if the set grows to the point where clearing it consumes too much gas to fit in a block. + */ + function clear(Bytes32x2Set storage self) internal { + bytes32[2][] storage v = self._values; + + uint256 len = length(self); + for (uint256 i = 0; i < len; ++i) { + delete self._positions[_hash(v[i])]; + } + assembly ("memory-safe") { + sstore(v.slot, 0) + } + } + + /** + * @dev Returns true if the value is in the set. O(1). + */ + function contains(Bytes32x2Set storage self, bytes32[2] memory value) internal view returns (bool) { + return self._positions[_hash(value)] != 0; + } + + /** + * @dev Returns the number of values on the set. O(1). + */ + function length(Bytes32x2Set storage self) internal view returns (uint256) { + return self._values.length; + } + + /** + * @dev Returns the value stored at position `index` in the set. O(1). + * + * Note that there are no guarantees on the ordering of values inside the + * array, and it may change when more values are added or removed. + * + * Requirements: + * + * - `index` must be strictly less than {length}. + */ + function at(Bytes32x2Set storage self, uint256 index) internal view returns (bytes32[2] memory) { + return self._values[index]; + } + + /** + * @dev Return the entire set in an array + * + * WARNING: This operation will copy the entire storage to memory, which can be quite expensive. This is designed + * to mostly be used by view accessors that are queried without any gas fees. Developers should keep in mind that + * this function has an unbounded cost, and using it as part of a state-changing function may render the function + * uncallable if the set grows to a point where copying to memory consumes too much gas to fit in a block. + */ + function values(Bytes32x2Set storage self) internal view returns (bytes32[2][] memory) { + return self._values; + } + + function _hash(bytes32[2] memory value) private pure returns (bytes32) { + return Hashes.efficientKeccak256(value[0], value[1]); + } +} diff --git a/lib/@openzeppelin-contracts b/lib/@openzeppelin-contracts index 1873ecb3..21c8312b 160000 --- a/lib/@openzeppelin-contracts +++ b/lib/@openzeppelin-contracts @@ -1 +1 @@ -Subproject commit 1873ecb38e0833fa3552f58e639eeeb134b82135 +Subproject commit 21c8312b022f495ebe3621d5daeed20552b43ff9 diff --git a/lib/@openzeppelin-contracts-upgradeable b/lib/@openzeppelin-contracts-upgradeable index 0e120f46..e874681b 160000 --- a/lib/@openzeppelin-contracts-upgradeable +++ b/lib/@openzeppelin-contracts-upgradeable @@ -1 +1 @@ -Subproject commit 0e120f467564d41e3569a63ee719025adfd06f70 +Subproject commit e874681bed440d5ef99d9c960f4b6c6d13950616 diff --git a/lib/forge-std b/lib/forge-std index 841c3a3e..6abf6698 160000 --- a/lib/forge-std +++ b/lib/forge-std @@ -1 +1 @@ -Subproject commit 841c3a3e8b7612f76599797b93da5a61899c2aa8 +Subproject commit 6abf66980050ab03a35b52bdab814f55001d6929 diff --git a/package.json b/package.json index 46535f8b..5cb35152 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,9 @@ "lint:sol": "prettier --log-level warn --ignore-path .gitignore '{contracts,test}/**/*.sol' --check && solhint '{contracts,test}/**/*.sol'", "lint:sol:fix": "prettier --log-level warn --ignore-path .gitignore '{contracts,test}/**/*.sol' --write", "coverage": "scripts/checks/coverage.sh", + "generate": "scripts/generate/run.js", "test": "hardhat test", + "test:generation": "scripts/checks/generation.sh", "test:inheritance": "scripts/checks/inheritance-ordering.js artifacts/build-info/*", "test:pragma": "scripts/checks/pragma-consistency.js artifacts/build-info/*" }, diff --git a/scripts/checks/generation.sh b/scripts/checks/generation.sh new file mode 100755 index 00000000..00d609f9 --- /dev/null +++ b/scripts/checks/generation.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -euo pipefail + +npm run generate +git diff -R --exit-code diff --git a/scripts/generate/run.js b/scripts/generate/run.js new file mode 100755 index 00000000..639e9478 --- /dev/null +++ b/scripts/generate/run.js @@ -0,0 +1,38 @@ +#!/usr/bin/env node + +const cp = require('child_process'); +const fs = require('fs'); +const path = require('path'); +const format = require('@openzeppelin/contracts/scripts/generate/format-lines'); + +function getVersion(path) { + try { + return fs.readFileSync(path, 'utf8').match(/\/\/ OpenZeppelin Community Contracts \(last updated v[^)]+\)/)[0]; + } catch { + return null; + } +} + +function generateFromTemplate(file, template, outputPrefix = '') { + const script = path.relative(path.join(__dirname, '../..'), __filename); + const input = path.join(path.dirname(script), template); + const output = path.join(outputPrefix, file); + const version = getVersion(output); + const content = format( + '// SPDX-License-Identifier: MIT', + ...(version ? [version + ` (${file})`] : []), + `// This file was procedurally generated from ${input}.`, + '', + require(template).trimEnd(), + ); + fs.writeFileSync(output, content); + cp.execFileSync('prettier', ['--write', output]); +} + +// Contracts +for (const [file, template] of Object.entries({ + 'utils/structs/EnumerableSetExtended.sol': './templates/EnumerableSetExtended.js', + 'utils/structs/EnumerableMapExtended.sol': './templates/EnumerableMapExtended.js', +})) { + generateFromTemplate(file, template, './contracts/'); +} diff --git a/scripts/generate/templates/Enumerable.opts.js b/scripts/generate/templates/Enumerable.opts.js new file mode 100644 index 00000000..0691b8d5 --- /dev/null +++ b/scripts/generate/templates/Enumerable.opts.js @@ -0,0 +1,46 @@ +const { capitalize, mapValues } = require('@openzeppelin/contracts/scripts/helpers'); + +const typeDescr = ({ type, size = 0, memory = false }) => { + memory |= size > 0; + + const name = [type == 'uint256' ? 'Uint' : capitalize(type), size].filter(Boolean).join('x'); + const base = size ? type : undefined; + const typeFull = size ? `${type}[${size}]` : type; + const typeLoc = memory ? `${typeFull} memory` : typeFull; + return { name, type: typeFull, typeLoc, base, size, memory }; +}; + +const toSetTypeDescr = value => ({ + name: value.name + 'Set', + value, +}); + +const toMapTypeDescr = ({ key, value }) => ({ + name: `${key.name}To${value.name}Map`, + keySet: toSetTypeDescr(key), + key, + value, +}); + +const SET_TYPES = [ + { type: 'bytes32', size: 2 }, + { type: 'string', memory: true }, + { type: 'bytes', memory: true }, +] + .map(typeDescr) + .map(toSetTypeDescr); + +const MAP_TYPES = [ + { key: { type: 'bytes', memory: true }, value: { type: 'uint256' } }, + { key: { type: 'string', memory: true }, value: { type: 'string', memory: true } }, +] + .map(entry => mapValues(entry, typeDescr)) + .map(toMapTypeDescr); + +module.exports = { + SET_TYPES, + MAP_TYPES, + typeDescr, + toSetTypeDescr, + toMapTypeDescr, +}; diff --git a/scripts/generate/templates/EnumerableMapExtended.js b/scripts/generate/templates/EnumerableMapExtended.js new file mode 100644 index 00000000..89277fd3 --- /dev/null +++ b/scripts/generate/templates/EnumerableMapExtended.js @@ -0,0 +1,179 @@ +const format = require('@openzeppelin/contracts/scripts/generate/format-lines'); +const { SET_TYPES, MAP_TYPES } = require('./Enumerable.opts'); + +const header = `\ +pragma solidity ^0.8.20; + +import {EnumerableSet} from "@openzeppelin/contracts/utils/structs/EnumerableSet.sol"; +import {EnumerableSetExtended} from "./EnumerableSetExtended.sol"; + +/** + * @dev Library for managing an enumerable variant of Solidity's + * https://solidity.readthedocs.io/en/latest/types.html#mapping-types[\`mapping\`] + * type for non-value types as keys. + * + * Maps have the following properties: + * + * - Entries are added, removed, and checked for existence in constant time + * (O(1)). + * - Entries are enumerated in O(n). No guarantees are made on the ordering. + * - Map can be cleared (all entries removed) in O(n). + * + * \`\`\`solidity + * contract Example { + * // Add the library methods + * using EnumerableMapExtended for EnumerableMapExtended.BytesToUintMap; + * + * // Declare a set state variable + * EnumerableMapExtended.BytesToUintMap private myMap; + * } + * \`\`\` + * + * The following map types are supported: + * + * - \`bytes -> uint256\` (\`BytesToUintMap\`) + * - \`string -> string\` (\`StringToStringMap\`) + * + * [WARNING] + * ==== + * Trying to delete such a structure from storage will likely result in data corruption, rendering the structure + * unusable. + * See https://github.com/ethereum/solidity/pull/11843[ethereum/solidity#11843] for more info. + * + * In order to clean an EnumerableMap, you can either remove all elements one by one or create a fresh instance using an + * array of EnumerableMap. + * ==== + * + * NOTE: Extensions of openzeppelin/contracts/utils/struct/EnumerableMap.sol. + */ +`; + +const map = ({ name, keySet, key, value }) => `\ +/** + * @dev Query for a nonexistent map key. + */ +error EnumerableMapNonexistent${key.name}Key(${key.type} key); + +struct ${name} { + // Storage of keys + ${SET_TYPES.some(el => el.name == keySet.name) ? 'EnumerableSetExtended' : 'EnumerableSet'}.${keySet.name} _keys; + mapping(${key.type} key => ${value.type}) _values; +} + +/** + * @dev Adds a key-value pair to a map, or updates the value for an existing + * key. O(1). + * + * Returns true if the key was added to the map, that is if it was not + * already present. + */ +function set(${name} storage map, ${key.typeLoc} key, ${value.typeLoc} value) internal returns (bool) { + map._values[key] = value; + return map._keys.add(key); +} + +/** + * @dev Removes a key-value pair from a map. O(1). + * + * Returns true if the key was removed from the map, that is if it was present. + */ +function remove(${name} storage map, ${key.typeLoc} key) internal returns (bool) { + delete map._values[key]; + return map._keys.remove(key); +} + +/** + * @dev Removes all the entries from a map. O(n). + * + * WARNING: Developers should keep in mind that this function has an unbounded cost and using it may render the + * function uncallable if the map grows to the point where clearing it consumes too much gas to fit in a block. + */ +function clear(${name} storage map) internal { + uint256 len = length(map); + for (uint256 i = 0; i < len; ++i) { + delete map._values[map._keys.at(i)]; + } + map._keys.clear(); +} + +/** + * @dev Returns true if the key is in the map. O(1). + */ +function contains(${name} storage map, ${key.typeLoc} key) internal view returns (bool) { + return map._keys.contains(key); +} + +/** + * @dev Returns the number of key-value pairs in the map. O(1). + */ +function length(${name} storage map) internal view returns (uint256) { + return map._keys.length(); +} + +/** + * @dev Returns the key-value pair stored at position \`index\` in the map. O(1). + * + * Note that there are no guarantees on the ordering of entries inside the + * array, and it may change when more entries are added or removed. + * + * Requirements: + * + * - \`index\` must be strictly less than {length}. + */ +function at( + ${name} storage map, + uint256 index +) internal view returns (${key.typeLoc} key, ${value.typeLoc} value) { + key = map._keys.at(index); + value = map._values[key]; +} + +/** + * @dev Tries to returns the value associated with \`key\`. O(1). + * Does not revert if \`key\` is not in the map. + */ +function tryGet( + ${name} storage map, + ${key.typeLoc} key +) internal view returns (bool exists, ${value.typeLoc} value) { + value = map._values[key]; + exists = ${value.memory ? 'bytes(value).length != 0' : `value != ${value.type}(0)`} || contains(map, key); +} + +/** + * @dev Returns the value associated with \`key\`. O(1). + * + * Requirements: + * + * - \`key\` must be in the map. + */ +function get(${name} storage map, ${key.typeLoc} key) internal view returns (${value.typeLoc} value) { + bool exists; + (exists, value) = tryGet(map, key); + if (!exists) { + revert EnumerableMapNonexistent${key.name}Key(key); + } +} + +/** + * @dev Return the an array containing all the keys + * + * WARNING: This operation will copy the entire storage to memory, which can be quite expensive. This is designed + * to mostly be used by view accessors that are queried without any gas fees. Developers should keep in mind that + * this function has an unbounded cost, and using it as part of a state-changing function may render the function + * uncallable if the map grows to a point where copying to memory consumes too much gas to fit in a block. + */ +function keys(${name} storage map) internal view returns (${key.type}[] memory) { + return map._keys.values(); +} +`; + +// GENERATE +module.exports = format( + header.trimEnd(), + 'library EnumerableMapExtended {', + format( + [].concat('using EnumerableSet for *;', 'using EnumerableSetExtended for *;', '', MAP_TYPES.map(map)), + ).trimEnd(), + '}', +); diff --git a/scripts/generate/templates/EnumerableSetExtended.js b/scripts/generate/templates/EnumerableSetExtended.js new file mode 100644 index 00000000..54d56f80 --- /dev/null +++ b/scripts/generate/templates/EnumerableSetExtended.js @@ -0,0 +1,316 @@ +const format = require('@openzeppelin/contracts/scripts/generate/format-lines'); +const { SET_TYPES } = require('./Enumerable.opts'); + +const header = `\ +pragma solidity ^0.8.20; + +import {Arrays} from "@openzeppelin/contracts/utils/Arrays.sol"; +import {Hashes} from "@openzeppelin/contracts/utils/cryptography/Hashes.sol"; + +/** + * @dev Library for managing + * https://en.wikipedia.org/wiki/Set_(abstract_data_type)[sets] of non-value + * types. + * + * Sets have the following properties: + * + * - Elements are added, removed, and checked for existence in constant time + * (O(1)). + * - Elements are enumerated in O(n). No guarantees are made on the ordering. + * - Set can be cleared (all elements removed) in O(n). + * + * \`\`\`solidity + * contract Example { + * // Add the library methods + * using EnumerableSetExtended for EnumerableSetExtended.StringSet; + * + * // Declare a set state variable + * EnumerableSetExtended.StringSet private mySet; + * } + * \`\`\` + * + * Sets of type \`string\` (\`StringSet\`), \`bytes\` (\`BytesSet\`) and + * \`bytes32[2]\` (\`Bytes32x2Set\`) are supported. + * + * [WARNING] + * ==== + * Trying to delete such a structure from storage will likely result in data corruption, rendering the structure + * unusable. + * See https://github.com/ethereum/solidity/pull/11843[ethereum/solidity#11843] for more info. + * + * In order to clean an EnumerableSet, you can either remove all elements one by one or create a fresh instance using an + * array of EnumerableSet. + * ==== + * + * NOTE: This is an extension of openzeppelin/contracts/utils/struct/EnumerableSet.sol. + */ +`; + +const set = ({ name, value }) => `\ +struct ${name} { + // Storage of set values + ${value.type}[] _values; + // Position is the index of the value in the \`values\` array plus 1. + // Position 0 is used to mean a value is not in the set. + mapping(${value.type} value => uint256) _positions; +} + +/** + * @dev Add a value to a set. O(1). + * + * Returns true if the value was added to the set, that is if it was not + * already present. + */ +function add(${name} storage self, ${value.type} memory value) internal returns (bool) { + if (!contains(self, value)) { + self._values.push(value); + // The value is stored at length-1, but we add 1 to all indexes + // and use 0 as a sentinel value + self._positions[value] = self._values.length; + return true; + } else { + return false; + } +} + +/** + * @dev Removes a value from a set. O(1). + * + * Returns true if the value was removed from the set, that is if it was + * present. + */ +function remove(${name} storage self, ${value.type} memory value) internal returns (bool) { + // We cache the value's position to prevent multiple reads from the same storage slot + uint256 position = self._positions[value]; + + if (position != 0) { + // Equivalent to contains(self, value) + // To delete an element from the _values array in O(1), we swap the element to delete with the last one in + // the array, and then remove the last element (sometimes called as 'swap and pop'). + // This modifies the order of the array, as noted in {at}. + + uint256 valueIndex = position - 1; + uint256 lastIndex = self._values.length - 1; + + if (valueIndex != lastIndex) { + ${value.type} memory lastValue = self._values[lastIndex]; + + // Move the lastValue to the index where the value to delete is + self._values[valueIndex] = lastValue; + // Update the tracked position of the lastValue (that was just moved) + self._positions[lastValue] = position; + } + + // Delete the slot where the moved value was stored + self._values.pop(); + + // Delete the tracked position for the deleted slot + delete self._positions[value]; + + return true; + } else { + return false; + } +} + +/** + * @dev Removes all the values from a set. O(n). + * + * WARNING: Developers should keep in mind that this function has an unbounded cost and using it may render the + * function uncallable if the set grows to the point where clearing it consumes too much gas to fit in a block. + */ +function clear(${name} storage set) internal { + uint256 len = length(set); + for (uint256 i = 0; i < len; ++i) { + delete set._positions[set._values[i]]; + } + Arrays.unsafeSetLength(set._values, 0); +} + +/** + * @dev Returns true if the value is in the set. O(1). + */ +function contains(${name} storage self, ${value.type} memory value) internal view returns (bool) { + return self._positions[value] != 0; +} + +/** + * @dev Returns the number of values on the set. O(1). + */ +function length(${name} storage self) internal view returns (uint256) { + return self._values.length; +} + +/** + * @dev Returns the value stored at position \`index\` in the set. O(1). + * + * Note that there are no guarantees on the ordering of values inside the + * array, and it may change when more values are added or removed. + * + * Requirements: + * + * - \`index\` must be strictly less than {length}. + */ +function at(${name} storage self, uint256 index) internal view returns (${value.type} memory) { + return self._values[index]; +} + +/** + * @dev Return the entire set in an array + * + * WARNING: This operation will copy the entire storage to memory, which can be quite expensive. This is designed + * to mostly be used by view accessors that are queried without any gas fees. Developers should keep in mind that + * this function has an unbounded cost, and using it as part of a state-changing function may render the function + * uncallable if the set grows to a point where copying to memory consumes too much gas to fit in a block. + */ +function values(${name} storage self) internal view returns (${value.type}[] memory) { + return self._values; +} +`; + +const arraySet = ({ name, value }) => `\ +struct ${name} { + // Storage of set values + ${value.type}[] _values; + // Position is the index of the value in the \`values\` array plus 1. + // Position 0 is used to mean a value is not in the set. + mapping(bytes32 valueHash => uint256) _positions; +} + +/** + * @dev Add a value to a set. O(1). + * + * Returns true if the value was added to the set, that is if it was not + * already present. + */ +function add(${name} storage self, ${value.type} memory value) internal returns (bool) { + if (!contains(self, value)) { + self._values.push(value); + // The value is stored at length-1, but we add 1 to all indexes + // and use 0 as a sentinel value + self._positions[_hash(value)] = self._values.length; + return true; + } else { + return false; + } +} + +/** + * @dev Removes a value from a set. O(1). + * + * Returns true if the value was removed from the set, that is if it was + * present. + */ +function remove(${name} storage self, ${value.type} memory value) internal returns (bool) { + // We cache the value's position to prevent multiple reads from the same storage slot + bytes32 valueHash = _hash(value); + uint256 position = self._positions[valueHash]; + + if (position != 0) { + // Equivalent to contains(self, value) + // To delete an element from the _values array in O(1), we swap the element to delete with the last one in + // the array, and then remove the last element (sometimes called as 'swap and pop'). + // This modifies the order of the array, as noted in {at}. + + uint256 valueIndex = position - 1; + uint256 lastIndex = self._values.length - 1; + + if (valueIndex != lastIndex) { + ${value.type} memory lastValue = self._values[lastIndex]; + + // Move the lastValue to the index where the value to delete is + self._values[valueIndex] = lastValue; + // Update the tracked position of the lastValue (that was just moved) + self._positions[_hash(lastValue)] = position; + } + + // Delete the slot where the moved value was stored + self._values.pop(); + + // Delete the tracked position for the deleted slot + delete self._positions[valueHash]; + + return true; + } else { + return false; + } +} + +/** + * @dev Removes all the values from a set. O(n). + * + * WARNING: Developers should keep in mind that this function has an unbounded cost and using it may render the + * function uncallable if the set grows to the point where clearing it consumes too much gas to fit in a block. + */ +function clear(${name} storage self) internal { + ${value.type}[] storage v = self._values; + + uint256 len = length(self); + for (uint256 i = 0; i < len; ++i) { + delete self._positions[_hash(v[i])]; + } + assembly ("memory-safe") { + sstore(v.slot, 0) + } +} + +/** + * @dev Returns true if the value is in the set. O(1). + */ +function contains(${name} storage self, ${value.type} memory value) internal view returns (bool) { + return self._positions[_hash(value)] != 0; +} + +/** + * @dev Returns the number of values on the set. O(1). + */ +function length(${name} storage self) internal view returns (uint256) { + return self._values.length; +} + +/** + * @dev Returns the value stored at position \`index\` in the set. O(1). + * + * Note that there are no guarantees on the ordering of values inside the + * array, and it may change when more values are added or removed. + * + * Requirements: + * + * - \`index\` must be strictly less than {length}. + */ +function at(${name} storage self, uint256 index) internal view returns (${value.type} memory) { + return self._values[index]; +} + +/** + * @dev Return the entire set in an array + * + * WARNING: This operation will copy the entire storage to memory, which can be quite expensive. This is designed + * to mostly be used by view accessors that are queried without any gas fees. Developers should keep in mind that + * this function has an unbounded cost, and using it as part of a state-changing function may render the function + * uncallable if the set grows to a point where copying to memory consumes too much gas to fit in a block. + */ +function values(${name} storage self) internal view returns (${value.type}[] memory) { + return self._values; +} +`; + +const hashes = `\ +function _hash(bytes32[2] memory value) private pure returns (bytes32) { + return Hashes.efficientKeccak256(value[0], value[1]); +} +`; + +// GENERATE +module.exports = format( + header.trimEnd(), + 'library EnumerableSetExtended {', + format( + [].concat( + SET_TYPES.filter(({ value }) => value.size == 0).map(set), + SET_TYPES.filter(({ value }) => value.size > 0).map(arraySet), + hashes, + ), + ).trimEnd(), + '}', +); diff --git a/test/utils/structs/EnumerableMapExtended.test.js b/test/utils/structs/EnumerableMapExtended.test.js new file mode 100644 index 00000000..ba4942d0 --- /dev/null +++ b/test/utils/structs/EnumerableMapExtended.test.js @@ -0,0 +1,66 @@ +const { ethers } = require('hardhat'); +const { loadFixture } = require('@nomicfoundation/hardhat-network-helpers'); + +const { mapValues } = require('@openzeppelin/contracts/test/helpers/iterate'); +const { generators } = require('@openzeppelin/contracts/test/helpers/random'); +const { MAP_TYPES } = require('../../../scripts/generate/templates/Enumerable.opts'); + +const { shouldBehaveLikeMap } = require('@openzeppelin/contracts/test/utils/structs/EnumerableMap.behavior'); + +async function fixture() { + const mock = await ethers.deployContract('$EnumerableMapExtended'); + + const env = Object.fromEntries( + MAP_TYPES.map(({ name, key, value }) => [ + name, + { + key, + value, + keys: Array.from({ length: 3 }, generators[key.type]), + values: Array.from({ length: 3 }, generators[value.type]), + zeroValue: generators[value.type].zero, + methods: mapValues( + { + set: `$set(uint256,${key.type},${value.type})`, + get: `$get(uint256,${key.type})`, + tryGet: `$tryGet(uint256,${key.type})`, + remove: `$remove(uint256,${key.type})`, + clear: `$clear_EnumerableMapExtended_${name}(uint256)`, + length: `$length_EnumerableMapExtended_${name}(uint256)`, + at: `$at_EnumerableMapExtended_${name}(uint256,uint256)`, + contains: `$contains(uint256,${key.type})`, + keys: `$keys_EnumerableMapExtended_${name}(uint256)`, + }, + fnSig => + (...args) => + mock.getFunction(fnSig)(0, ...args), + ), + events: { + setReturn: `return$set_EnumerableMapExtended_${name}_${key.type}_${value.type}`, + removeReturn: `return$remove_EnumerableMapExtended_${name}_${key.type}`, + }, + error: key.memory || value.memory ? `EnumerableMapNonexistent${key.name}Key` : `EnumerableMapNonexistentKey`, + }, + ]), + ); + + return { mock, env }; +} + +describe('EnumerableMapExtended', function () { + beforeEach(async function () { + Object.assign(this, await loadFixture(fixture)); + }); + + for (const { name, key, value } of MAP_TYPES) { + describe(`${name} (enumerable map from ${key.type} to ${value.type})`, function () { + beforeEach(async function () { + Object.assign(this, this.env[name]); + [this.keyA, this.keyB, this.keyC] = this.keys; + [this.valueA, this.valueB, this.valueC] = this.values; + }); + + shouldBehaveLikeMap(); + }); + } +}); diff --git a/test/utils/structs/EnumerableSetExtended.test.js b/test/utils/structs/EnumerableSetExtended.test.js new file mode 100644 index 00000000..d3f698c7 --- /dev/null +++ b/test/utils/structs/EnumerableSetExtended.test.js @@ -0,0 +1,62 @@ +const { ethers } = require('hardhat'); +const { loadFixture } = require('@nomicfoundation/hardhat-network-helpers'); + +const { mapValues } = require('@openzeppelin/contracts/test/helpers/iterate'); +const { generators } = require('@openzeppelin/contracts/test/helpers/random'); +const { SET_TYPES } = require('../../../scripts/generate/templates/Enumerable.opts'); + +const { shouldBehaveLikeSet } = require('@openzeppelin/contracts/test/utils/structs/EnumerableSet.behavior'); + +async function fixture() { + const mock = await ethers.deployContract('$EnumerableSetExtended'); + + const env = Object.fromEntries( + SET_TYPES.map(({ name, value }) => [ + name, + { + value, + values: Array.from( + { length: 3 }, + value.size ? () => Array.from({ length: value.size }, generators[value.base]) : generators[value.type], + ), + methods: mapValues( + { + add: `$add(uint256,${value.type})`, + remove: `$remove(uint256,${value.type})`, + contains: `$contains(uint256,${value.type})`, + clear: `$clear_EnumerableSetExtended_${name}(uint256)`, + length: `$length_EnumerableSetExtended_${name}(uint256)`, + at: `$at_EnumerableSetExtended_${name}(uint256,uint256)`, + values: `$values_EnumerableSetExtended_${name}(uint256)`, + }, + fnSig => + (...args) => + mock.getFunction(fnSig)(0, ...args), + ), + events: { + addReturn: `return$add_EnumerableSetExtended_${name}_${value.type.replace(/[[\]]/g, '_')}`, + removeReturn: `return$remove_EnumerableSetExtended_${name}_${value.type.replace(/[[\]]/g, '_')}`, + }, + }, + ]), + ); + + return { mock, env }; +} + +describe('EnumerableSetExtended', function () { + beforeEach(async function () { + Object.assign(this, await loadFixture(fixture)); + }); + + for (const { name, value } of SET_TYPES) { + describe(`${name} (enumerable set of ${value.type})`, function () { + beforeEach(function () { + Object.assign(this, this.env[name]); + [this.valueA, this.valueB, this.valueC] = this.values; + }); + + shouldBehaveLikeSet(); + }); + } +});