From ce9c0059708e3cc634fd0c820a180702e47b4296 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Thu, 13 Mar 2025 15:48:30 +0100 Subject: [PATCH 1/7] Add EnumerableSetExtended and EnumerableMapExtended --- .../utils/structs/EnumerableMapExtended.sol | 249 +++++++++++ .../utils/structs/EnumerableSetExtended.sol | 385 ++++++++++++++++++ lib/@openzeppelin-contracts | 2 +- lib/@openzeppelin-contracts-upgradeable | 2 +- lib/forge-std | 2 +- package.json | 1 + scripts/generate/run.js | 41 ++ scripts/generate/templates/Enumerable.opts.js | 63 +++ .../templates/EnumerableMapExtended.js | 147 +++++++ .../templates/EnumerableSetExtended.js | 286 +++++++++++++ .../structs/EnumerableMapExtended.test.js | 66 +++ .../structs/EnumerableSetExtended.test.js | 62 +++ 12 files changed, 1303 insertions(+), 3 deletions(-) create mode 100644 contracts/utils/structs/EnumerableMapExtended.sol create mode 100644 contracts/utils/structs/EnumerableSetExtended.sol create mode 100755 scripts/generate/run.js create mode 100644 scripts/generate/templates/Enumerable.opts.js create mode 100644 scripts/generate/templates/EnumerableMapExtended.js create mode 100644 scripts/generate/templates/EnumerableSetExtended.js create mode 100644 test/utils/structs/EnumerableMapExtended.test.js create mode 100644 test/utils/structs/EnumerableSetExtended.test.js diff --git a/contracts/utils/structs/EnumerableMapExtended.sol b/contracts/utils/structs/EnumerableMapExtended.sol new file mode 100644 index 00000000..a2468ca4 --- /dev/null +++ b/contracts/utils/structs/EnumerableMapExtended.sol @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: MIT +// This file was procedurally generated from scripts/generate/templates/EnumerableMapExtended.js. + +pragma solidity ^0.8.20; + +import {EnumerableSet} from "@openzeppelin/contracts/utils/structs/EnumerableSet.sol"; +import {EnumerableSetExtended} from "./EnumerableSetExtended.sol"; + +/** + * @dev Library for managing an enumerable variant of Solidity's + * https://solidity.readthedocs.io/en/latest/types.html#mapping-types[`mapping`] + * type. + * + * Note: Extensions of openzeppelin/contracts/utils/struct/EnumerableMap.sol. + */ +library EnumerableMapExtended { + using EnumerableSet for *; + using EnumerableSetExtended for *; + + /** + * @dev Query for a nonexistent map key. + */ + error EnumerableMapNonexistentBytesKey(bytes key); + + struct BytesToUintMap { + // Storage of keys + EnumerableSetExtended.BytesSet _keys; + mapping(bytes key => uint256) _values; + } + + /** + * @dev Adds a key-value pair to a map, or updates the value for an existing + * key. O(1). + * + * Returns true if the key was added to the map, that is if it was not + * already present. + */ + function set(BytesToUintMap storage map, bytes memory key, uint256 value) internal returns (bool) { + map._values[key] = value; + return map._keys.add(key); + } + + /** + * @dev Removes a key-value pair from a map. O(1). + * + * Returns true if the key was removed from the map, that is if it was present. + */ + function remove(BytesToUintMap storage map, bytes memory key) internal returns (bool) { + delete map._values[key]; + return map._keys.remove(key); + } + + /** + * @dev Removes all the entries from a map. O(n). + * + * WARNING: Developers should keep in mind that this function has an unbounded cost and using it may render the + * function uncallable if the map grows to the point where clearing it consumes too much gas to fit in a block. + */ + function clear(BytesToUintMap storage map) internal { + uint256 len = length(map); + for (uint256 i = 0; i < len; ++i) { + delete map._values[map._keys.at(i)]; + } + map._keys.clear(); + } + + /** + * @dev Returns true if the key is in the map. O(1). + */ + function contains(BytesToUintMap storage map, bytes memory key) internal view returns (bool) { + return map._keys.contains(key); + } + + /** + * @dev Returns the number of key-value pairs in the map. O(1). + */ + function length(BytesToUintMap storage map) internal view returns (uint256) { + return map._keys.length(); + } + + /** + * @dev Returns the key-value pair stored at position `index` in the map. O(1). + * + * Note that there are no guarantees on the ordering of entries inside the + * array, and it may change when more entries are added or removed. + * + * Requirements: + * + * - `index` must be strictly less than {length}. + */ + function at(BytesToUintMap storage map, uint256 index) internal view returns (bytes memory key, uint256 value) { + key = map._keys.at(index); + value = map._values[key]; + } + + /** + * @dev Tries to returns the value associated with `key`. O(1). + * Does not revert if `key` is not in the map. + */ + function tryGet(BytesToUintMap storage map, bytes memory key) internal view returns (bool exists, uint256 value) { + value = map._values[key]; + exists = value != uint256(0) || contains(map, key); + } + + /** + * @dev Returns the value associated with `key`. O(1). + * + * Requirements: + * + * - `key` must be in the map. + */ + function get(BytesToUintMap storage map, bytes memory key) internal view returns (uint256 value) { + bool exists; + (exists, value) = tryGet(map, key); + if (!exists) { + revert EnumerableMapNonexistentBytesKey(key); + } + } + + /** + * @dev Return the an array containing all the keys + * + * WARNING: This operation will copy the entire storage to memory, which can be quite expensive. This is designed + * to mostly be used by view accessors that are queried without any gas fees. Developers should keep in mind that + * this function has an unbounded cost, and using it as part of a state-changing function may render the function + * uncallable if the map grows to a point where copying to memory consumes too much gas to fit in a block. + */ + function keys(BytesToUintMap storage map) internal view returns (bytes[] memory) { + return map._keys.values(); + } + + /** + * @dev Query for a nonexistent map key. + */ + error EnumerableMapNonexistentStringKey(string key); + + struct StringToStringMap { + // Storage of keys + EnumerableSetExtended.StringSet _keys; + mapping(string key => string) _values; + } + + /** + * @dev Adds a key-value pair to a map, or updates the value for an existing + * key. O(1). + * + * Returns true if the key was added to the map, that is if it was not + * already present. + */ + function set(StringToStringMap storage map, string memory key, string memory value) internal returns (bool) { + map._values[key] = value; + return map._keys.add(key); + } + + /** + * @dev Removes a key-value pair from a map. O(1). + * + * Returns true if the key was removed from the map, that is if it was present. + */ + function remove(StringToStringMap storage map, string memory key) internal returns (bool) { + delete map._values[key]; + return map._keys.remove(key); + } + + /** + * @dev Removes all the entries from a map. O(n). + * + * WARNING: Developers should keep in mind that this function has an unbounded cost and using it may render the + * function uncallable if the map grows to the point where clearing it consumes too much gas to fit in a block. + */ + function clear(StringToStringMap storage map) internal { + uint256 len = length(map); + for (uint256 i = 0; i < len; ++i) { + delete map._values[map._keys.at(i)]; + } + map._keys.clear(); + } + + /** + * @dev Returns true if the key is in the map. O(1). + */ + function contains(StringToStringMap storage map, string memory key) internal view returns (bool) { + return map._keys.contains(key); + } + + /** + * @dev Returns the number of key-value pairs in the map. O(1). + */ + function length(StringToStringMap storage map) internal view returns (uint256) { + return map._keys.length(); + } + + /** + * @dev Returns the key-value pair stored at position `index` in the map. O(1). + * + * Note that there are no guarantees on the ordering of entries inside the + * array, and it may change when more entries are added or removed. + * + * Requirements: + * + * - `index` must be strictly less than {length}. + */ + function at( + StringToStringMap storage map, + uint256 index + ) internal view returns (string memory key, string memory value) { + key = map._keys.at(index); + value = map._values[key]; + } + + /** + * @dev Tries to returns the value associated with `key`. O(1). + * Does not revert if `key` is not in the map. + */ + function tryGet( + StringToStringMap storage map, + string memory key + ) internal view returns (bool exists, string memory value) { + value = map._values[key]; + exists = bytes(value).length != 0 || contains(map, key); + } + + /** + * @dev Returns the value associated with `key`. O(1). + * + * Requirements: + * + * - `key` must be in the map. + */ + function get(StringToStringMap storage map, string memory key) internal view returns (string memory value) { + bool exists; + (exists, value) = tryGet(map, key); + if (!exists) { + revert EnumerableMapNonexistentStringKey(key); + } + } + + /** + * @dev Return the an array containing all the keys + * + * WARNING: This operation will copy the entire storage to memory, which can be quite expensive. This is designed + * to mostly be used by view accessors that are queried without any gas fees. Developers should keep in mind that + * this function has an unbounded cost, and using it as part of a state-changing function may render the function + * uncallable if the map grows to a point where copying to memory consumes too much gas to fit in a block. + */ + function keys(StringToStringMap storage map) internal view returns (string[] memory) { + return map._keys.values(); + } +} diff --git a/contracts/utils/structs/EnumerableSetExtended.sol b/contracts/utils/structs/EnumerableSetExtended.sol new file mode 100644 index 00000000..819b27c4 --- /dev/null +++ b/contracts/utils/structs/EnumerableSetExtended.sol @@ -0,0 +1,385 @@ +// SPDX-License-Identifier: MIT +// This file was procedurally generated from scripts/generate/templates/EnumerableSetExtended.js. + +pragma solidity ^0.8.20; + +import {Arrays} from "@openzeppelin/contracts/utils/Arrays.sol"; +import {Hashes} from "@openzeppelin/contracts/utils/cryptography/Hashes.sol"; + +/** + * @dev Library for managing + * https://en.wikipedia.org/wiki/Set_(abstract_data_type)[sets] of primitive + * types. + * + * Note: Extensions of openzeppelin/contracts/utils/struct/EnumerableSet.sol. + */ +library EnumerableSetExtended { + struct StringSet { + // Storage of set values + string[] _values; + // Position is the index of the value in the `values` array plus 1. + // Position 0 is used to mean a value is not in the self. + mapping(string value => uint256) _positions; + } + + /** + * @dev Add a value to a self. O(1). + * + * Returns true if the value was added to the set, that is if it was not + * already present. + */ + function add(StringSet storage self, string memory value) internal returns (bool) { + if (!contains(self, value)) { + self._values.push(value); + // The value is stored at length-1, but we add 1 to all indexes + // and use 0 as a sentinel value + self._positions[value] = self._values.length; + return true; + } else { + return false; + } + } + + /** + * @dev Removes a value from a self. O(1). + * + * Returns true if the value was removed from the set, that is if it was + * present. + */ + function remove(StringSet storage self, string memory value) internal returns (bool) { + // We cache the value's position to prevent multiple reads from the same storage slot + uint256 position = self._positions[value]; + + if (position != 0) { + // Equivalent to contains(self, value) + // To delete an element from the _values array in O(1), we swap the element to delete with the last one in + // the array, and then remove the last element (sometimes called as 'swap and pop'). + // This modifies the order of the array, as noted in {at}. + + uint256 valueIndex = position - 1; + uint256 lastIndex = self._values.length - 1; + + if (valueIndex != lastIndex) { + string memory lastValue = self._values[lastIndex]; + + // Move the lastValue to the index where the value to delete is + self._values[valueIndex] = lastValue; + // Update the tracked position of the lastValue (that was just moved) + self._positions[lastValue] = position; + } + + // Delete the slot where the moved value was stored + self._values.pop(); + + // Delete the tracked position for the deleted slot + delete self._positions[value]; + + return true; + } else { + return false; + } + } + + /** + * @dev Removes all the values from a set. O(n). + * + * WARNING: Developers should keep in mind that this function has an unbounded cost and using it may render the + * function uncallable if the set grows to the point where clearing it consumes too much gas to fit in a block. + */ + function clear(StringSet storage set) internal { + uint256 len = length(set); + for (uint256 i = 0; i < len; ++i) { + delete set._positions[set._values[i]]; + } + Arrays.unsafeSetLength(set._values, 0); + } + + /** + * @dev Returns true if the value is in the self. O(1). + */ + function contains(StringSet storage self, string memory value) internal view returns (bool) { + return self._positions[value] != 0; + } + + /** + * @dev Returns the number of values on the self. O(1). + */ + function length(StringSet storage self) internal view returns (uint256) { + return self._values.length; + } + + /** + * @dev Returns the value stored at position `index` in the self. O(1). + * + * Note that there are no guarantees on the ordering of values inside the + * array, and it may change when more values are added or removed. + * + * Requirements: + * + * - `index` must be strictly less than {length}. + */ + function at(StringSet storage self, uint256 index) internal view returns (string memory) { + return self._values[index]; + } + + /** + * @dev Return the entire set in an array + * + * WARNING: This operation will copy the entire storage to memory, which can be quite expensive. This is designed + * to mostly be used by view accessors that are queried without any gas fees. Developers should keep in mind that + * this function has an unbounded cost, and using it as part of a state-changing function may render the function + * uncallable if the set grows to a point where copying to memory consumes too much gas to fit in a block. + */ + function values(StringSet storage self) internal view returns (string[] memory) { + return self._values; + } + + struct BytesSet { + // Storage of set values + bytes[] _values; + // Position is the index of the value in the `values` array plus 1. + // Position 0 is used to mean a value is not in the self. + mapping(bytes value => uint256) _positions; + } + + /** + * @dev Add a value to a self. O(1). + * + * Returns true if the value was added to the set, that is if it was not + * already present. + */ + function add(BytesSet storage self, bytes memory value) internal returns (bool) { + if (!contains(self, value)) { + self._values.push(value); + // The value is stored at length-1, but we add 1 to all indexes + // and use 0 as a sentinel value + self._positions[value] = self._values.length; + return true; + } else { + return false; + } + } + + /** + * @dev Removes a value from a self. O(1). + * + * Returns true if the value was removed from the set, that is if it was + * present. + */ + function remove(BytesSet storage self, bytes memory value) internal returns (bool) { + // We cache the value's position to prevent multiple reads from the same storage slot + uint256 position = self._positions[value]; + + if (position != 0) { + // Equivalent to contains(self, value) + // To delete an element from the _values array in O(1), we swap the element to delete with the last one in + // the array, and then remove the last element (sometimes called as 'swap and pop'). + // This modifies the order of the array, as noted in {at}. + + uint256 valueIndex = position - 1; + uint256 lastIndex = self._values.length - 1; + + if (valueIndex != lastIndex) { + bytes memory lastValue = self._values[lastIndex]; + + // Move the lastValue to the index where the value to delete is + self._values[valueIndex] = lastValue; + // Update the tracked position of the lastValue (that was just moved) + self._positions[lastValue] = position; + } + + // Delete the slot where the moved value was stored + self._values.pop(); + + // Delete the tracked position for the deleted slot + delete self._positions[value]; + + return true; + } else { + return false; + } + } + + /** + * @dev Removes all the values from a set. O(n). + * + * WARNING: Developers should keep in mind that this function has an unbounded cost and using it may render the + * function uncallable if the set grows to the point where clearing it consumes too much gas to fit in a block. + */ + function clear(BytesSet storage set) internal { + uint256 len = length(set); + for (uint256 i = 0; i < len; ++i) { + delete set._positions[set._values[i]]; + } + Arrays.unsafeSetLength(set._values, 0); + } + + /** + * @dev Returns true if the value is in the self. O(1). + */ + function contains(BytesSet storage self, bytes memory value) internal view returns (bool) { + return self._positions[value] != 0; + } + + /** + * @dev Returns the number of values on the self. O(1). + */ + function length(BytesSet storage self) internal view returns (uint256) { + return self._values.length; + } + + /** + * @dev Returns the value stored at position `index` in the self. O(1). + * + * Note that there are no guarantees on the ordering of values inside the + * array, and it may change when more values are added or removed. + * + * Requirements: + * + * - `index` must be strictly less than {length}. + */ + function at(BytesSet storage self, uint256 index) internal view returns (bytes memory) { + return self._values[index]; + } + + /** + * @dev Return the entire set in an array + * + * WARNING: This operation will copy the entire storage to memory, which can be quite expensive. This is designed + * to mostly be used by view accessors that are queried without any gas fees. Developers should keep in mind that + * this function has an unbounded cost, and using it as part of a state-changing function may render the function + * uncallable if the set grows to a point where copying to memory consumes too much gas to fit in a block. + */ + function values(BytesSet storage self) internal view returns (bytes[] memory) { + return self._values; + } + + struct Bytes32x2Set { + // Storage of set values + bytes32[2][] _values; + // Position is the index of the value in the `values` array plus 1. + // Position 0 is used to mean a value is not in the self. + mapping(bytes32 valueHash => uint256) _positions; + } + + /** + * @dev Add a value to a self. O(1). + * + * Returns true if the value was added to the set, that is if it was not + * already present. + */ + function add(Bytes32x2Set storage self, bytes32[2] memory value) internal returns (bool) { + if (!contains(self, value)) { + self._values.push(value); + // The value is stored at length-1, but we add 1 to all indexes + // and use 0 as a sentinel value + self._positions[_hash(value)] = self._values.length; + return true; + } else { + return false; + } + } + + /** + * @dev Removes a value from a self. O(1). + * + * Returns true if the value was removed from the set, that is if it was + * present. + */ + function remove(Bytes32x2Set storage self, bytes32[2] memory value) internal returns (bool) { + // We cache the value's position to prevent multiple reads from the same storage slot + bytes32 valueHash = _hash(value); + uint256 position = self._positions[valueHash]; + + if (position != 0) { + // Equivalent to contains(self, value) + // To delete an element from the _values array in O(1), we swap the element to delete with the last one in + // the array, and then remove the last element (sometimes called as 'swap and pop'). + // This modifies the order of the array, as noted in {at}. + + uint256 valueIndex = position - 1; + uint256 lastIndex = self._values.length - 1; + + if (valueIndex != lastIndex) { + bytes32[2] memory lastValue = self._values[lastIndex]; + + // Move the lastValue to the index where the value to delete is + self._values[valueIndex] = lastValue; + // Update the tracked position of the lastValue (that was just moved) + self._positions[_hash(lastValue)] = position; + } + + // Delete the slot where the moved value was stored + self._values.pop(); + + // Delete the tracked position for the deleted slot + delete self._positions[valueHash]; + + return true; + } else { + return false; + } + } + + /** + * @dev Removes all the values from a set. O(n). + * + * WARNING: Developers should keep in mind that this function has an unbounded cost and using it may render the + * function uncallable if the set grows to the point where clearing it consumes too much gas to fit in a block. + */ + function clear(Bytes32x2Set storage self) internal { + bytes32[2][] storage v = self._values; + + uint256 len = length(self); + for (uint256 i = 0; i < len; ++i) { + delete self._positions[_hash(v[i])]; + } + assembly ("memory-safe") { + sstore(v.slot, 0) + } + } + + /** + * @dev Returns true if the value is in the self. O(1). + */ + function contains(Bytes32x2Set storage self, bytes32[2] memory value) internal view returns (bool) { + return self._positions[_hash(value)] != 0; + } + + /** + * @dev Returns the number of values on the self. O(1). + */ + function length(Bytes32x2Set storage self) internal view returns (uint256) { + return self._values.length; + } + + /** + * @dev Returns the value stored at position `index` in the self. O(1). + * + * Note that there are no guarantees on the ordering of values inside the + * array, and it may change when more values are added or removed. + * + * Requirements: + * + * - `index` must be strictly less than {length}. + */ + function at(Bytes32x2Set storage self, uint256 index) internal view returns (bytes32[2] memory) { + return self._values[index]; + } + + /** + * @dev Return the entire set in an array + * + * WARNING: This operation will copy the entire storage to memory, which can be quite expensive. This is designed + * to mostly be used by view accessors that are queried without any gas fees. Developers should keep in mind that + * this function has an unbounded cost, and using it as part of a state-changing function may render the function + * uncallable if the set grows to a point where copying to memory consumes too much gas to fit in a block. + */ + function values(Bytes32x2Set storage self) internal view returns (bytes32[2][] memory) { + return self._values; + } + + function _hash(bytes32[2] memory value) private pure returns (bytes32) { + return Hashes.efficientKeccak256(value[0], value[1]); + } +} diff --git a/lib/@openzeppelin-contracts b/lib/@openzeppelin-contracts index 441dc141..fda6b85f 160000 --- a/lib/@openzeppelin-contracts +++ b/lib/@openzeppelin-contracts @@ -1 +1 @@ -Subproject commit 441dc141ac99622de7e535fa75dfc74af939019c +Subproject commit fda6b85f2c65d146b86d513a604554d15abd6679 diff --git a/lib/@openzeppelin-contracts-upgradeable b/lib/@openzeppelin-contracts-upgradeable index 266b24b1..14afff72 160000 --- a/lib/@openzeppelin-contracts-upgradeable +++ b/lib/@openzeppelin-contracts-upgradeable @@ -1 +1 @@ -Subproject commit 266b24b1338f88281040cab1e805f96795d59d3e +Subproject commit 14afff724bb819e517f6eed872f01bc1b4c5381e diff --git a/lib/forge-std b/lib/forge-std index 3b20d60d..8ba9031f 160000 --- a/lib/forge-std +++ b/lib/forge-std @@ -1 +1 @@ -Subproject commit 3b20d60d14b343ee4f908cb8079495c07f5e8981 +Subproject commit 8ba9031ffcbe25aa0d1224d3ca263a995026e477 diff --git a/package.json b/package.json index 46535f8b..ebebaaea 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "lint:sol": "prettier --log-level warn --ignore-path .gitignore '{contracts,test}/**/*.sol' --check && solhint '{contracts,test}/**/*.sol'", "lint:sol:fix": "prettier --log-level warn --ignore-path .gitignore '{contracts,test}/**/*.sol' --write", "coverage": "scripts/checks/coverage.sh", + "generate": "scripts/generate/run.js", "test": "hardhat test", "test:inheritance": "scripts/checks/inheritance-ordering.js artifacts/build-info/*", "test:pragma": "scripts/checks/pragma-consistency.js artifacts/build-info/*" diff --git a/scripts/generate/run.js b/scripts/generate/run.js new file mode 100755 index 00000000..2f54e4cf --- /dev/null +++ b/scripts/generate/run.js @@ -0,0 +1,41 @@ +#!/usr/bin/env node + +const cp = require('child_process'); +const fs = require('fs'); +const path = require('path'); +const format = require('@openzeppelin/contracts/scripts/generate/format-lines'); + +function getVersion(path) { + try { + return fs.readFileSync(path, 'utf8').match(/\/\/ OpenZeppelin Community Contracts \(last updated v[^)]+\)/)[0]; + } catch { + return null; + } +} + +function generateFromTemplate(file, template, outputPrefix = '', lint = false) { + const script = path.relative(path.join(__dirname, '../..'), __filename); + const input = path.join(path.dirname(script), template); + const output = path.join(outputPrefix, file); + const version = getVersion(output); + const content = format( + '// SPDX-License-Identifier: MIT', + ...(version ? [version + ` (${file})`] : []), + `// This file was procedurally generated from ${input}.`, + '', + require(template).trimEnd(), + ); + fs.writeFileSync(output, content); + lint && cp.execFileSync('prettier', ['--write', output]); +} + +// Some templates needs to go through the linter after generation +const needsLinter = ['utils/structs/EnumerableMapExtended.sol']; + +// Contracts +for (const [file, template] of Object.entries({ + 'utils/structs/EnumerableSetExtended.sol': './templates/EnumerableSetExtended.js', + 'utils/structs/EnumerableMapExtended.sol': './templates/EnumerableMapExtended.js', +})) { + generateFromTemplate(file, template, './contracts/', needsLinter.indexOf(file) != -1); +} diff --git a/scripts/generate/templates/Enumerable.opts.js b/scripts/generate/templates/Enumerable.opts.js new file mode 100644 index 00000000..34c09c5c --- /dev/null +++ b/scripts/generate/templates/Enumerable.opts.js @@ -0,0 +1,63 @@ +const { capitalize, mapValues } = require('@openzeppelin/contracts/scripts/helpers'); + +const typeDescr = ({ type, size = 0, memory = false }) => { + memory |= size > 0; + + const name = [type == 'uint256' ? 'Uint' : capitalize(type), size].filter(Boolean).join('x'); + const base = size ? type : undefined; + const typeFull = size ? `${type}[${size}]` : type; + const typeLoc = memory ? `${typeFull} memory` : typeFull; + return { name, type: typeFull, typeLoc, base, size, memory }; +}; + +const toSetTypeDescr = value => ({ + name: value.name + 'Set', + value, +}); + +const toMapTypeDescr = ({ key, value }) => ({ + name: `${key.name}To${value.name}Map`, + keySet: toSetTypeDescr(key), + key, + value, +}); + +const SET_TYPES = [ + // { type: 'bytes32' }, // part of the vanilla repo + // { type: 'address' }, // part of the vanilla repo + // { type: 'uint256' }, // part of the vanilla repo + { type: 'bytes32', size: 2 }, + { type: 'string', memory: true }, + { type: 'bytes', memory: true }, +] + .map(typeDescr) + .map(toSetTypeDescr); + +const MAP_TYPES = [ + // { key: { type: 'uint256' }, value: { type: 'uint256' } }, // part of the vanilla repo + // { key: { type: 'uint256' }, value: { type: 'address' } }, // part of the vanilla repo + // { key: { type: 'uint256' }, value: { type: 'bytes32' } }, // part of the vanilla repo + // { key: { type: 'address' }, value: { type: 'uint256' } }, // part of the vanilla repo + // { key: { type: 'address' }, value: { type: 'address' } }, // part of the vanilla repo + // { key: { type: 'address' }, value: { type: 'bytes32' } }, // part of the vanilla repo + // { key: { type: 'bytes32' }, value: { type: 'uint256' } }, // part of the vanilla repo + // { key: { type: 'bytes32' }, value: { type: 'address' } }, // part of the vanilla repo + { key: { type: 'bytes', memory: true }, value: { type: 'uint256' } }, + { key: { type: 'string', memory: true }, value: { type: 'string', memory: true } }, +] + .map(entry => mapValues(entry, typeDescr)) + .map(toMapTypeDescr); + +/// Sanity - Disabled because some types might be provided by the vanilla repository. +// MAP_TYPES.forEach(entry => { +// if (!SET_TYPES.some(set => set.structName == entry.key.structName)) +// throw new Error(`${entry.structName} requires a "${entry.key.structName}" set of "${entry.key.type}"`); +// }); + +module.exports = { + SET_TYPES, + MAP_TYPES, + typeDescr, + toSetTypeDescr, + toMapTypeDescr, +}; diff --git a/scripts/generate/templates/EnumerableMapExtended.js b/scripts/generate/templates/EnumerableMapExtended.js new file mode 100644 index 00000000..8d0c1096 --- /dev/null +++ b/scripts/generate/templates/EnumerableMapExtended.js @@ -0,0 +1,147 @@ +const format = require('@openzeppelin/contracts/scripts/generate/format-lines'); +const { SET_TYPES, MAP_TYPES } = require('./Enumerable.opts'); + +const header = `\ +pragma solidity ^0.8.20; + +import {EnumerableSet} from "@openzeppelin/contracts/utils/structs/EnumerableSet.sol"; +import {EnumerableSetExtended} from "./EnumerableSetExtended.sol"; + +/** + * @dev Library for managing an enumerable variant of Solidity's + * https://solidity.readthedocs.io/en/latest/types.html#mapping-types[\`mapping\`] + * type. + * + * Note: Extensions of openzeppelin/contracts/utils/struct/EnumerableMap.sol. + */ +`; + +const map = ({ name, keySet, key, value }) => `\ +/** + * @dev Query for a nonexistent map key. + */ +error EnumerableMapNonexistent${key.name}Key(${key.type} key); + +struct ${name} { + // Storage of keys + ${SET_TYPES.some(el => el.name == keySet.name) ? 'EnumerableSetExtended' : 'EnumerableSet'}.${keySet.name} _keys; + mapping(${key.type} key => ${value.type}) _values; +} + +/** + * @dev Adds a key-value pair to a map, or updates the value for an existing + * key. O(1). + * + * Returns true if the key was added to the map, that is if it was not + * already present. + */ +function set(${name} storage map, ${key.typeLoc} key, ${value.typeLoc} value) internal returns (bool) { + map._values[key] = value; + return map._keys.add(key); +} + +/** + * @dev Removes a key-value pair from a map. O(1). + * + * Returns true if the key was removed from the map, that is if it was present. + */ +function remove(${name} storage map, ${key.typeLoc} key) internal returns (bool) { + delete map._values[key]; + return map._keys.remove(key); +} + +/** + * @dev Removes all the entries from a map. O(n). + * + * WARNING: Developers should keep in mind that this function has an unbounded cost and using it may render the + * function uncallable if the map grows to the point where clearing it consumes too much gas to fit in a block. + */ +function clear(${name} storage map) internal { + uint256 len = length(map); + for (uint256 i = 0; i < len; ++i) { + delete map._values[map._keys.at(i)]; + } + map._keys.clear(); +} + +/** + * @dev Returns true if the key is in the map. O(1). + */ +function contains(${name} storage map, ${key.typeLoc} key) internal view returns (bool) { + return map._keys.contains(key); +} + +/** + * @dev Returns the number of key-value pairs in the map. O(1). + */ +function length(${name} storage map) internal view returns (uint256) { + return map._keys.length(); +} + +/** + * @dev Returns the key-value pair stored at position \`index\` in the map. O(1). + * + * Note that there are no guarantees on the ordering of entries inside the + * array, and it may change when more entries are added or removed. + * + * Requirements: + * + * - \`index\` must be strictly less than {length}. + */ +function at( + ${name} storage map, + uint256 index +) internal view returns (${key.typeLoc} key, ${value.typeLoc} value) { + key = map._keys.at(index); + value = map._values[key]; +} + +/** + * @dev Tries to returns the value associated with \`key\`. O(1). + * Does not revert if \`key\` is not in the map. + */ +function tryGet( + ${name} storage map, + ${key.typeLoc} key +) internal view returns (bool exists, ${value.typeLoc} value) { + value = map._values[key]; + exists = ${value.memory ? 'bytes(value).length != 0' : `value != ${value.type}(0)`} || contains(map, key); +} + +/** + * @dev Returns the value associated with \`key\`. O(1). + * + * Requirements: + * + * - \`key\` must be in the map. + */ +function get(${name} storage map, ${key.typeLoc} key) internal view returns (${value.typeLoc} value) { + bool exists; + (exists, value) = tryGet(map, key); + if (!exists) { + revert EnumerableMapNonexistent${key.name}Key(key); + } +} + +/** + * @dev Return the an array containing all the keys + * + * WARNING: This operation will copy the entire storage to memory, which can be quite expensive. This is designed + * to mostly be used by view accessors that are queried without any gas fees. Developers should keep in mind that + * this function has an unbounded cost, and using it as part of a state-changing function may render the function + * uncallable if the map grows to a point where copying to memory consumes too much gas to fit in a block. + */ +function keys(${name} storage map) internal view returns (${key.type}[] memory) { + return map._keys.values(); +} +`; + +// GENERATE +module.exports = format( + header.trimEnd(), + 'library EnumerableMapExtended {', + format( + [].concat('using EnumerableSet for *;', 'using EnumerableSetExtended for *;', '', MAP_TYPES.map(map)), + ).trimEnd(), + '}', +); diff --git a/scripts/generate/templates/EnumerableSetExtended.js b/scripts/generate/templates/EnumerableSetExtended.js new file mode 100644 index 00000000..d1a2f19f --- /dev/null +++ b/scripts/generate/templates/EnumerableSetExtended.js @@ -0,0 +1,286 @@ +const format = require('@openzeppelin/contracts/scripts/generate/format-lines'); +const { SET_TYPES } = require('./Enumerable.opts'); + +const header = `\ +pragma solidity ^0.8.20; + +import {Arrays} from "@openzeppelin/contracts/utils/Arrays.sol"; +import {Hashes} from "@openzeppelin/contracts/utils/cryptography/Hashes.sol"; + +/** + * @dev Library for managing + * https://en.wikipedia.org/wiki/Set_(abstract_data_type)[sets] of primitive + * types. + * + * Note: Extensions of openzeppelin/contracts/utils/struct/EnumerableSet.sol. + */ +`; + +const set = ({ name, value }) => `\ +struct ${name} { + // Storage of set values + ${value.type}[] _values; + // Position is the index of the value in the \`values\` array plus 1. + // Position 0 is used to mean a value is not in the self. + mapping(${value.type} value => uint256) _positions; +} + +/** + * @dev Add a value to a self. O(1). + * + * Returns true if the value was added to the set, that is if it was not + * already present. + */ +function add(${name} storage self, ${value.type} memory value) internal returns (bool) { + if (!contains(self, value)) { + self._values.push(value); + // The value is stored at length-1, but we add 1 to all indexes + // and use 0 as a sentinel value + self._positions[value] = self._values.length; + return true; + } else { + return false; + } +} + +/** + * @dev Removes a value from a self. O(1). + * + * Returns true if the value was removed from the set, that is if it was + * present. + */ +function remove(${name} storage self, ${value.type} memory value) internal returns (bool) { + // We cache the value's position to prevent multiple reads from the same storage slot + uint256 position = self._positions[value]; + + if (position != 0) { + // Equivalent to contains(self, value) + // To delete an element from the _values array in O(1), we swap the element to delete with the last one in + // the array, and then remove the last element (sometimes called as 'swap and pop'). + // This modifies the order of the array, as noted in {at}. + + uint256 valueIndex = position - 1; + uint256 lastIndex = self._values.length - 1; + + if (valueIndex != lastIndex) { + ${value.type} memory lastValue = self._values[lastIndex]; + + // Move the lastValue to the index where the value to delete is + self._values[valueIndex] = lastValue; + // Update the tracked position of the lastValue (that was just moved) + self._positions[lastValue] = position; + } + + // Delete the slot where the moved value was stored + self._values.pop(); + + // Delete the tracked position for the deleted slot + delete self._positions[value]; + + return true; + } else { + return false; + } +} + +/** + * @dev Removes all the values from a set. O(n). + * + * WARNING: Developers should keep in mind that this function has an unbounded cost and using it may render the + * function uncallable if the set grows to the point where clearing it consumes too much gas to fit in a block. + */ +function clear(${name} storage set) internal { + uint256 len = length(set); + for (uint256 i = 0; i < len; ++i) { + delete set._positions[set._values[i]]; + } + Arrays.unsafeSetLength(set._values, 0); +} + +/** + * @dev Returns true if the value is in the self. O(1). + */ +function contains(${name} storage self, ${value.type} memory value) internal view returns (bool) { + return self._positions[value] != 0; +} + +/** + * @dev Returns the number of values on the self. O(1). + */ +function length(${name} storage self) internal view returns (uint256) { + return self._values.length; +} + +/** + * @dev Returns the value stored at position \`index\` in the self. O(1). + * + * Note that there are no guarantees on the ordering of values inside the + * array, and it may change when more values are added or removed. + * + * Requirements: + * + * - \`index\` must be strictly less than {length}. + */ +function at(${name} storage self, uint256 index) internal view returns (${value.type} memory) { + return self._values[index]; +} + +/** + * @dev Return the entire set in an array + * + * WARNING: This operation will copy the entire storage to memory, which can be quite expensive. This is designed + * to mostly be used by view accessors that are queried without any gas fees. Developers should keep in mind that + * this function has an unbounded cost, and using it as part of a state-changing function may render the function + * uncallable if the set grows to a point where copying to memory consumes too much gas to fit in a block. + */ +function values(${name} storage self) internal view returns (${value.type}[] memory) { + return self._values; +} +`; + +const arraySet = ({ name, value }) => `\ +struct ${name} { + // Storage of set values + ${value.type}[] _values; + // Position is the index of the value in the \`values\` array plus 1. + // Position 0 is used to mean a value is not in the self. + mapping(bytes32 valueHash => uint256) _positions; +} + +/** + * @dev Add a value to a self. O(1). + * + * Returns true if the value was added to the set, that is if it was not + * already present. + */ +function add(${name} storage self, ${value.type} memory value) internal returns (bool) { + if (!contains(self, value)) { + self._values.push(value); + // The value is stored at length-1, but we add 1 to all indexes + // and use 0 as a sentinel value + self._positions[_hash(value)] = self._values.length; + return true; + } else { + return false; + } +} + +/** + * @dev Removes a value from a self. O(1). + * + * Returns true if the value was removed from the set, that is if it was + * present. + */ +function remove(${name} storage self, ${value.type} memory value) internal returns (bool) { + // We cache the value's position to prevent multiple reads from the same storage slot + bytes32 valueHash = _hash(value); + uint256 position = self._positions[valueHash]; + + if (position != 0) { + // Equivalent to contains(self, value) + // To delete an element from the _values array in O(1), we swap the element to delete with the last one in + // the array, and then remove the last element (sometimes called as 'swap and pop'). + // This modifies the order of the array, as noted in {at}. + + uint256 valueIndex = position - 1; + uint256 lastIndex = self._values.length - 1; + + if (valueIndex != lastIndex) { + ${value.type} memory lastValue = self._values[lastIndex]; + + // Move the lastValue to the index where the value to delete is + self._values[valueIndex] = lastValue; + // Update the tracked position of the lastValue (that was just moved) + self._positions[_hash(lastValue)] = position; + } + + // Delete the slot where the moved value was stored + self._values.pop(); + + // Delete the tracked position for the deleted slot + delete self._positions[valueHash]; + + return true; + } else { + return false; + } +} + +/** + * @dev Removes all the values from a set. O(n). + * + * WARNING: Developers should keep in mind that this function has an unbounded cost and using it may render the + * function uncallable if the set grows to the point where clearing it consumes too much gas to fit in a block. + */ +function clear(${name} storage self) internal { + ${value.type}[] storage v = self._values; + + uint256 len = length(self); + for (uint256 i = 0; i < len; ++i) { + delete self._positions[_hash(v[i])]; + } + assembly ("memory-safe") { + sstore(v.slot, 0) + } +} + +/** + * @dev Returns true if the value is in the self. O(1). + */ +function contains(${name} storage self, ${value.type} memory value) internal view returns (bool) { + return self._positions[_hash(value)] != 0; +} + +/** + * @dev Returns the number of values on the self. O(1). + */ +function length(${name} storage self) internal view returns (uint256) { + return self._values.length; +} + +/** + * @dev Returns the value stored at position \`index\` in the self. O(1). + * + * Note that there are no guarantees on the ordering of values inside the + * array, and it may change when more values are added or removed. + * + * Requirements: + * + * - \`index\` must be strictly less than {length}. + */ +function at(${name} storage self, uint256 index) internal view returns (${value.type} memory) { + return self._values[index]; +} + +/** + * @dev Return the entire set in an array + * + * WARNING: This operation will copy the entire storage to memory, which can be quite expensive. This is designed + * to mostly be used by view accessors that are queried without any gas fees. Developers should keep in mind that + * this function has an unbounded cost, and using it as part of a state-changing function may render the function + * uncallable if the set grows to a point where copying to memory consumes too much gas to fit in a block. + */ +function values(${name} storage self) internal view returns (${value.type}[] memory) { + return self._values; +} +`; + +const hashes = `\ +function _hash(bytes32[2] memory value) private pure returns (bytes32) { + return Hashes.efficientKeccak256(value[0], value[1]); +} +`; + +// GENERATE +module.exports = format( + header.trimEnd(), + 'library EnumerableSetExtended {', + format( + [].concat( + SET_TYPES.filter(({ value }) => value.size == 0).map(set), + SET_TYPES.filter(({ value }) => value.size > 0).map(arraySet), + hashes, + ), + ).trimEnd(), + '}', +); diff --git a/test/utils/structs/EnumerableMapExtended.test.js b/test/utils/structs/EnumerableMapExtended.test.js new file mode 100644 index 00000000..ba4942d0 --- /dev/null +++ b/test/utils/structs/EnumerableMapExtended.test.js @@ -0,0 +1,66 @@ +const { ethers } = require('hardhat'); +const { loadFixture } = require('@nomicfoundation/hardhat-network-helpers'); + +const { mapValues } = require('@openzeppelin/contracts/test/helpers/iterate'); +const { generators } = require('@openzeppelin/contracts/test/helpers/random'); +const { MAP_TYPES } = require('../../../scripts/generate/templates/Enumerable.opts'); + +const { shouldBehaveLikeMap } = require('@openzeppelin/contracts/test/utils/structs/EnumerableMap.behavior'); + +async function fixture() { + const mock = await ethers.deployContract('$EnumerableMapExtended'); + + const env = Object.fromEntries( + MAP_TYPES.map(({ name, key, value }) => [ + name, + { + key, + value, + keys: Array.from({ length: 3 }, generators[key.type]), + values: Array.from({ length: 3 }, generators[value.type]), + zeroValue: generators[value.type].zero, + methods: mapValues( + { + set: `$set(uint256,${key.type},${value.type})`, + get: `$get(uint256,${key.type})`, + tryGet: `$tryGet(uint256,${key.type})`, + remove: `$remove(uint256,${key.type})`, + clear: `$clear_EnumerableMapExtended_${name}(uint256)`, + length: `$length_EnumerableMapExtended_${name}(uint256)`, + at: `$at_EnumerableMapExtended_${name}(uint256,uint256)`, + contains: `$contains(uint256,${key.type})`, + keys: `$keys_EnumerableMapExtended_${name}(uint256)`, + }, + fnSig => + (...args) => + mock.getFunction(fnSig)(0, ...args), + ), + events: { + setReturn: `return$set_EnumerableMapExtended_${name}_${key.type}_${value.type}`, + removeReturn: `return$remove_EnumerableMapExtended_${name}_${key.type}`, + }, + error: key.memory || value.memory ? `EnumerableMapNonexistent${key.name}Key` : `EnumerableMapNonexistentKey`, + }, + ]), + ); + + return { mock, env }; +} + +describe('EnumerableMapExtended', function () { + beforeEach(async function () { + Object.assign(this, await loadFixture(fixture)); + }); + + for (const { name, key, value } of MAP_TYPES) { + describe(`${name} (enumerable map from ${key.type} to ${value.type})`, function () { + beforeEach(async function () { + Object.assign(this, this.env[name]); + [this.keyA, this.keyB, this.keyC] = this.keys; + [this.valueA, this.valueB, this.valueC] = this.values; + }); + + shouldBehaveLikeMap(); + }); + } +}); diff --git a/test/utils/structs/EnumerableSetExtended.test.js b/test/utils/structs/EnumerableSetExtended.test.js new file mode 100644 index 00000000..d3f698c7 --- /dev/null +++ b/test/utils/structs/EnumerableSetExtended.test.js @@ -0,0 +1,62 @@ +const { ethers } = require('hardhat'); +const { loadFixture } = require('@nomicfoundation/hardhat-network-helpers'); + +const { mapValues } = require('@openzeppelin/contracts/test/helpers/iterate'); +const { generators } = require('@openzeppelin/contracts/test/helpers/random'); +const { SET_TYPES } = require('../../../scripts/generate/templates/Enumerable.opts'); + +const { shouldBehaveLikeSet } = require('@openzeppelin/contracts/test/utils/structs/EnumerableSet.behavior'); + +async function fixture() { + const mock = await ethers.deployContract('$EnumerableSetExtended'); + + const env = Object.fromEntries( + SET_TYPES.map(({ name, value }) => [ + name, + { + value, + values: Array.from( + { length: 3 }, + value.size ? () => Array.from({ length: value.size }, generators[value.base]) : generators[value.type], + ), + methods: mapValues( + { + add: `$add(uint256,${value.type})`, + remove: `$remove(uint256,${value.type})`, + contains: `$contains(uint256,${value.type})`, + clear: `$clear_EnumerableSetExtended_${name}(uint256)`, + length: `$length_EnumerableSetExtended_${name}(uint256)`, + at: `$at_EnumerableSetExtended_${name}(uint256,uint256)`, + values: `$values_EnumerableSetExtended_${name}(uint256)`, + }, + fnSig => + (...args) => + mock.getFunction(fnSig)(0, ...args), + ), + events: { + addReturn: `return$add_EnumerableSetExtended_${name}_${value.type.replace(/[[\]]/g, '_')}`, + removeReturn: `return$remove_EnumerableSetExtended_${name}_${value.type.replace(/[[\]]/g, '_')}`, + }, + }, + ]), + ); + + return { mock, env }; +} + +describe('EnumerableSetExtended', function () { + beforeEach(async function () { + Object.assign(this, await loadFixture(fixture)); + }); + + for (const { name, value } of SET_TYPES) { + describe(`${name} (enumerable set of ${value.type})`, function () { + beforeEach(function () { + Object.assign(this, this.env[name]); + [this.valueA, this.valueB, this.valueC] = this.values; + }); + + shouldBehaveLikeSet(); + }); + } +}); From 66d0a96403fbc003561bb3324fff0aa6ea2bba20 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Thu, 13 Mar 2025 15:53:28 +0100 Subject: [PATCH 2/7] Test generation in CI --- .github/workflows/checks.yml | 2 ++ package.json | 1 + scripts/checks/generation.sh | 6 ++++++ 3 files changed, 9 insertions(+) create mode 100755 scripts/checks/generation.sh diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index fce5b1c8..81a62184 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -45,6 +45,8 @@ jobs: run: npm run test:inheritance - name: Check pragma consistency between files run: npm run test:pragma + - name: Check procedurally generated contracts are up-to-date + run: npm run test:generation coverage: runs-on: ubuntu-latest diff --git a/package.json b/package.json index ebebaaea..5cb35152 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "coverage": "scripts/checks/coverage.sh", "generate": "scripts/generate/run.js", "test": "hardhat test", + "test:generation": "scripts/checks/generation.sh", "test:inheritance": "scripts/checks/inheritance-ordering.js artifacts/build-info/*", "test:pragma": "scripts/checks/pragma-consistency.js artifacts/build-info/*" }, diff --git a/scripts/checks/generation.sh b/scripts/checks/generation.sh new file mode 100755 index 00000000..00d609f9 --- /dev/null +++ b/scripts/checks/generation.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -euo pipefail + +npm run generate +git diff -R --exit-code From 87008a516d9f4903f28004243c6ea63477042be8 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Thu, 13 Mar 2025 15:56:41 +0100 Subject: [PATCH 3/7] add Changelog entry --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b94f3acd..1e1f5ea7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## XX-XX-XXXX + +- `EnumerableSetExtended` and `EnumerableMapExtended`: Extensions of the `EnumerableSet` and `EnumerableMap` libraries with more types, including non-value types. + ## 31-01-2025 - `PaymasterCore`: Add a simple ERC-4337 paymaster implementation with minimal logic. From 8af16c5115ac50b51034ac43e6eeeac7f6316a16 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Thu, 13 Mar 2025 16:00:39 +0100 Subject: [PATCH 4/7] documentation --- contracts/utils/README.adoc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/contracts/utils/README.adoc b/contracts/utils/README.adoc index bb4c33f3..f8d0ab15 100644 --- a/contracts/utils/README.adoc +++ b/contracts/utils/README.adoc @@ -9,6 +9,7 @@ Miscellaneous contracts and libraries containing utility functions you can use t * {ERC7739}: An abstract contract to validate signatures following the rehashing scheme from `ERC7739Utils`. * {ERC7739Utils}: Utilities library that implements a defensive rehashing mechanism to prevent replayability of smart contract signatures based on ERC-7739. * {SignerECDSA}, {SignerP256}, {SignerRSA}: Implementations of an {AbstractSigner} with specific signature validation algorithms. + * {EnumerableSetExtended} and {EnumerableMapExtended}: Extensions of the `EnumerableSet` and `EnumerableMap` libraries with more types, including non-value types. * {Masks}: Library to handle `bytes32` masks. == Cryptography @@ -25,6 +26,12 @@ Miscellaneous contracts and libraries containing utility functions you can use t {{SignerRSA}} +== Structs + +{{EnumerableSetExtended}} + +{{EnumerableMapExtended}} + == Libraries {{Masks}} From 0796a043ea6617413b9f085e4ad8e04f6e7ad970 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Thu, 13 Mar 2025 16:06:56 +0100 Subject: [PATCH 5/7] update packge-lock.json --- package-lock.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4e285b47..09d1f1a3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,7 +39,7 @@ "eslint-config-prettier": "^9.0.0", "ethers": "^6.13.4", "glob": "^11.0.0", - "globals": "^15.3.0", + "globals": "^16.0.0", "graphlib": "^2.1.8", "hardhat": "^2.22.2", "hardhat-exposed": "^0.3.15", @@ -59,7 +59,7 @@ "solidity-ast": "^0.4.50", "solidity-coverage": "^0.8.5", "solidity-docgen": "^0.6.0-beta.29", - "undici": "^7.0.0", + "undici": "^7.4.0", "yargs": "^17.0.0" } }, @@ -4986,9 +4986,9 @@ } }, "node_modules/globals": { - "version": "15.13.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-15.13.0.tgz", - "integrity": "sha512-49TewVEz0UxZjr1WYYsWpPrhyC/B/pA8Bq0fUmet2n+eR7yn0IvNzNaoBwnK6mdkzcN+se7Ez9zUgULTz2QH4g==", + "version": "16.0.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-16.0.0.tgz", + "integrity": "sha512-iInW14XItCXET01CQFqudPOWP2jYMl7T+QRQT+UNcR/iQncN/F0UNpgd76iFkBPgNQb4+X3LV9tLJYzwh+Gl3A==", "dev": true, "license": "MIT", "engines": { @@ -9612,9 +9612,9 @@ } }, "node_modules/undici": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.2.0.tgz", - "integrity": "sha512-klt+0S55GBViA9nsq48/NSCo4YX5mjydjypxD7UmHh/brMu8h/Mhd/F7qAeoH2NOO8SDTk6kjnTFc4WpzmfYpQ==", + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.5.0.tgz", + "integrity": "sha512-NFQG741e8mJ0fLQk90xKxFdaSM7z4+IQpAgsFI36bCDY9Z2+aXXZjVy2uUksMouWfMI9+w5ejOq5zYYTBCQJDQ==", "dev": true, "license": "MIT", "engines": { From 255f0b5a61f0281a4e585a48a0ab9329a3a038cb Mon Sep 17 00:00:00 2001 From: ernestognw Date: Fri, 11 Apr 2025 21:04:38 -0600 Subject: [PATCH 6/7] Improve docs --- .../utils/structs/EnumerableMapExtended.sol | 36 +++++++++- .../utils/structs/EnumerableSetExtended.sol | 70 +++++++++++++------ lib/@openzeppelin-contracts | 2 +- lib/@openzeppelin-contracts-upgradeable | 2 +- lib/forge-std | 2 +- scripts/generate/run.js | 4 +- scripts/generate/templates/Enumerable.opts.js | 17 ----- .../templates/EnumerableMapExtended.js | 36 +++++++++- .../templates/EnumerableSetExtended.js | 58 +++++++++++---- 9 files changed, 167 insertions(+), 60 deletions(-) diff --git a/contracts/utils/structs/EnumerableMapExtended.sol b/contracts/utils/structs/EnumerableMapExtended.sol index a2468ca4..b568dec1 100644 --- a/contracts/utils/structs/EnumerableMapExtended.sol +++ b/contracts/utils/structs/EnumerableMapExtended.sol @@ -9,9 +9,41 @@ import {EnumerableSetExtended} from "./EnumerableSetExtended.sol"; /** * @dev Library for managing an enumerable variant of Solidity's * https://solidity.readthedocs.io/en/latest/types.html#mapping-types[`mapping`] - * type. + * type for non-value types as keys. * - * Note: Extensions of openzeppelin/contracts/utils/struct/EnumerableMap.sol. + * Maps have the following properties: + * + * - Entries are added, removed, and checked for existence in constant time + * (O(1)). + * - Entries are enumerated in O(n). No guarantees are made on the ordering. + * - Map can be cleared (all entries removed) in O(n). + * + * ```solidity + * contract Example { + * // Add the library methods + * using EnumerableMapExtended for EnumerableMapExtended.BytesToUintMap; + * + * // Declare a set state variable + * EnumerableMapExtended.BytesToUintMap private myMap; + * } + * ``` + * + * The following map types are supported: + * + * - `bytes -> uint256` (`BytesToUintMap`) + * - `string -> string` (`StringToStringMap`) + * + * [WARNING] + * ==== + * Trying to delete such a structure from storage will likely result in data corruption, rendering the structure + * unusable. + * See https://github.com/ethereum/solidity/pull/11843[ethereum/solidity#11843] for more info. + * + * In order to clean an EnumerableMap, you can either remove all elements one by one or create a fresh instance using an + * array of EnumerableMap. + * ==== + * + * NOTE: Extensions of openzeppelin/contracts/utils/struct/EnumerableMap.sol. */ library EnumerableMapExtended { using EnumerableSet for *; diff --git a/contracts/utils/structs/EnumerableSetExtended.sol b/contracts/utils/structs/EnumerableSetExtended.sol index 819b27c4..cc38406b 100644 --- a/contracts/utils/structs/EnumerableSetExtended.sol +++ b/contracts/utils/structs/EnumerableSetExtended.sol @@ -8,22 +8,52 @@ import {Hashes} from "@openzeppelin/contracts/utils/cryptography/Hashes.sol"; /** * @dev Library for managing - * https://en.wikipedia.org/wiki/Set_(abstract_data_type)[sets] of primitive + * https://en.wikipedia.org/wiki/Set_(abstract_data_type)[sets] of non-value * types. * - * Note: Extensions of openzeppelin/contracts/utils/struct/EnumerableSet.sol. + * Sets have the following properties: + * + * - Elements are added, removed, and checked for existence in constant time + * (O(1)). + * - Elements are enumerated in O(n). No guarantees are made on the ordering. + * - Set can be cleared (all elements removed) in O(n). + * + * ```solidity + * contract Example { + * // Add the library methods + * using EnumerableSetExtended for EnumerableSetExtended.StringSet; + * + * // Declare a set state variable + * EnumerableSetExtended.StringSet private mySet; + * } + * ``` + * + * Sets of type `string` (`StringSet`), `bytes` (`BytesSet`) and + * `bytes32[2]` (`Bytes32x2Set`) are supported. + * + * [WARNING] + * ==== + * Trying to delete such a structure from storage will likely result in data corruption, rendering the structure + * unusable. + * See https://github.com/ethereum/solidity/pull/11843[ethereum/solidity#11843] for more info. + * + * In order to clean an EnumerableSet, you can either remove all elements one by one or create a fresh instance using an + * array of EnumerableSet. + * ==== + * + * NOTE: This is an extension of openzeppelin/contracts/utils/struct/EnumerableSet.sol. */ library EnumerableSetExtended { struct StringSet { // Storage of set values string[] _values; // Position is the index of the value in the `values` array plus 1. - // Position 0 is used to mean a value is not in the self. + // Position 0 is used to mean a value is not in the set. mapping(string value => uint256) _positions; } /** - * @dev Add a value to a self. O(1). + * @dev Add a value to a set. O(1). * * Returns true if the value was added to the set, that is if it was not * already present. @@ -41,7 +71,7 @@ library EnumerableSetExtended { } /** - * @dev Removes a value from a self. O(1). + * @dev Removes a value from a set. O(1). * * Returns true if the value was removed from the set, that is if it was * present. @@ -95,21 +125,21 @@ library EnumerableSetExtended { } /** - * @dev Returns true if the value is in the self. O(1). + * @dev Returns true if the value is in the set. O(1). */ function contains(StringSet storage self, string memory value) internal view returns (bool) { return self._positions[value] != 0; } /** - * @dev Returns the number of values on the self. O(1). + * @dev Returns the number of values on the set. O(1). */ function length(StringSet storage self) internal view returns (uint256) { return self._values.length; } /** - * @dev Returns the value stored at position `index` in the self. O(1). + * @dev Returns the value stored at position `index` in the set. O(1). * * Note that there are no guarantees on the ordering of values inside the * array, and it may change when more values are added or removed. @@ -138,12 +168,12 @@ library EnumerableSetExtended { // Storage of set values bytes[] _values; // Position is the index of the value in the `values` array plus 1. - // Position 0 is used to mean a value is not in the self. + // Position 0 is used to mean a value is not in the set. mapping(bytes value => uint256) _positions; } /** - * @dev Add a value to a self. O(1). + * @dev Add a value to a set. O(1). * * Returns true if the value was added to the set, that is if it was not * already present. @@ -161,7 +191,7 @@ library EnumerableSetExtended { } /** - * @dev Removes a value from a self. O(1). + * @dev Removes a value from a set. O(1). * * Returns true if the value was removed from the set, that is if it was * present. @@ -215,21 +245,21 @@ library EnumerableSetExtended { } /** - * @dev Returns true if the value is in the self. O(1). + * @dev Returns true if the value is in the set. O(1). */ function contains(BytesSet storage self, bytes memory value) internal view returns (bool) { return self._positions[value] != 0; } /** - * @dev Returns the number of values on the self. O(1). + * @dev Returns the number of values on the set. O(1). */ function length(BytesSet storage self) internal view returns (uint256) { return self._values.length; } /** - * @dev Returns the value stored at position `index` in the self. O(1). + * @dev Returns the value stored at position `index` in the set. O(1). * * Note that there are no guarantees on the ordering of values inside the * array, and it may change when more values are added or removed. @@ -258,12 +288,12 @@ library EnumerableSetExtended { // Storage of set values bytes32[2][] _values; // Position is the index of the value in the `values` array plus 1. - // Position 0 is used to mean a value is not in the self. + // Position 0 is used to mean a value is not in the set. mapping(bytes32 valueHash => uint256) _positions; } /** - * @dev Add a value to a self. O(1). + * @dev Add a value to a set. O(1). * * Returns true if the value was added to the set, that is if it was not * already present. @@ -281,7 +311,7 @@ library EnumerableSetExtended { } /** - * @dev Removes a value from a self. O(1). + * @dev Removes a value from a set. O(1). * * Returns true if the value was removed from the set, that is if it was * present. @@ -340,21 +370,21 @@ library EnumerableSetExtended { } /** - * @dev Returns true if the value is in the self. O(1). + * @dev Returns true if the value is in the set. O(1). */ function contains(Bytes32x2Set storage self, bytes32[2] memory value) internal view returns (bool) { return self._positions[_hash(value)] != 0; } /** - * @dev Returns the number of values on the self. O(1). + * @dev Returns the number of values on the set. O(1). */ function length(Bytes32x2Set storage self) internal view returns (uint256) { return self._values.length; } /** - * @dev Returns the value stored at position `index` in the self. O(1). + * @dev Returns the value stored at position `index` in the set. O(1). * * Note that there are no guarantees on the ordering of values inside the * array, and it may change when more values are added or removed. diff --git a/lib/@openzeppelin-contracts b/lib/@openzeppelin-contracts index 0a77e54c..21c8312b 160000 --- a/lib/@openzeppelin-contracts +++ b/lib/@openzeppelin-contracts @@ -1 +1 @@ -Subproject commit 0a77e54c307d9becf0473b73541c4f9d3b2743a3 +Subproject commit 21c8312b022f495ebe3621d5daeed20552b43ff9 diff --git a/lib/@openzeppelin-contracts-upgradeable b/lib/@openzeppelin-contracts-upgradeable index 5fc3fee1..e874681b 160000 --- a/lib/@openzeppelin-contracts-upgradeable +++ b/lib/@openzeppelin-contracts-upgradeable @@ -1 +1 @@ -Subproject commit 5fc3fee14043035097ae718387200f4f4daaa982 +Subproject commit e874681bed440d5ef99d9c960f4b6c6d13950616 diff --git a/lib/forge-std b/lib/forge-std index 8ba9031f..6abf6698 160000 --- a/lib/forge-std +++ b/lib/forge-std @@ -1 +1 @@ -Subproject commit 8ba9031ffcbe25aa0d1224d3ca263a995026e477 +Subproject commit 6abf66980050ab03a35b52bdab814f55001d6929 diff --git a/scripts/generate/run.js b/scripts/generate/run.js index 2f54e4cf..ac0b7f12 100755 --- a/scripts/generate/run.js +++ b/scripts/generate/run.js @@ -13,7 +13,7 @@ function getVersion(path) { } } -function generateFromTemplate(file, template, outputPrefix = '', lint = false) { +function generateFromTemplate(file, template, outputPrefix = '') { const script = path.relative(path.join(__dirname, '../..'), __filename); const input = path.join(path.dirname(script), template); const output = path.join(outputPrefix, file); @@ -26,7 +26,7 @@ function generateFromTemplate(file, template, outputPrefix = '', lint = false) { require(template).trimEnd(), ); fs.writeFileSync(output, content); - lint && cp.execFileSync('prettier', ['--write', output]); + cp.execFileSync('prettier', ['--write', output]); } // Some templates needs to go through the linter after generation diff --git a/scripts/generate/templates/Enumerable.opts.js b/scripts/generate/templates/Enumerable.opts.js index 34c09c5c..0691b8d5 100644 --- a/scripts/generate/templates/Enumerable.opts.js +++ b/scripts/generate/templates/Enumerable.opts.js @@ -23,9 +23,6 @@ const toMapTypeDescr = ({ key, value }) => ({ }); const SET_TYPES = [ - // { type: 'bytes32' }, // part of the vanilla repo - // { type: 'address' }, // part of the vanilla repo - // { type: 'uint256' }, // part of the vanilla repo { type: 'bytes32', size: 2 }, { type: 'string', memory: true }, { type: 'bytes', memory: true }, @@ -34,26 +31,12 @@ const SET_TYPES = [ .map(toSetTypeDescr); const MAP_TYPES = [ - // { key: { type: 'uint256' }, value: { type: 'uint256' } }, // part of the vanilla repo - // { key: { type: 'uint256' }, value: { type: 'address' } }, // part of the vanilla repo - // { key: { type: 'uint256' }, value: { type: 'bytes32' } }, // part of the vanilla repo - // { key: { type: 'address' }, value: { type: 'uint256' } }, // part of the vanilla repo - // { key: { type: 'address' }, value: { type: 'address' } }, // part of the vanilla repo - // { key: { type: 'address' }, value: { type: 'bytes32' } }, // part of the vanilla repo - // { key: { type: 'bytes32' }, value: { type: 'uint256' } }, // part of the vanilla repo - // { key: { type: 'bytes32' }, value: { type: 'address' } }, // part of the vanilla repo { key: { type: 'bytes', memory: true }, value: { type: 'uint256' } }, { key: { type: 'string', memory: true }, value: { type: 'string', memory: true } }, ] .map(entry => mapValues(entry, typeDescr)) .map(toMapTypeDescr); -/// Sanity - Disabled because some types might be provided by the vanilla repository. -// MAP_TYPES.forEach(entry => { -// if (!SET_TYPES.some(set => set.structName == entry.key.structName)) -// throw new Error(`${entry.structName} requires a "${entry.key.structName}" set of "${entry.key.type}"`); -// }); - module.exports = { SET_TYPES, MAP_TYPES, diff --git a/scripts/generate/templates/EnumerableMapExtended.js b/scripts/generate/templates/EnumerableMapExtended.js index 8d0c1096..89277fd3 100644 --- a/scripts/generate/templates/EnumerableMapExtended.js +++ b/scripts/generate/templates/EnumerableMapExtended.js @@ -10,9 +10,41 @@ import {EnumerableSetExtended} from "./EnumerableSetExtended.sol"; /** * @dev Library for managing an enumerable variant of Solidity's * https://solidity.readthedocs.io/en/latest/types.html#mapping-types[\`mapping\`] - * type. + * type for non-value types as keys. * - * Note: Extensions of openzeppelin/contracts/utils/struct/EnumerableMap.sol. + * Maps have the following properties: + * + * - Entries are added, removed, and checked for existence in constant time + * (O(1)). + * - Entries are enumerated in O(n). No guarantees are made on the ordering. + * - Map can be cleared (all entries removed) in O(n). + * + * \`\`\`solidity + * contract Example { + * // Add the library methods + * using EnumerableMapExtended for EnumerableMapExtended.BytesToUintMap; + * + * // Declare a set state variable + * EnumerableMapExtended.BytesToUintMap private myMap; + * } + * \`\`\` + * + * The following map types are supported: + * + * - \`bytes -> uint256\` (\`BytesToUintMap\`) + * - \`string -> string\` (\`StringToStringMap\`) + * + * [WARNING] + * ==== + * Trying to delete such a structure from storage will likely result in data corruption, rendering the structure + * unusable. + * See https://github.com/ethereum/solidity/pull/11843[ethereum/solidity#11843] for more info. + * + * In order to clean an EnumerableMap, you can either remove all elements one by one or create a fresh instance using an + * array of EnumerableMap. + * ==== + * + * NOTE: Extensions of openzeppelin/contracts/utils/struct/EnumerableMap.sol. */ `; diff --git a/scripts/generate/templates/EnumerableSetExtended.js b/scripts/generate/templates/EnumerableSetExtended.js index d1a2f19f..54d56f80 100644 --- a/scripts/generate/templates/EnumerableSetExtended.js +++ b/scripts/generate/templates/EnumerableSetExtended.js @@ -9,10 +9,40 @@ import {Hashes} from "@openzeppelin/contracts/utils/cryptography/Hashes.sol"; /** * @dev Library for managing - * https://en.wikipedia.org/wiki/Set_(abstract_data_type)[sets] of primitive + * https://en.wikipedia.org/wiki/Set_(abstract_data_type)[sets] of non-value * types. * - * Note: Extensions of openzeppelin/contracts/utils/struct/EnumerableSet.sol. + * Sets have the following properties: + * + * - Elements are added, removed, and checked for existence in constant time + * (O(1)). + * - Elements are enumerated in O(n). No guarantees are made on the ordering. + * - Set can be cleared (all elements removed) in O(n). + * + * \`\`\`solidity + * contract Example { + * // Add the library methods + * using EnumerableSetExtended for EnumerableSetExtended.StringSet; + * + * // Declare a set state variable + * EnumerableSetExtended.StringSet private mySet; + * } + * \`\`\` + * + * Sets of type \`string\` (\`StringSet\`), \`bytes\` (\`BytesSet\`) and + * \`bytes32[2]\` (\`Bytes32x2Set\`) are supported. + * + * [WARNING] + * ==== + * Trying to delete such a structure from storage will likely result in data corruption, rendering the structure + * unusable. + * See https://github.com/ethereum/solidity/pull/11843[ethereum/solidity#11843] for more info. + * + * In order to clean an EnumerableSet, you can either remove all elements one by one or create a fresh instance using an + * array of EnumerableSet. + * ==== + * + * NOTE: This is an extension of openzeppelin/contracts/utils/struct/EnumerableSet.sol. */ `; @@ -21,12 +51,12 @@ struct ${name} { // Storage of set values ${value.type}[] _values; // Position is the index of the value in the \`values\` array plus 1. - // Position 0 is used to mean a value is not in the self. + // Position 0 is used to mean a value is not in the set. mapping(${value.type} value => uint256) _positions; } /** - * @dev Add a value to a self. O(1). + * @dev Add a value to a set. O(1). * * Returns true if the value was added to the set, that is if it was not * already present. @@ -44,7 +74,7 @@ function add(${name} storage self, ${value.type} memory value) internal returns } /** - * @dev Removes a value from a self. O(1). + * @dev Removes a value from a set. O(1). * * Returns true if the value was removed from the set, that is if it was * present. @@ -98,21 +128,21 @@ function clear(${name} storage set) internal { } /** - * @dev Returns true if the value is in the self. O(1). + * @dev Returns true if the value is in the set. O(1). */ function contains(${name} storage self, ${value.type} memory value) internal view returns (bool) { return self._positions[value] != 0; } /** - * @dev Returns the number of values on the self. O(1). + * @dev Returns the number of values on the set. O(1). */ function length(${name} storage self) internal view returns (uint256) { return self._values.length; } /** - * @dev Returns the value stored at position \`index\` in the self. O(1). + * @dev Returns the value stored at position \`index\` in the set. O(1). * * Note that there are no guarantees on the ordering of values inside the * array, and it may change when more values are added or removed. @@ -143,12 +173,12 @@ struct ${name} { // Storage of set values ${value.type}[] _values; // Position is the index of the value in the \`values\` array plus 1. - // Position 0 is used to mean a value is not in the self. + // Position 0 is used to mean a value is not in the set. mapping(bytes32 valueHash => uint256) _positions; } /** - * @dev Add a value to a self. O(1). + * @dev Add a value to a set. O(1). * * Returns true if the value was added to the set, that is if it was not * already present. @@ -166,7 +196,7 @@ function add(${name} storage self, ${value.type} memory value) internal returns } /** - * @dev Removes a value from a self. O(1). + * @dev Removes a value from a set. O(1). * * Returns true if the value was removed from the set, that is if it was * present. @@ -225,21 +255,21 @@ function clear(${name} storage self) internal { } /** - * @dev Returns true if the value is in the self. O(1). + * @dev Returns true if the value is in the set. O(1). */ function contains(${name} storage self, ${value.type} memory value) internal view returns (bool) { return self._positions[_hash(value)] != 0; } /** - * @dev Returns the number of values on the self. O(1). + * @dev Returns the number of values on the set. O(1). */ function length(${name} storage self) internal view returns (uint256) { return self._values.length; } /** - * @dev Returns the value stored at position \`index\` in the self. O(1). + * @dev Returns the value stored at position \`index\` in the set. O(1). * * Note that there are no guarantees on the ordering of values inside the * array, and it may change when more values are added or removed. From c3d966e461446fac49df3d7427a8033047eb8384 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Fri, 11 Apr 2025 21:06:06 -0600 Subject: [PATCH 7/7] up --- scripts/generate/run.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/generate/run.js b/scripts/generate/run.js index ac0b7f12..639e9478 100755 --- a/scripts/generate/run.js +++ b/scripts/generate/run.js @@ -29,13 +29,10 @@ function generateFromTemplate(file, template, outputPrefix = '') { cp.execFileSync('prettier', ['--write', output]); } -// Some templates needs to go through the linter after generation -const needsLinter = ['utils/structs/EnumerableMapExtended.sol']; - // Contracts for (const [file, template] of Object.entries({ 'utils/structs/EnumerableSetExtended.sol': './templates/EnumerableSetExtended.js', 'utils/structs/EnumerableMapExtended.sol': './templates/EnumerableMapExtended.js', })) { - generateFromTemplate(file, template, './contracts/', needsLinter.indexOf(file) != -1); + generateFromTemplate(file, template, './contracts/'); }