From e1a9fce436cef6a25ec1f3acf295d9d609a24db1 Mon Sep 17 00:00:00 2001 From: cyl19970726 <15258378443@163.com> Date: Fri, 16 Dec 2022 13:29:01 +0800 Subject: [PATCH] feat: merge LargeStorageManager into git3 project --- contracts/Git3.sol | 3 +- contracts/IFileOperator.sol | 52 ---- contracts/v2/LargeStorageManagerV2.sol | 278 ++++++++++++++++++ contracts/v2/LargerStorageManagerV2Test.sol | 84 ++++++ contracts/v2/Memory.sol | 187 ++++++++++++ contracts/v2/StorageHelperV2.sol | 224 ++++++++++++++ contracts/v2/StorageSlotFactory.sol | 51 ++++ .../v2/StorageSlotSelfDestructableV2.sol | 32 ++ contracts/v2/optimize/SlotHelper.sol | 151 ++++++++++ contracts/v2/optimize/SlotHelperTest.sol | 47 +++ package.json | 1 - test/large-storage-helper-v2-test.js | 138 +++++++++ test/slot-helper-test.js | 80 +++++ 13 files changed, 1273 insertions(+), 55 deletions(-) delete mode 100644 contracts/IFileOperator.sol create mode 100644 contracts/v2/LargeStorageManagerV2.sol create mode 100644 contracts/v2/LargerStorageManagerV2Test.sol create mode 100644 contracts/v2/Memory.sol create mode 100644 contracts/v2/StorageHelperV2.sol create mode 100644 contracts/v2/StorageSlotFactory.sol create mode 100644 contracts/v2/StorageSlotSelfDestructableV2.sol create mode 100644 contracts/v2/optimize/SlotHelper.sol create mode 100644 contracts/v2/optimize/SlotHelperTest.sol create mode 100644 test/large-storage-helper-v2-test.js create mode 100644 test/slot-helper-test.js diff --git a/contracts/Git3.sol b/contracts/Git3.sol index 60c825f..43eb84e 100644 --- a/contracts/Git3.sol +++ b/contracts/Git3.sol @@ -2,9 +2,8 @@ pragma solidity ^0.8.0; import "hardhat/console.sol"; -import "./IFileOperator.sol"; import "@openzeppelin/contracts/access/Ownable.sol"; -import "git3-evm-large-storage/contracts/v2/LargeStorageManagerV2.sol"; +import "./v2/LargeStorageManagerV2.sol"; contract Git3 is LargeStorageManagerV2 { struct refInfo { diff --git a/contracts/IFileOperator.sol b/contracts/IFileOperator.sol deleted file mode 100644 index 4f7b80e..0000000 --- a/contracts/IFileOperator.sol +++ /dev/null @@ -1,52 +0,0 @@ -//SPDX-License-Identifier: Unlicense -pragma solidity ^0.8.0; - -interface IFileOperator { - // Large storage methods - function write(bytes memory name, bytes memory data) external payable; - - function read(bytes memory name) external view returns (bytes memory, bool); - - // return (size, # of chunks) - function size(bytes memory name) external view returns (uint256, uint256); - - function remove(bytes memory name) external returns (uint256); - - function countChunks(bytes memory name) external view returns (uint256); - - // Chunk-based large storage methods - function writeChunk( - bytes memory name, - uint256 chunkId, - bytes memory data - ) external payable; - - function readChunk( - bytes memory name, - uint256 chunkId - ) external view returns (bytes memory, bool); - - function chunkSize( - bytes memory name, - uint256 chunkId - ) external view returns (uint256, bool); - - function removeChunk( - bytes memory name, - uint256 chunkId - ) external returns (bool); - - function truncate( - bytes memory name, - uint256 chunkId - ) external returns (uint256); - - function refund() external; - - function destruct() external; - - function getChunkHash( - bytes memory name, - uint256 chunkId - ) external view returns (bytes32); -} diff --git a/contracts/v2/LargeStorageManagerV2.sol b/contracts/v2/LargeStorageManagerV2.sol new file mode 100644 index 0000000..c912c7b --- /dev/null +++ b/contracts/v2/LargeStorageManagerV2.sol @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +import "./optimize/SlotHelper.sol"; +import "./StorageHelperV2.sol"; +import "./StorageSlotSelfDestructableV2.sol"; + +// Large storage manager to support arbitrarily-sized data with multiple chunk +contract LargeStorageManagerV2 { + using SlotHelper for bytes32; + using SlotHelper for address; + + uint8 internal immutable SLOT_LIMIT; + + mapping(bytes32 => mapping(uint256 => bytes32)) internal keyToMetadata; + mapping(bytes32 => mapping(uint256 => mapping(uint256 => bytes32))) + internal keyToSlots; + + constructor(uint8 slotLimit) { + SLOT_LIMIT = slotLimit; + } + + function isOptimize() public view returns (bool) { + return SLOT_LIMIT > 0; + } + + function _preparePut(bytes32 key, uint256 chunkId) private { + bytes32 metadata = keyToMetadata[key][chunkId]; + + if (metadata == bytes32(0)) { + require( + chunkId == 0 || keyToMetadata[key][chunkId - 1] != bytes32(0x0), + "must replace or append" + ); + } + + if (!metadata.isInSlot()) { + address addr = metadata.bytes32ToAddr(); + if (addr != address(0x0)) { + // remove the KV first if it exists + StorageSlotSelfDestructableV2(addr).destruct(); + } + } + } + + function _putChunkFromCalldata( + bytes32 key, + uint256 chunkId, + bytes calldata data, + uint256 value + ) internal { + _preparePut(key, chunkId); + + // store data and rewrite metadata + if (data.length > SLOT_LIMIT) { + keyToMetadata[key][chunkId] = StorageHelperV2 + .putRawFromCalldata(data, value) + .addrToBytes32(); + } else { + keyToMetadata[key][chunkId] = SlotHelper.putRaw( + keyToSlots[key][chunkId], + data + ); + } + } + + function _putChunk( + bytes32 key, + uint256 chunkId, + bytes memory data, + uint256 value + ) internal { + _preparePut(key, chunkId); + + // store data and rewrite metadata + if (data.length > SLOT_LIMIT) { + keyToMetadata[key][chunkId] = StorageHelperV2 + .putRaw(data, value) + .addrToBytes32(); + } else { + keyToMetadata[key][chunkId] = SlotHelper.putRaw( + keyToSlots[key][chunkId], + data + ); + } + } + + function _getChunkAddr( + bytes32 key, + uint256 chunkId + ) internal view returns (address) { + bytes32 metadata = keyToMetadata[key][chunkId]; + address addr = metadata.bytes32ToAddr(); + return addr; + } + + function _getChunk( + bytes32 key, + uint256 chunkId + ) internal view returns (bytes memory, bool) { + bytes32 metadata = keyToMetadata[key][chunkId]; + + if (metadata.isInSlot()) { + bytes memory res = SlotHelper.getRaw( + keyToSlots[key][chunkId], + metadata + ); + return (res, true); + } else { + address addr = metadata.bytes32ToAddr(); + return StorageHelperV2.getRaw(addr); + } + } + + function _stakeTokens( + bytes32 key, + uint256 chunkId + ) internal view returns (uint256) { + uint256 stakeNum = 0; + + while (true) { + (uint256 count, bool found) = _chunkStakeTokens(key, chunkId); + if (!found) { + return stakeNum; + } + stakeNum += count; + chunkId++; + } + + return stakeNum; + } + + function _chunkStakeTokens( + bytes32 key, + uint256 chunkId + ) internal view returns (uint256, bool) { + bytes32 metadata = keyToMetadata[key][chunkId]; + if (metadata == bytes32(0)) { + return (0, false); + } else if (metadata.isInSlot()) { + return (0, true); + } else { + address addr = metadata.bytes32ToAddr(); + return (addr.balance, true); + } + } + + function _chunkSize( + bytes32 key, + uint256 chunkId + ) internal view returns (uint256, bool) { + bytes32 metadata = keyToMetadata[key][chunkId]; + + if (metadata == bytes32(0)) { + return (0, false); + } else if (metadata.isInSlot()) { + uint256 len = metadata.decodeLen(); + return (len, true); + } else { + address addr = metadata.bytes32ToAddr(); + return StorageHelperV2.sizeRaw(addr); + } + } + + function _countChunks(bytes32 key) internal view returns (uint256) { + uint256 chunkId = 0; + + while (true) { + bytes32 metadata = keyToMetadata[key][chunkId]; + if (metadata == bytes32(0x0)) { + break; + } + + chunkId++; + } + + return chunkId; + } + + // Returns (size, # of chunks). + function _size(bytes32 key) internal view returns (uint256, uint256) { + uint256 size = 0; + uint256 chunkId = 0; + + while (true) { + (uint256 chunkSize, bool found) = _chunkSize(key, chunkId); + if (!found) { + break; + } + + size += chunkSize; + chunkId++; + } + + return (size, chunkId); + } + + function _get(bytes32 key) internal view returns (bytes memory, bool) { + (uint256 size, uint256 chunkNum) = _size(key); + if (chunkNum == 0) { + return (new bytes(0), false); + } + + bytes memory data = new bytes(size); // solidity should auto-align the memory-size to 32 + uint256 dataPtr; + assembly { + dataPtr := add(data, 0x20) + } + for (uint256 chunkId = 0; chunkId < chunkNum; chunkId++) { + bytes32 metadata = keyToMetadata[key][chunkId]; + + uint256 chunkSize = 0; + if (metadata.isInSlot()) { + chunkSize = metadata.decodeLen(); + SlotHelper.getRawAt( + keyToSlots[key][chunkId], + metadata, + dataPtr + ); + } else { + address addr = metadata.bytes32ToAddr(); + (chunkSize, ) = StorageHelperV2.sizeRaw(addr); + StorageHelperV2.getRawAt(addr, dataPtr); + } + + dataPtr += chunkSize; + } + + return (data, true); + } + + // Returns # of chunks deleted + function _remove(bytes32 key, uint256 chunkId) internal returns (uint256) { + while (true) { + bytes32 metadata = keyToMetadata[key][chunkId]; + if (metadata == bytes32(0x0)) { + break; + } + + if (!metadata.isInSlot()) { + address addr = metadata.bytes32ToAddr(); + // remove new contract + StorageSlotSelfDestructableV2(addr).destruct(); + } + + keyToMetadata[key][chunkId] = bytes32(0x0); + + chunkId++; + } + + return chunkId; + } + + function _removeChunk( + bytes32 key, + uint256 chunkId + ) internal returns (bool) { + bytes32 metadata = keyToMetadata[key][chunkId]; + if (metadata == bytes32(0x0)) { + return false; + } + + if (keyToMetadata[key][chunkId + 1] != bytes32(0x0)) { + // only the last chunk can be removed + return false; + } + + if (!metadata.isInSlot()) { + address addr = metadata.bytes32ToAddr(); + // remove new contract + StorageSlotSelfDestructableV2(addr).destruct(); + } + + keyToMetadata[key][chunkId] = bytes32(0x0); + + return true; + } +} diff --git a/contracts/v2/LargerStorageManagerV2Test.sol b/contracts/v2/LargerStorageManagerV2Test.sol new file mode 100644 index 0000000..e8af688 --- /dev/null +++ b/contracts/v2/LargerStorageManagerV2Test.sol @@ -0,0 +1,84 @@ +pragma solidity ^0.8.0; + +import "./LargeStorageManagerV2.sol"; + +contract LargeStorageManagerV2Test is LargeStorageManagerV2 { + constructor(uint8 slotLimit) LargeStorageManagerV2(slotLimit) {} + + function get(bytes32 key) public view returns (bytes memory, bool) { + (bytes memory data, bool found) = _get(key); + return (data, found); + } + + function getChunk( + bytes32 key, + uint256 chunkId + ) public view returns (bytes memory, bool) { + (bytes memory data, bool found) = _getChunk(key, chunkId); + return (data, found); + } + + function putChunk( + bytes32 key, + uint256 chunkId, + bytes memory data + ) public payable { + _putChunk(key, chunkId, data, msg.value); + } + + function putChunkFromCalldata( + bytes32 key, + uint256 chunkId, + bytes calldata data + ) public payable { + _putChunkFromCalldata(key, chunkId, data, msg.value); + } + + function size(bytes32 key) public view returns (uint256, uint256) { + return _size(key); + } + + function chunkSize( + bytes32 key, + uint256 chunkId + ) public view returns (uint256, bool) { + return _chunkSize(key, chunkId); + } + + function countChunks(bytes32 key) public view returns (uint256) { + return _countChunks(key); + } + + function remove(bytes32 key) public { + _remove(key, 0); + } + + function removeChunk(bytes32 key, uint256 chunkId) public { + _removeChunk(key, chunkId); + } + + function getBalance() public view returns (uint256 balance) { + return address(this).balance; + } + + function stakeTokens( + bytes32 key, + uint256 chunkId + ) public view returns (uint256) { + return _stakeTokens(key, chunkId); + } + + function chunkStakeTokens( + bytes32 key, + uint256 chunkId + ) public view returns (uint256, bool) { + return _chunkStakeTokens(key, chunkId); + } + + function getChunkAddr( + bytes32 key, + uint256 chunkId + ) public view returns (address) { + return _getChunkAddr(key, chunkId); + } +} diff --git a/contracts/v2/Memory.sol b/contracts/v2/Memory.sol new file mode 100644 index 0000000..ac6243b --- /dev/null +++ b/contracts/v2/Memory.sol @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: MIT + +pragma solidity ^0.8.0; + +library Memory { + // Size of a word, in bytes. + uint256 internal constant WORD_SIZE = 32; + // Size of the header of a 'bytes' array. + uint256 internal constant BYTES_HEADER_SIZE = 32; + // Address of the free memory pointer. + uint256 internal constant FREE_MEM_PTR = 0x40; + + // Compares the 'len' bytes starting at address 'addr' in memory with the 'len' + // bytes starting at 'addr2'. + // Returns 'true' if the bytes are the same, otherwise 'false'. + function equals( + uint256 addr, + uint256 addr2, + uint256 len + ) internal pure returns (bool equal) { + assembly { + equal := eq(keccak256(addr, len), keccak256(addr2, len)) + } + } + + // Compares the 'len' bytes starting at address 'addr' in memory with the bytes stored in + // 'bts'. It is allowed to set 'len' to a lower value then 'bts.length', in which case only + // the first 'len' bytes will be compared. + // Requires that 'bts.length >= len' + function equals( + uint256 addr, + uint256 len, + bytes memory bts + ) internal pure returns (bool equal) { + require(bts.length >= len); + uint256 addr2; + assembly { + addr2 := add( + bts, + /*BYTES_HEADER_SIZE*/ + 32 + ) + } + return equals(addr, addr2, len); + } + + // Allocates 'numBytes' bytes in memory. This will prevent the Solidity compiler + // from using this area of memory. It will also initialize the area by setting + // each byte to '0'. + function allocate(uint256 numBytes) internal pure returns (uint256 addr) { + // Take the current value of the free memory pointer, and update. + assembly { + addr := mload( + /*FREE_MEM_PTR*/ + 0x40 + ) + mstore( + /*FREE_MEM_PTR*/ + 0x40, + add(addr, numBytes) + ) + } + uint256 words = (numBytes + WORD_SIZE - 1) / WORD_SIZE; + for (uint256 i = 0; i < words; i++) { + assembly { + mstore( + add( + addr, + mul( + i, + /*WORD_SIZE*/ + 32 + ) + ), + 0 + ) + } + } + } + + // Copy 'len' bytes from memory address 'src', to address 'dest'. + // This function does not check the or destination, it only copies + // the bytes. + function copy(uint256 src, uint256 dest, uint256 len) internal pure { + // Copy word-length chunks while possible + // Reverse copy to prevent out of memory bound error + src = src + len; + dest = dest + len; + for (; len >= WORD_SIZE; len -= WORD_SIZE) { + dest -= WORD_SIZE; + src -= WORD_SIZE; + + assembly { + mstore(dest, mload(src)) + } + } + + if (len == 0) { + return; + } + + // Copy remaining bytes + src = src - len; + dest = dest - len; + assembly { + mstore(dest, mload(src)) + } + } + + // Returns a memory pointer to the provided bytes array. + function ptr(bytes memory bts) internal pure returns (uint256 addr) { + assembly { + addr := bts + } + } + + // Returns a memory pointer to the data portion of the provided bytes array. + function dataPtr(bytes memory bts) internal pure returns (uint256 addr) { + assembly { + addr := add( + bts, + /*BYTES_HEADER_SIZE*/ + 32 + ) + } + } + + // This function does the same as 'dataPtr(bytes memory)', but will also return the + // length of the provided bytes array. + function fromBytes( + bytes memory bts + ) internal pure returns (uint256 addr, uint256 len) { + len = bts.length; + assembly { + addr := add( + bts, + /*BYTES_HEADER_SIZE*/ + 32 + ) + } + } + + // Creates a 'bytes memory' variable from the memory address 'addr', with the + // length 'len'. The function will allocate new memory for the bytes array, and + // the 'len bytes starting at 'addr' will be copied into that new memory. + function toBytes( + uint256 addr, + uint256 len + ) internal pure returns (bytes memory bts) { + bts = new bytes(len); + uint256 btsptr; + assembly { + btsptr := add( + bts, + /*BYTES_HEADER_SIZE*/ + 32 + ) + } + copy(addr, btsptr, len); + } + + // Get the word stored at memory address 'addr' as a 'uint'. + function toUint(uint256 addr) internal pure returns (uint256 n) { + assembly { + n := mload(addr) + } + } + + // Get the word stored at memory address 'addr' as a 'bytes32'. + function toBytes32(uint256 addr) internal pure returns (bytes32 bts) { + assembly { + bts := mload(addr) + } + } + + /* + // Get the byte stored at memory address 'addr' as a 'byte'. + function toByte(uint addr, uint8 index) internal pure returns (byte b) { + require(index < WORD_SIZE); + uint8 n; + assembly { + n := byte(index, mload(addr)) + } + b = byte(n); + } + */ +} diff --git a/contracts/v2/StorageHelperV2.sol b/contracts/v2/StorageHelperV2.sol new file mode 100644 index 0000000..f638e7c --- /dev/null +++ b/contracts/v2/StorageHelperV2.sol @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +import "./Memory.sol"; +import "./StorageSlotFactory.sol"; + +library StorageHelperV2 { + // StorageSlotSelfDestructableV2 compiled via solc 0.8.7 optimized 200 + bytes internal constant STORAGE_SLOT_CODE_V2 = + hex"6080604052348015600f57600080fd5b506004361060285760003560e01c80632b68b9c614602d575b600080fd5b60336035565b005b336001600160a01b037f0000000000000000000000000000000000000000000000000000000000000000161460965760405162461bcd60e51b81526020600482015260036024820152624e464f60e81b604482015260640160405180910390fd5b7f00000000000000000000000000000000000000000000000000000000000000006001600160a01b0316fffea2646970667358221220154417754813d1989858c876ab2ded2ba1aa380679fff7a4c8faea076ba020e664736f6c63430008070033"; + uint256 internal constant OWNER_ADDR_OFF = 64; + uint256 internal constant USER_ADDR_OFF = 152; + + // StorageSlotFactoryFromInput compiled via solc 0.8.7 optimized 200 + STORAGE_SLOT_CODE + bytes internal constant FACTORY_CODE = + hex"60806040526040516101113803806101118339810160408190526100229161002b565b80518060208301f35b6000602080838503121561003e57600080fd5b82516001600160401b038082111561005557600080fd5b818501915085601f83011261006957600080fd5b81518181111561007b5761007b6100fa565b604051601f8201601f19908116603f011681019083821181831017156100a3576100a36100fa565b8160405282815288868487010111156100bb57600080fd5b600093505b828410156100dd57848401860151818501870152928501926100c0565b828411156100ee5760008684830101525b98975050505050505050565b634e487b7160e01b600052604160045260246000fdfe000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000006080604052348015600f57600080fd5b506004361060325760003560e01c80632b68b9c61460375780638da5cb5b14603f575b600080fd5b603d6081565b005b60657f000000000000000000000000000000000000000000000000000000000000000081565b6040516001600160a01b03909116815260200160405180910390f35b336001600160a01b037f0000000000000000000000000000000000000000000000000000000000000000161460ed5760405162461bcd60e51b815260206004820152600e60248201526d3737ba10333937b69037bbb732b960911b604482015260640160405180910390fd5b33fffea2646970667358221220fc66c9afb7cb2f6209ae28167cf26c6c06f86a82cbe3c56de99027979389a1be64736f6c63430008070033"; + uint256 internal constant FACTORY_SIZE_OFF = 305; + uint256 internal constant FACTORY_ADDR_OFF0 = 305 + 32 + OWNER_ADDR_OFF; + uint256 internal constant FACTORY_ADDR_OFF1 = 305 + 32 + USER_ADDR_OFF; + + function putRawFromCalldata( + bytes calldata data, + uint256 value + ) internal returns (address) { + bytes memory bytecode = bytes.concat(STORAGE_SLOT_CODE_V2, data); + address userAddr = msg.sender; + { + // revise the owner to the contract (so that it is destructable) + uint256 off = OWNER_ADDR_OFF + 0x20; + assembly { + mstore(add(bytecode, off), address()) + } + off = USER_ADDR_OFF + 0x20; + assembly { + mstore(add(bytecode, off), userAddr) + } + } + + StorageSlotFactoryFromInput c = new StorageSlotFactoryFromInput{ + value: value + }(bytecode); + return address(c); + } + + function putRaw( + bytes memory data, + uint256 value + ) internal returns (address) { + // create the new contract code with the data + bytes memory bytecode = STORAGE_SLOT_CODE_V2; + uint256 bytecodeLen = bytecode.length; + uint256 newSize = bytecode.length + data.length; + assembly { + // in-place resize of bytecode bytes + // note that this must be done when bytecode is the last allocated object by solidity. + mstore(bytecode, newSize) + // notify solidity about the memory size increase, must be 32-bytes aligned + mstore( + 0x40, + add(bytecode, and(add(add(newSize, 0x20), 0x1f), not(0x1f))) + ) + } + // append data to self-destruct byte code + Memory.copy( + Memory.dataPtr(data), + Memory.dataPtr(bytecode) + bytecodeLen, + data.length + ); + address userAddr = msg.sender; + { + // revise the owner to the contract (so that it is destructable) + uint256 off = OWNER_ADDR_OFF + 0x20; + assembly { + mstore(add(bytecode, off), address()) + } + off = USER_ADDR_OFF + 0x20; + assembly { + mstore(add(bytecode, off), userAddr) + } + } + + StorageSlotFactoryFromInput c = new StorageSlotFactoryFromInput{ + value: value + }(bytecode); + return address(c); + } + + function putRaw2( + bytes32 key, + bytes memory data, + uint256 value + ) internal returns (address) { + // create the new contract code with the data + bytes memory bytecode = FACTORY_CODE; + uint256 bytecodeLen = bytecode.length; + uint256 newSize = bytecode.length + data.length; + assembly { + // in-place resize of bytecode bytes + // note that this must be done when bytecode is the last allocated object by solidity. + mstore(bytecode, newSize) + // notify solidity about the memory size increase, must be 32-bytes aligned + mstore( + 0x40, + add(bytecode, and(add(add(newSize, 0x20), 0x1f), not(0x1f))) + ) + } + // append data to self-destruct byte code + Memory.copy( + Memory.dataPtr(data), + Memory.dataPtr(bytecode) + bytecodeLen, + data.length + ); + { + // revise the size of calldata + uint256 calldataSize = STORAGE_SLOT_CODE_V2.length + data.length; + uint256 off = FACTORY_SIZE_OFF + 0x20; + assembly { + mstore(add(bytecode, off), calldataSize) + } + } + { + // revise the owner to the contract (so that it is destructable) + uint256 off = FACTORY_ADDR_OFF0 + 0x20; + assembly { + mstore(add(bytecode, off), address()) + } + off = FACTORY_ADDR_OFF1 + 0x20; + assembly { + mstore(add(bytecode, off), address()) + } + } + + address addr; + assembly { + addr := create2( + value, + add(bytecode, 0x20), // data offset + mload(bytecode), // size + key + ) + + if iszero(extcodesize(addr)) { + revert(0, 0) + } + } + return addr; + } + + function sizeRaw(address addr) internal view returns (uint256, bool) { + if (addr == address(0x0)) { + return (0, false); + } + uint256 codeSize; + uint256 off = STORAGE_SLOT_CODE_V2.length; + assembly { + codeSize := extcodesize(addr) + } + if (codeSize < off) { + return (0, false); + } + + return (codeSize - off, true); + } + + function getRaw(address addr) internal view returns (bytes memory, bool) { + (uint256 dataSize, bool found) = sizeRaw(addr); + + if (!found) { + return (new bytes(0), false); + } + + // copy the data without the "code" + bytes memory data = new bytes(dataSize); + uint256 off = STORAGE_SLOT_CODE_V2.length; + assembly { + // retrieve data size + extcodecopy(addr, add(data, 0x20), off, dataSize) + } + return (data, true); + } + + function getRawAt( + address addr, + uint256 memoryPtr + ) internal view returns (uint256, bool) { + (uint256 dataSize, bool found) = sizeRaw(addr); + + if (!found) { + return (0, false); + } + + uint256 off = STORAGE_SLOT_CODE_V2.length; + assembly { + // retrieve data size + extcodecopy(addr, memoryPtr, off, dataSize) + } + return (dataSize, true); + } + + function returnBytesInplace(bytes memory content) internal pure { + // equal to return abi.encode(content) + uint256 size = content.length + 0x40; // pointer + size + size = (size + 0x20 + 0x1f) & ~uint256(0x1f); + assembly { + // (DATA CORRUPTION): the caller method must be "external returns (bytes)", cannot be public! + mstore(sub(content, 0x20), 0x20) + return(sub(content, 0x20), size) + } + } + + function calculateValueForData( + uint256 datalen, + uint256 chunkSize, + uint256 codeStakingPerChunk + ) internal pure returns (uint256) { + return + ((datalen + STORAGE_SLOT_CODE_V2.length - 1) / chunkSize) * + codeStakingPerChunk; + } + + function storageSlotCodeLength() internal pure returns (uint256) { + return STORAGE_SLOT_CODE_V2.length; + } +} diff --git a/contracts/v2/StorageSlotFactory.sol b/contracts/v2/StorageSlotFactory.sol new file mode 100644 index 0000000..b9e88be --- /dev/null +++ b/contracts/v2/StorageSlotFactory.sol @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +import "./Memory.sol"; + +// Create a storage slot by appending data to the end +contract StorageSlotFromContract { + constructor(address contractAddr, bytes memory data) payable { + uint256 codeSize; + assembly { + // retrieve the size of the code, this needs assembly + codeSize := extcodesize(contractAddr) + } + + uint256 totalSize = codeSize + data.length + 32; + bytes memory deployCode = new bytes(totalSize); + + // Copy contract code + assembly { + // actually retrieve the code, this needs assembly + extcodecopy(contractAddr, add(deployCode, 0x20), 0, codeSize) + } + + // Copy data + uint256 off = Memory.dataPtr(deployCode) + codeSize; + Memory.copy(Memory.dataPtr(data), off, data.length); + + off += data.length; + uint256 len = data.length; + // Set data size + assembly { + mstore(off, len) + } + + // Return the contract manually + assembly { + return(add(deployCode, 0x20), totalSize) + } + } +} + +// Create a storage slot +contract StorageSlotFactoryFromInput { + constructor(bytes memory codeAndData) payable { + uint256 size = codeAndData.length; + // Return the contract manually + assembly { + return(add(codeAndData, 0x20), size) + } + } +} diff --git a/contracts/v2/StorageSlotSelfDestructableV2.sol b/contracts/v2/StorageSlotSelfDestructableV2.sol new file mode 100644 index 0000000..ae6c5d3 --- /dev/null +++ b/contracts/v2/StorageSlotSelfDestructableV2.sol @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +contract StorageSlotSelfDestructableV2 { + address immutable owner; + address immutable userToRefund; + + constructor(address user) payable { + owner = msg.sender; + userToRefund = user; + } + + function destruct() public { + require(msg.sender == owner, "NFO"); + selfdestruct(payable(userToRefund)); + } +} + +contract StorageSlotSelfDestructableV2_DEBUG { + address public immutable owner; + address public immutable userToRefund; + + constructor(address user) payable { + owner = msg.sender; + userToRefund = user; + } + + function destruct() public { + require(msg.sender == owner, "NFO"); + selfdestruct(payable(userToRefund)); + } +} diff --git a/contracts/v2/optimize/SlotHelper.sol b/contracts/v2/optimize/SlotHelper.sol new file mode 100644 index 0000000..8abb86a --- /dev/null +++ b/contracts/v2/optimize/SlotHelper.sol @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +library SlotHelper { + uint256 internal constant SLOTDATA_RIGHT_SHIFT = 32; + uint256 internal constant LEN_OFFSET = 224; + uint256 internal constant FIRST_SLOT_DATA_SIZE = 28; + + function putRaw( + mapping(uint256 => bytes32) storage slots, + bytes memory datas + ) internal returns (bytes32 mdata) { + uint256 len = datas.length; + mdata = encodeMetadata(datas); + if (len > FIRST_SLOT_DATA_SIZE) { + bytes32 value; + uint256 ptr; + assembly { + ptr := add(datas, add(0x20, FIRST_SLOT_DATA_SIZE)) + } + for ( + uint256 i = 0; + i < (len - FIRST_SLOT_DATA_SIZE + 32 - 1) / 32; + i++ + ) { + assembly { + value := mload(ptr) + } + ptr = ptr + 32; + slots[i] = value; + } + } + } + + function encodeMetadata( + bytes memory data + ) internal pure returns (bytes32 medata) { + uint256 datLen = data.length; + uint256 value; + assembly { + value := mload(add(data, 0x20)) + } + + datLen = datLen << LEN_OFFSET; + value = value >> SLOTDATA_RIGHT_SHIFT; + + medata = bytes32(value | datLen); + } + + function decodeMetadata( + bytes32 mdata + ) internal pure returns (uint256 len, bytes32 data) { + len = decodeLen(mdata); + data = mdata << SLOTDATA_RIGHT_SHIFT; + } + + function decodeMetadataToData( + bytes32 mdata + ) internal pure returns (uint256 len, bytes memory data) { + len = decodeLen(mdata); + mdata = mdata << SLOTDATA_RIGHT_SHIFT; + data = new bytes(len); + assembly { + mstore(add(data, 0x20), mdata) + } + } + + function getRaw( + mapping(uint256 => bytes32) storage slots, + bytes32 mdata + ) internal view returns (bytes memory data) { + uint256 datalen; + (datalen, data) = decodeMetadataToData(mdata); + + if (datalen > FIRST_SLOT_DATA_SIZE) { + uint256 ptr = 0; + bytes32 value = 0; + assembly { + ptr := add(data, add(0x20, FIRST_SLOT_DATA_SIZE)) + } + for ( + uint256 i = 0; + i < (datalen - FIRST_SLOT_DATA_SIZE + 32 - 1) / 32; + i++ + ) { + value = slots[i]; + assembly { + mstore(ptr, value) + } + ptr = ptr + 32; + } + } + } + + function getRawAt( + mapping(uint256 => bytes32) storage slots, + bytes32 mdata, + uint256 memoryPtr + ) internal view returns (uint256 datalen, bool found) { + bytes32 datapart; + (datalen, datapart) = decodeMetadata(mdata); + + // memoryPtr:memoryPtr+32 is allocated for the data + uint256 dataPtr = memoryPtr; + assembly { + mstore(dataPtr, datapart) + } + + if (datalen > FIRST_SLOT_DATA_SIZE) { + uint256 ptr = 0; + bytes32 value = 0; + + assembly { + ptr := add(dataPtr, FIRST_SLOT_DATA_SIZE) + } + for ( + uint256 i = 0; + i < (datalen - FIRST_SLOT_DATA_SIZE + 32 - 1) / 32; + i++ + ) { + value = slots[i]; + assembly { + mstore(ptr, value) + } + ptr = ptr + 32; + } + } + + found = true; + } + + function isInSlot(bytes32 mdata) internal pure returns (bool succeed) { + return decodeLen(mdata) > 0; + } + + function encodeLen(uint256 datalen) internal pure returns (bytes32 res) { + res = bytes32(datalen << LEN_OFFSET); + } + + function decodeLen(bytes32 mdata) internal pure returns (uint256 res) { + res = uint256(mdata) >> LEN_OFFSET; + } + + function addrToBytes32(address addr) internal pure returns (bytes32) { + return bytes32(uint256(uint160(addr))); + } + + function bytes32ToAddr(bytes32 bt) internal pure returns (address) { + return address(uint160(uint256(bt))); + } +} diff --git a/contracts/v2/optimize/SlotHelperTest.sol b/contracts/v2/optimize/SlotHelperTest.sol new file mode 100644 index 0000000..f630bd0 --- /dev/null +++ b/contracts/v2/optimize/SlotHelperTest.sol @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +import "./SlotHelper.sol"; + +contract SlotHelperTest { + mapping(bytes32 => bytes32) public metadatas; + mapping(bytes32 => mapping(uint256 => bytes32)) public slots; + + function put(bytes32 key, bytes memory data) public { + metadatas[key] = SlotHelper.putRaw(slots[key], data); + } + + function get(bytes32 key) public view returns (bytes memory res) { + bytes32 md = metadatas[key]; + res = SlotHelper.getRaw(slots[key], md); + } + + function encodeMetadata(bytes memory data) public pure returns (bytes32) { + return SlotHelper.encodeMetadata(data); + } + + function decodeMetadata( + bytes32 mdata + ) public pure returns (uint256, bytes32) { + return SlotHelper.decodeMetadata(mdata); + } + + function decodeMetadata1( + bytes32 mdata + ) public pure returns (uint256, bytes memory) { + return SlotHelper.decodeMetadataToData(mdata); + } + + function encodeLen(uint256 datalen) public pure returns (bytes32) { + return SlotHelper.encodeLen(datalen); + } + + function decodeLen(bytes32 mdata) public pure returns (uint256 res) { + res = SlotHelper.decodeLen(mdata); + } + + function getLen(bytes32 key) public view returns (uint256 resLen) { + bytes32 mdata = metadatas[key]; + resLen = SlotHelper.decodeLen(mdata); + } +} diff --git a/package.json b/package.json index 03a8547..44ffc0f 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,6 @@ "@nomiclabs/hardhat-etherscan": "^3.1.3", "@openzeppelin/contracts": "^4.8.0", "chai": "^4.3.7", - "git3-evm-large-storage": "^1.1.0", "typechain": "^8.1.1" } } diff --git a/test/large-storage-helper-v2-test.js b/test/large-storage-helper-v2-test.js new file mode 100644 index 0000000..a2afce4 --- /dev/null +++ b/test/large-storage-helper-v2-test.js @@ -0,0 +1,138 @@ +const { web3 } = require("hardhat"); +const { expect } = require("chai"); +const { ethers } = require("hardhat"); +const { defaultAbiCoder } = require("ethers/lib/utils"); + +var ToBig = (x) => ethers.BigNumber.from(x); +const contractName = "LargeStorageManagerV2Test"; +let key = Buffer.from("a".repeat(32)); +let ETH = ethers.BigNumber.from("10").pow("18"); +describe("FlatDirectory Test", function () { + it("read/write", async function () { + const FlatDirectory = await ethers.getContractFactory(contractName); + const fd = await FlatDirectory.deploy(0); + await fd.deployed(); + + await fd.putChunk(key, 0, "0x112233"); + expect(await fd.get(key)).to.eql(["0x112233", true]); + + let data = Array.from({ length: 40 }, () => + Math.floor(Math.random() * 256) + ); + await fd.putChunk(key, 0, data); + expect(await fd.get(key)).to.eql([ethers.utils.hexlify(data), true]); + expect(await fd.size(key)).to.eql([ToBig(40), ToBig(1)]); + }); + + it("read/write chunks", async function () { + const FlatDirectory = await ethers.getContractFactory(contractName); + const fd = await FlatDirectory.deploy(0); + await fd.deployed(); + + let data0 = Array.from({ length: 1024 }, () => + Math.floor(Math.random() * 256) + ); + await fd.putChunk(key, 0, data0); + expect(await fd.get(key)).to.eql([ethers.utils.hexlify(data0), true]); + + let data1 = Array.from({ length: 512 }, () => + Math.floor(Math.random() * 256) + ); + await fd.putChunk(key, 1, data1); + expect(await fd.getChunk(key, 1)).to.eql([ + ethers.utils.hexlify(data1), + true, + ]); + + let data = data0.concat(data1); + expect(await fd.get(key)).to.eql([ethers.utils.hexlify(data), true]); + expect(await fd.size(key)).to.eql([ToBig(1536), ToBig(2)]); + }); + + it("write/remove chunks", async function () { + const FlatDirectory = await ethers.getContractFactory(contractName); + const fd = await FlatDirectory.deploy(0); + await fd.deployed(); + + expect(await fd.countChunks(key)).to.eql(ToBig(0)); + + let data0 = Array.from({ length: 10 }, () => + Math.floor(Math.random() * 256) + ); + await fd.putChunk(key, 0, data0); + expect(await fd.get(key)).to.eql([ethers.utils.hexlify(data0), true]); + + let data1 = Array.from({ length: 20 }, () => + Math.floor(Math.random() * 256) + ); + await fd.putChunk(key, 1, data1); + expect(await fd.getChunk(key, 1)).to.eql([ + ethers.utils.hexlify(data1), + true, + ]); + + await fd.removeChunk(key, 0); // should do nothing + expect(await fd.size(key)).to.eql([ToBig(30), ToBig(2)]); + expect(await fd.countChunks(key)).to.eql(ToBig(2)); + expect(await fd.getChunk(key, 0)).to.eql([ + ethers.utils.hexlify(data0), + true, + ]); + + await fd.removeChunk(key, 1); // should succeed + expect(await fd.size(key)).to.eql([ToBig(10), ToBig(1)]); + expect(await fd.get(key)).to.eql([ethers.utils.hexlify(data0), true]); + expect(await fd.getChunk(key, 1)).to.eql(["0x", false]); + expect(await fd.countChunks(key)).to.eql(ToBig(1)); + }); + + it("remove chunks and refund to user", async function () { + const FlatDirectory = await ethers.getContractFactory(contractName); + const fd = await FlatDirectory.deploy(0); + await fd.deployed(); + + let stakeTokenNum = ETH; + let signer; + [signer] = await ethers.getSigners(); + + let data0 = Array.from({ length: 10 }, () => + Math.floor(Math.random() * 256) + ); + await fd.putChunk(key, 0, data0, { value: stakeTokenNum }); + expect(await fd.get(key)).to.eql([ethers.utils.hexlify(data0), true]); + + let data1 = Array.from({ length: 20 }, () => + Math.floor(Math.random() * 256) + ); + await fd.putChunkFromCalldata(key, 1, data1, { value: stakeTokenNum }); + expect(await fd.getChunk(key, 1)).to.eql([ + ethers.utils.hexlify(data1), + true, + ]); + + // get stake tokens from files + let amount = await fd.stakeTokens(key, 0); + expect(amount).to.equal(stakeTokenNum.mul(2)); + + let amount1 = await fd.stakeTokens(key, 1); + expect(amount1).to.equal(stakeTokenNum); + + let chunk0Addr = await fd.getChunkAddr(key, 0); + let chunk0Balance = await ethers.provider.getBalance(chunk0Addr); + expect(chunk0Balance).to.equal(stakeTokenNum); + + // check the balance of user after removing chunk + // The tokens pledged by the chunk should all be returned to the user + let balBefore = await signer.getBalance(); + let tx1 = await fd.remove(key); // should succeed + let rec1 = await tx1.wait(); + amount1 = await fd.stakeTokens(key, 1); + expect(amount1).to.equal(ethers.BigNumber.from("0")); + let removeTxCost = rec1.gasUsed.mul(rec1.effectiveGasPrice); + let balAfter = await signer.getBalance(); + // check balance after refunding + expect( + balBefore.add(stakeTokenNum).add(stakeTokenNum).sub(removeTxCost) + ).to.eq(balAfter); + }); +}); diff --git a/test/slot-helper-test.js b/test/slot-helper-test.js new file mode 100644 index 0000000..e427680 --- /dev/null +++ b/test/slot-helper-test.js @@ -0,0 +1,80 @@ +const { web3, ethers } = require("hardhat"); +const { expect } = require("chai"); +const { Contract, BigNumber } = require("ethers"); + +describe("SlotHelper Library Test", function () { + let SlotHelperTest; + const SHITLEFT224BIT = BigNumber.from(1).mul(16).pow(56); + + beforeEach(async () => { + let factory = await ethers.getContractFactory("SlotHelperTest"); + SlotHelperTest = await factory.deploy(); + await SlotHelperTest.deployed(); + }); + + it("SlotHelper/encodeLen & decodeLen", async function () { + const len = 20; + let res = await SlotHelperTest.encodeLen(len); + let expectRes = BigNumber.from(len).mul(SHITLEFT224BIT).toHexString(); + + expect(BigNumber.from(res).eq(expectRes)).to.eq(true); + + let resLen = await SlotHelperTest.decodeLen(res); + expect(resLen.eq(BigNumber.from(len))).to.eq(true); + }); + + it("SlotHelper/encodeMetadata & decodeMetadata & decodeMetadata1", async function () { + const len = 20; + const data = []; + for (let i = 0; i < len; i++) { + data.push(1); + } + + //return mdata = "0x0000001401010101010101010101010101010101010101010000000000000000" + let mdata = await SlotHelperTest.encodeMetadata(data); + + //return [resLen1,resData1] = [20,"0x0101010101010101010101010101010101010101000000000000000000000000"] + let [resLen1, resData1] = await SlotHelperTest.decodeMetadata(mdata); + expect(resLen1.toNumber()).to.eq(len); + + //return [resLen2,resData2] = [20,"0x0101010101010101010101010101010101010101"] + let [resLen2, resData2] = await SlotHelperTest.decodeMetadata1(mdata); + expect(resLen2.toNumber()).to.eq(len); + expect(BigNumber.from(data).toHexString()).to.eq(resData2); + + // 20 + let resLen3 = await SlotHelperTest.decodeLen(mdata); + expect(resLen2.eq(BigNumber.from(len))).to.eq(true); + }); + + it("SlotHelper/ put & get", async function () { + let key = + "0x00000000000000000000000000000000000000000000000000000000000000aa"; + let data = "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + let datalen = 17; + + let tx1 = await SlotHelperTest.put(key, data); + await tx1.wait(); + let resData = await SlotHelperTest.get(key); + let resLen = await SlotHelperTest.getLen(key); + + expect(data).to.eq(resData); + expect(resLen.toNumber()).to.eq(datalen); + }); + + it("SlotHelper/ put & get over 28byte", async function () { + let key = + "0x00000000000000000000000000000000000000000000000000000000000000aa"; + let data = + "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + let datalen = 64; + + let tx1 = await SlotHelperTest.put(key, data); + await tx1.wait(); + let resData = await SlotHelperTest.get(key); + let resLen = await SlotHelperTest.getLen(key); + + expect(data).to.eq(resData); + expect(resLen.toNumber()).to.eq(datalen); + }); +});