Running tests multiple times leads to strange events
Fr33maan opened this issue · 3 comments
Fr33maan commented
Hello, I'm unfamiliar with solidity development and I wanted to write a serie of tests to see how things are working.
When running my tests several times, I sometimes have the following fail:
Note that the tests pass 90% of the time, and when they pass there is NOT the Token::permit method call in the trace.
I might not using things correctly because when tests are failing, I can see addresses which look like real ones instead of the 0x000...1 I'm creating in the tests.
Trying to get a repo
[FAIL. Reason: Assertion failed.]
[Sequence]
sender=0x0140824b993da43327d54c81fd32a2277ab47bac addr=[src/Token2.sol:Token]0x522b3294e6d06aa25ad0f1b8891242e335d3b459 calldata=permit(address,address,uint256,uint256,uint8,bytes32,bytes32), args=[0x229Eb494c4b68b7Cc314fB8a4e717F4aCE9CFDD8, 0xbeC391010395e74f6292E4e06d9B8605ac1DDc53, 115792089237316195423570985008687907853269984665640564039457584007913129639932, 115792089237316195423570985008687907853269984665640564039457584007913129639933, 1, 0x886bfd0cc58cd4daddab93500b19f4ee37e5c2d5f7b68ddd2da13db5c7c813fe, 0x712366c9249cf91dc662b9030cccdb66604b171143065657cac791d8f2298582]
sender=0x7890a3f43436d53a44602ba459a31acad0dfe620 addr=[src/Token2.sol:Token]0x522b3294e6d06aa25ad0f1b8891242e335d3b459 calldata=permit(address,address,uint256,uint256,uint8,bytes32,bytes32), args=[0x9ee17364098446F0674432d42A01229e81Fc3B71, 0x34cB980A981aBed97ae1e2E285Ca4CeDCFeB7f7c, 77047204794853382989177471990994, 5879125286370892607319398688471526795382463370954319295604, 255, 0x427b42ecde1d9c9a7aa5ac8cdbd18b951a439262ad36dc06617936fa8a79a7d1, 0x69e3816b8b02abc21ce6c97d754087f0a05f4cdc97bfe1bb74819fa17bbeb8a0]
sender=0x9ea182ee9a779debf29dcef52ff73050ab14589c addr=[src/Token2.sol:Token]0x522b3294e6d06aa25ad0f1b8891242e335d3b459 calldata=transfer(address,uint256):(bool), args=[0xADA1C96036Aa77c7a406893b646Ac52a955a185b, 67664557742138549244274836232634576278032314904979713654224389817491185308434]
sender=0xd7763e2f0e055589baf036651bdf5df94dd07856 addr=[src/Token2.sol:Token]0x522b3294e6d06aa25ad0f1b8891242e335d3b459 calldata=transfer(address,uint256):(bool), args=[0xFEC67FFA15D3F0d4Fe56373A2f4ba8D96B703225, 109689367063643566346524579496926362108891960872396993398579174959659490297229]
sender=0x859a484408f1100cf900b7c4e1ca4d80a7bcd7fe addr=[src/Token2.sol:Token]0x522b3294e6d06aa25ad0f1b8891242e335d3b459 calldata=permit(address,address,uint256,uint256,uint8,bytes32,bytes32), args=[0x2e302e313A38353435000000000000000000002A, 0x21b4928010Ac4bE76df80f100e6f31A369F8465F, 594227428863140510946470919175471612830314640842, 29602434937496747995246116920812564931095749185482214476498398010353477746710, 234, 0x000000000000000000000000a19fe569e5b824428c8813b619b8ac8b68bb7540, 0x2c008fd5f3120682281f895e9572f2fddc156af335133c41e1725e4a0286551b]
sender=0x0000000000000000000000000000000000000003 addr=[src/Token2.sol:Token]0x522b3294e6d06aa25ad0f1b8891242e335d3b459 calldata=transfer(address,uint256):(bool), args=[0x00000000000024A72B20A624A22Fa9a4a3A722a9, 27]
invariantMetadata() (runs: 82, calls: 1221, reverts: 871)
With the following trace:
Traces:
[764043] TokenTest::setUp()
├─ [0] VM::prank(0x0000000000000000000000000000000000000001)
│ └─ ← ()
├─ [628720] → new Token@0x522B3294E6d06aA25Ad0f1B8891242E335D3B459
│ ├─ emit Transfer(from: 0x0000000000000000000000000000000000000000, to: 0x0000000000000000000000000000000000000001, amount: 1000000)
│ └─ ← 2680 bytes of code
├─ [0] VM::prank(0x0000000000000000000000000000000000000001)
│ └─ ← ()
├─ [24874] Token::transfer(0x0000000000000000000000000000000000000002, 100)
│ ├─ emit Transfer(from: 0x0000000000000000000000000000000000000001, to: 0x0000000000000000000000000000000000000002, amount: 100)
│ └─ ← true
├─ [0] VM::prank(0x0000000000000000000000000000000000000001)
│ └─ ← ()
├─ [24874] Token::transfer(0x0000000000000000000000000000000000000003, 100)
│ ├─ emit Transfer(from: 0x0000000000000000000000000000000000000001, to: 0x0000000000000000000000000000000000000003, amount: 100)
│ └─ ← true
├─ [0] VM::prank(0x0000000000000000000000000000000000000001)
│ └─ ← ()
├─ [24874] Token::transfer(0x0000000000000000000000000000000000000004, 100)
│ ├─ emit Transfer(from: 0x0000000000000000000000000000000000000001, to: 0x0000000000000000000000000000000000000004, amount: 100)
│ └─ ← true
└─ ← ()
[26879] Token::permit(0x229Eb494c4b68b7Cc314fB8a4e717F4aCE9CFDD8, 0xbeC391010395e74f6292E4e06d9B8605ac1DDc53, 115792089237316195423570985008687907853269984665640564039457584007913129639932, 115792089237316195423570985008687907853269984665640564039457584007913129639933, 1, 0x886bfd0cc58cd4daddab93500b19f4ee37e5c2d5f7b68ddd2da13db5c7c813fe, 0x712366c9249cf91dc662b9030cccdb66604b171143065657cac791d8f2298582)
├─ [3000] PRECOMPILE::ecrecover(0xed3b009b26c8a4b046442dc013b4f93f702f1a115d5ae89f60641b509efe0ba0, 1, 61705346529405070123404304253690289214064479288778388738945695195492608644094, 51173900938108815564782920788572643345773495694317048238136663068501879653762) [staticcall]
│ └─ ←
└─ ← "INVALID_SIGNER"
[26879] Token::permit(0x9ee17364098446F0674432d42A01229e81Fc3B71, 0x34cB980A981aBed97ae1e2E285Ca4CeDCFeB7f7c, 77047204794853382989177471990994, 5879125286370892607319398688471526795382463370954319295604, 255, 0x427b42ecde1d9c9a7aa5ac8cdbd18b951a439262ad36dc06617936fa8a79a7d1, 0x69e3816b8b02abc21ce6c97d754087f0a05f4cdc97bfe1bb74819fa17bbeb8a0)
├─ [3000] PRECOMPILE::ecrecover(0xfcce608885e47a9efd59db0677e6d1d059e40b265cd8a382b440e737ff1dbd86, 255, 30070432096661134840415247957962109595239660820435638748811936579871477180369, 47894816609580200688941559983993220984307361383798774317424980769868930267296) [staticcall]
│ └─ ←
└─ ← "INVALID_SIGNER"
[2657] Token::transfer(0xADA1C96036Aa77c7a406893b646Ac52a955a185b, 67664557742138549244274836232634576278032314904979713654224389817491185308434)
└─ ← "Arithmetic over/underflow"
[2657] Token::transfer(0xFEC67FFA15D3F0d4Fe56373A2f4ba8D96B703225, 109689367063643566346524579496926362108891960872396993398579174959659490297229)
└─ ← "Arithmetic over/underflow"
[26879] Token::permit(0x2e302e313A38353435000000000000000000002A, 0x21b4928010Ac4bE76df80f100e6f31A369F8465F, 594227428863140510946470919175471612830314640842, 29602434937496747995246116920812564931095749185482214476498398010353477746710, 234, 0x000000000000000000000000a19fe569e5b824428c8813b619b8ac8b68bb7540, 0x2c008fd5f3120682281f895e9572f2fddc156af335133c41e1725e4a0286551b)
├─ [3000] PRECOMPILE::ecrecover(0x7163f5bd123fbeb19981d00fd172ec3d2bbc83b33acba161140c9f7d524d6e29, 234, 922713317337680761372074093008928705118000084288, 19902758055458254346736793183809061935887408011597054343345126583433939014939) [staticcall]
│ └─ ←
└─ ← "INVALID_SIGNER"
[29674] Token::transfer(0x00000000000024A72B20A624A22Fa9a4a3A722a9, 27)
├─ emit Transfer(from: 0x0000000000000000000000000000000000000003, to: 0x00000000000024A72B20A624A22Fa9a4a3A722a9, amount: 27)
└─ ← true
[PASS] testAllowanceToken() (gas: 34333)
Traces:
[34333] TokenTest::testAllowanceToken()
├─ [0] VM::prank(0x0000000000000000000000000000000000000002)
│ └─ ← ()
├─ [24523] Token::approve(0x0000000000000000000000000000000000000004, 2)
│ ├─ emit Approval(owner: 0x0000000000000000000000000000000000000002, spender: 0x0000000000000000000000000000000000000004, amount: 2)
│ └─ ← true
├─ [0] VM::prank(0x0000000000000000000000000000000000000004)
│ └─ ← ()
├─ [803] Token::allowance(0x0000000000000000000000000000000000000002, 0x0000000000000000000000000000000000000004) [staticcall]
│ └─ ← 2
└─ ← ()
[PASS] testTransferFromToken() (gas: 34918)
Traces:
[39131] TokenTest::testTransferFromToken()
├─ [0] VM::prank(0x0000000000000000000000000000000000000002)
│ └─ ← ()
├─ [24523] Token::approve(0x0000000000000000000000000000000000000004, 100)
│ ├─ emit Approval(owner: 0x0000000000000000000000000000000000000002, spender: 0x0000000000000000000000000000000000000004, amount: 100)
│ └─ ← true
├─ [0] VM::prank(0x0000000000000000000000000000000000000004)
│ └─ ← ()
├─ [10663] Token::transferFrom(0x0000000000000000000000000000000000000002, 0x0000000000000000000000000000000000000003, 100)
│ ├─ emit Transfer(from: 0x0000000000000000000000000000000000000002, to: 0x0000000000000000000000000000000000000003, amount: 100)
│ └─ ← true
├─ [519] Token::balanceOf(0x0000000000000000000000000000000000000002) [staticcall]
│ └─ ← 0
├─ [519] Token::balanceOf(0x0000000000000000000000000000000000000003) [staticcall]
│ └─ ← 200
└─ ← ()
[PASS] testTransferToken() (gas: 23080)
Traces:
[23080] TokenTest::testTransferToken()
├─ [0] VM::prank(0x0000000000000000000000000000000000000001)
│ └─ ← ()
├─ [12574] Token::transfer(0x0000000000000000000000000000000000000002, 100)
│ ├─ emit Transfer(from: 0x0000000000000000000000000000000000000001, to: 0x0000000000000000000000000000000000000002, amount: 100)
│ └─ ← true
├─ [519] Token::balanceOf(0x0000000000000000000000000000000000000001) [staticcall]
│ └─ ← 999600
├─ [519] Token::balanceOf(0x0000000000000000000000000000000000000002) [staticcall]
│ └─ ← 200
├─ [0] VM::prank(0x0000000000000000000000000000000000000002)
│ └─ ← ()
├─ [2380] Token::transfer(0x0000000000000000000000000000000000000001, 100)
│ ├─ emit Transfer(from: 0x0000000000000000000000000000000000000002, to: 0x0000000000000000000000000000000000000001, amount: 100)
│ └─ ← true
├─ [519] Token::balanceOf(0x0000000000000000000000000000000000000001) [staticcall]
│ └─ ← 999700
├─ [519] Token::balanceOf(0x0000000000000000000000000000000000000002) [staticcall]
│ └─ ← 100
└─ ← ()
Test result: FAILED. 3 passed; 1 failed; finished in 119.04ms
Fr33maan commented
Am I doing something fancy in the tests ?
// SPDX-License-Identifier: UNLICENSED
pragma solidity ^0.8.17;
import "forge-std/Test.sol";
import {Token} from "../src/Token2.sol";
contract TokenTest is Test {
using stdStorage for StdStorage;
Token private token;
address internal constant jb = address(1);
address internal constant dany = address(2);
address internal constant sara = address(3);
address internal constant lbp = address(4);
function setUp() public {
vm.prank(jb);
// Deploy NFT contract
token = new Token();
vm.prank(jb);
token.transfer(dany, 100);
vm.prank(jb);
token.transfer(sara, 100);
vm.prank(jb);
token.transfer(lbp, 100);
}
function invariantMetadata() public {
assertEq(token.name(), "My Token");
assertEq(token.symbol(), "TKN2");
assertEq(token.decimals(), 18);
assertEq(token.totalSupply(), 1_000_000);
assertEq(token.balanceOf(jb), 1_000_000 - 300);
assertEq(token.balanceOf(dany), 100);
assertEq(token.balanceOf(sara), 100);
assertEq(token.balanceOf(lbp), 100);
}
// Transfer money back and forth
function testTransferToken() public {
// First transfer from JB to Dany
vm.prank(jb);
token.transfer(dany, 100);
assertEq(token.balanceOf(jb), 1_000_000 - 400);
assertEq(token.balanceOf(dany), 200);
// Send back from Dany to JB
vm.prank(dany);
token.transfer(jb, 100);
assertEq(token.balanceOf(jb), 1_000_000 - 300);
assertEq(token.balanceOf(dany), 100);
}
// With allowance
function testTransferFromToken() public {
vm.prank(dany);
token.approve(lbp, 100);
vm.prank(lbp);
token.transferFrom(dany, sara, 100);
assertEq(token.balanceOf(dany), 0);
assertEq(token.balanceOf(sara), 200);
}
function testAllowanceToken() public {
vm.prank(dany);
token.approve(lbp, 2);
vm.prank(lbp);
assertEq(token.allowance(dany, lbp), 2);
}
}
Fr33maan commented
The problem does not seem to be relative to solmate, using openzeppelin contracts leads to the same issue.
Likely to be a problem with forge or anything foundry uses to run the tests.
Fr33maan commented
Solved, calling balanceOf in invariant is a bad idea