Skip to content

Commit c4ae688

Browse files
grandizzygrandizzy
authored
feat(invariant): generate failed call sequence as solidity (#9827)
* feat(invariant): generate failed call sequence as solidity * Fix test, format * Tests nits
1 parent 867484f commit c4ae688

File tree

8 files changed

+148
-18
lines changed

8 files changed

+148
-18
lines changed

‎crates/config/src/invariant.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ pub struct InvariantConfig {
3232
pub show_metrics: bool,
3333
/// Optional timeout (in seconds) for each invariant test.
3434
pub timeout: Option<u32>,
35+
/// Display counterexample as solidity calls.
36+
pub show_solidity: bool,
3537
}
3638

3739
impl Default for InvariantConfig {
@@ -48,6 +50,7 @@ impl Default for InvariantConfig {
4850
failure_persist_dir: None,
4951
show_metrics: false,
5052
timeout: None,
53+
show_solidity: false,
5154
}
5255
}
5356
}
@@ -67,6 +70,7 @@ impl InvariantConfig {
6770
failure_persist_dir: Some(cache_dir),
6871
show_metrics: false,
6972
timeout: None,
73+
show_solidity: false,
7074
}
7175
}
7276

‎crates/evm/evm/src/executors/invariant/replay.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ pub fn replay_run(
3232
coverage: &mut Option<HitMaps>,
3333
deprecated_cheatcodes: &mut HashMap<&'static str, Option<&'static str>>,
3434
inputs: &[BasicTxDetails],
35+
show_solidity: bool,
3536
) -> Result<Vec<BaseCounterExample>> {
3637
// We want traces for a failed case.
3738
if executor.inspector().tracer.is_none() {
@@ -64,6 +65,7 @@ pub fn replay_run(
6465
&tx.call_details.calldata,
6566
&ided_contracts,
6667
call_result.traces,
68+
show_solidity,
6769
));
6870
}
6971

@@ -110,6 +112,7 @@ pub fn replay_error(
110112
coverage: &mut Option<HitMaps>,
111113
deprecated_cheatcodes: &mut HashMap<&'static str, Option<&'static str>>,
112114
progress: Option<&ProgressBar>,
115+
show_solidity: bool,
113116
) -> Result<Vec<BaseCounterExample>> {
114117
match failed_case.test_error {
115118
// Don't use at the moment.
@@ -137,6 +140,7 @@ pub fn replay_error(
137140
coverage,
138141
deprecated_cheatcodes,
139142
&calls,
143+
show_solidity,
140144
)
141145
}
142146
}

‎crates/evm/fuzz/src/lib.rs‎

Lines changed: 47 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -41,21 +41,28 @@ pub enum CounterExample {
4141

4242
#[derive(Clone, Debug, Serialize, Deserialize)]
4343
pub struct BaseCounterExample {
44-
/// Address which makes the call
44+
/// Address which makes the call.
4545
pub sender: Option<Address>,
46-
/// Address to which to call to
46+
/// Address to which to call to.
4747
pub addr: Option<Address>,
48-
/// The data to provide
48+
/// The data to provide.
4949
pub calldata: Bytes,
50-
/// Contract name if it exists
50+
/// Contract name if it exists.
5151
pub contract_name: Option<String>,
52-
/// Function signature if it exists
52+
/// Function name if it exists.
53+
pub func_name: Option<String>,
54+
/// Function signature if it exists.
5355
pub signature: Option<String>,
54-
/// Args used to call the function
56+
/// Pretty formatted args used to call the function.
5557
pub args: Option<String>,
56-
/// Traces
58+
/// Unformatted args used to call the function.
59+
pub raw_args: Option<String>,
60+
/// Counter example traces.
5761
#[serde(skip)]
5862
pub traces: Option<SparsedTraceArena>,
63+
/// Whether to display sequence as solidity.
64+
#[serde(skip)]
65+
pub show_solidity: bool,
5966
}
6067

6168
impl BaseCounterExample {
@@ -66,6 +73,7 @@ impl BaseCounterExample {
6673
bytes: &Bytes,
6774
contracts: &ContractsByAddress,
6875
traces: Option<SparsedTraceArena>,
76+
show_solidity: bool,
6977
) -> Self {
7078
if let Some((name, abi)) = &contracts.get(&addr) {
7179
if let Some(func) = abi.functions().find(|f| f.selector() == bytes[..4]) {
@@ -76,11 +84,16 @@ impl BaseCounterExample {
7684
addr: Some(addr),
7785
calldata: bytes.clone(),
7886
contract_name: Some(name.clone()),
87+
func_name: Some(func.name.clone()),
7988
signature: Some(func.signature()),
8089
args: Some(
8190
foundry_common::fmt::format_tokens(&args).format(", ").to_string(),
8291
),
92+
raw_args: Some(
93+
foundry_common::fmt::format_tokens_raw(&args).format(", ").to_string(),
94+
),
8395
traces,
96+
show_solidity,
8497
};
8598
}
8699
}
@@ -91,9 +104,12 @@ impl BaseCounterExample {
91104
addr: Some(addr),
92105
calldata: bytes.clone(),
93106
contract_name: None,
107+
func_name: None,
94108
signature: None,
95109
args: None,
110+
raw_args: None,
96111
traces,
112+
show_solidity: false,
97113
}
98114
}
99115

@@ -108,17 +124,40 @@ impl BaseCounterExample {
108124
addr: None,
109125
calldata: bytes,
110126
contract_name: None,
127+
func_name: None,
111128
signature: None,
112129
args: Some(foundry_common::fmt::format_tokens(&args).format(", ").to_string()),
130+
raw_args: Some(foundry_common::fmt::format_tokens_raw(&args).format(", ").to_string()),
113131
traces,
132+
show_solidity: false,
114133
}
115134
}
116135
}
117136

118137
impl fmt::Display for BaseCounterExample {
119138
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
139+
// Display counterexample as solidity.
140+
if self.show_solidity {
141+
if let (Some(sender), Some(contract), Some(address), Some(func_name), Some(args)) =
142+
(&self.sender, &self.contract_name, &self.addr, &self.func_name, &self.raw_args)
143+
{
144+
writeln!(f, "\t\tvm.prank({sender});")?;
145+
write!(
146+
f,
147+
"\t\t{}({}).{}({});",
148+
contract.split_once(':').map_or(contract.as_str(), |(_, contract)| contract),
149+
address,
150+
func_name,
151+
args
152+
)?;
153+
154+
return Ok(())
155+
}
156+
}
157+
158+
// Regular counterexample display.
120159
if let Some(sender) = self.sender {
121-
write!(f, "sender={sender} addr=")?
160+
write!(f, "\t\tsender={sender} addr=")?
122161
}
123162

124163
if let Some(name) = &self.contract_name {

‎crates/forge/src/result.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -455,7 +455,7 @@ impl fmt::Display for TestResult {
455455
.as_str(),
456456
);
457457
for ex in sequence {
458-
writeln!(s, "\t\t{ex}").unwrap();
458+
writeln!(s, "{ex}").unwrap();
459459
}
460460
}
461461
}

‎crates/forge/src/runner.rs‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -581,20 +581,24 @@ impl<'a> FunctionRunner<'a> {
581581

582582
let failure_dir = invariant_config.clone().failure_dir(self.cr.name);
583583
let failure_file = failure_dir.join(&invariant_contract.invariant_function.name);
584+
let show_solidity = invariant_config.clone().show_solidity;
584585

585586
// Try to replay recorded failure if any.
586-
if let Ok(call_sequence) =
587+
if let Ok(mut call_sequence) =
587588
foundry_common::fs::read_json_file::<Vec<BaseCounterExample>>(failure_file.as_path())
588589
{
589590
// Create calls from failed sequence and check if invariant still broken.
590591
let txes = call_sequence
591-
.iter()
592-
.map(|seq| BasicTxDetails {
593-
sender: seq.sender.unwrap_or_default(),
594-
call_details: CallDetails {
595-
target: seq.addr.unwrap_or_default(),
596-
calldata: seq.calldata.clone(),
597-
},
592+
.iter_mut()
593+
.map(|seq| {
594+
seq.show_solidity = show_solidity;
595+
BasicTxDetails {
596+
sender: seq.sender.unwrap_or_default(),
597+
call_details: CallDetails {
598+
target: seq.addr.unwrap_or_default(),
599+
calldata: seq.calldata.clone(),
600+
},
601+
}
598602
})
599603
.collect::<Vec<BasicTxDetails>>();
600604
if let Ok((success, replayed_entirely)) = check_sequence(
@@ -624,6 +628,7 @@ impl<'a> FunctionRunner<'a> {
624628
&mut self.result.coverage,
625629
&mut self.result.deprecated_cheatcodes,
626630
&txes,
631+
show_solidity,
627632
);
628633
self.result.invariant_replay_fail(
629634
replayed_entirely,
@@ -674,6 +679,7 @@ impl<'a> FunctionRunner<'a> {
674679
&mut self.result.coverage,
675680
&mut self.result.deprecated_cheatcodes,
676681
progress.as_ref(),
682+
show_solidity,
677683
) {
678684
Ok(call_sequence) => {
679685
if !call_sequence.is_empty() {
@@ -719,6 +725,7 @@ impl<'a> FunctionRunner<'a> {
719725
&mut self.result.coverage,
720726
&mut self.result.deprecated_cheatcodes,
721727
&invariant_result.last_run_inputs,
728+
show_solidity,
722729
) {
723730
error!(%err, "Failed to replay last invariant run");
724731
}

‎crates/forge/tests/cli/config.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1090,6 +1090,7 @@ max_assume_rejects = 65536
10901090
gas_report_samples = 256
10911091
failure_persist_dir = "cache/invariant"
10921092
show_metrics = false
1093+
show_solidity = false
10931094
10941095
[labels]
10951096
@@ -1193,7 +1194,8 @@ exclude = []
11931194
"gas_report_samples": 256,
11941195
"failure_persist_dir": "cache/invariant",
11951196
"show_metrics": false,
1196-
"timeout": null
1197+
"timeout": null,
1198+
"show_solidity": false
11971199
},
11981200
"ffi": false,
11991201
"allow_internal_expect_revert": false,

‎crates/forge/tests/it/invariant.rs‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1066,6 +1066,7 @@ contract InvariantSelectorsWeightTest is Test {
10661066
});
10671067

10681068
// Tests original and new counterexample lengths are displayed on failure.
1069+
// Tests switch from regular sequence output to solidity.
10691070
forgetest_init!(invariant_sequence_len, |prj, cmd| {
10701071
prj.update_config(|config| {
10711072
config.fuzz.seed = Some(U256::from(100u32));
@@ -1099,4 +1100,76 @@ contract InvariantSequenceLenTest is Test {
10991100
[Sequence] (original: 4, shrunk: 1)
11001101
...
11011102
"#]]);
1103+
1104+
// Check regular sequence output. Shrink disabled to show several lines.
1105+
cmd.forge_fuse().arg("clean").assert_success();
1106+
prj.update_config(|config| {
1107+
config.invariant.shrink_run_limit = 0;
1108+
});
1109+
cmd.forge_fuse().args(["test", "--mt", "invariant_increment"]).assert_failure().stdout_eq(
1110+
str![[r#"
1111+
...
1112+
Failing tests:
1113+
Encountered 1 failing test in test/InvariantSequenceLenTest.t.sol:InvariantSequenceLenTest
1114+
[FAIL: revert: invariant increment failure]
1115+
[Sequence] (original: 4, shrunk: 4)
1116+
sender=0x00000000000000000000000000000000000018dE addr=[src/Counter.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=setNumber(uint256) args=[1931387396117645594923 [1.931e21]]
1117+
sender=0x00000000000000000000000000000000000009d5 addr=[src/Counter.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=increment() args=[]
1118+
sender=0x0000000000000000000000000000000000000105 addr=[src/Counter.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=increment() args=[]
1119+
sender=0x00000000000000000000000000000000000009B2 addr=[src/Counter.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=setNumber(uint256) args=[996881781832960761274744263729582347 [9.968e35]]
1120+
invariant_increment() (runs: 0, calls: 0, reverts: 0)
1121+
1122+
Encountered a total of 1 failing tests, 0 tests succeeded
1123+
1124+
"#]],
1125+
);
1126+
1127+
// Check solidity sequence output on same failure.
1128+
cmd.forge_fuse().arg("clean").assert_success();
1129+
prj.update_config(|config| {
1130+
config.invariant.show_solidity = true;
1131+
});
1132+
cmd.forge_fuse().args(["test", "--mt", "invariant_increment"]).assert_failure().stdout_eq(
1133+
str![[r#"
1134+
...
1135+
Failing tests:
1136+
Encountered 1 failing test in test/InvariantSequenceLenTest.t.sol:InvariantSequenceLenTest
1137+
[FAIL: revert: invariant increment failure]
1138+
[Sequence] (original: 4, shrunk: 4)
1139+
vm.prank(0x00000000000000000000000000000000000018dE);
1140+
Counter(0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f).setNumber(1931387396117645594923);
1141+
vm.prank(0x00000000000000000000000000000000000009d5);
1142+
Counter(0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f).increment();
1143+
vm.prank(0x0000000000000000000000000000000000000105);
1144+
Counter(0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f).increment();
1145+
vm.prank(0x00000000000000000000000000000000000009B2);
1146+
Counter(0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f).setNumber(996881781832960761274744263729582347);
1147+
invariant_increment() (runs: 0, calls: 0, reverts: 0)
1148+
1149+
Encountered a total of 1 failing tests, 0 tests succeeded
1150+
1151+
"#]],
1152+
);
1153+
1154+
// Persisted failures should be able to switch output.
1155+
prj.update_config(|config| {
1156+
config.invariant.show_solidity = false;
1157+
});
1158+
cmd.forge_fuse().args(["test", "--mt", "invariant_increment"]).assert_failure().stdout_eq(
1159+
str![[r#"
1160+
...
1161+
Failing tests:
1162+
Encountered 1 failing test in test/InvariantSequenceLenTest.t.sol:InvariantSequenceLenTest
1163+
[FAIL: invariant_increment replay failure]
1164+
[Sequence] (original: 4, shrunk: 4)
1165+
sender=0x00000000000000000000000000000000000018dE addr=[src/Counter.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=setNumber(uint256) args=[1931387396117645594923 [1.931e21]]
1166+
sender=0x00000000000000000000000000000000000009d5 addr=[src/Counter.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=increment() args=[]
1167+
sender=0x0000000000000000000000000000000000000105 addr=[src/Counter.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=increment() args=[]
1168+
sender=0x00000000000000000000000000000000000009B2 addr=[src/Counter.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=setNumber(uint256) args=[996881781832960761274744263729582347 [9.968e35]]
1169+
invariant_increment() (runs: 1, calls: 1, reverts: 1)
1170+
1171+
Encountered a total of 1 failing tests, 0 tests succeeded
1172+
1173+
"#]],
1174+
);
11021175
});

‎crates/forge/tests/it/test_helpers.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,7 @@ impl ForgeTestProfile {
149149
),
150150
show_metrics: false,
151151
timeout: None,
152+
show_solidity: false,
152153
};
153154

154155
config.sanitized()

0 commit comments

Comments
 (0)