From 8d7b6fb8c9cac4008624527e567213b8ef756dfa Mon Sep 17 00:00:00 2001 From: "Lucas @ StarkWare" <70894690+LucasLvy@users.noreply.github.com> Date: Thu, 21 Sep 2023 23:57:36 +0200 Subject: [PATCH] test(output): compare python output to rust output (#9) * test(output): compare python output to rust output * test(hints): add test where output is diff * izldf,sqmdq --- .github/workflows/ci.yml | 2 +- .gitignore | 1 + Cargo.lock | 23 +++ Cargo.toml | 1 + build/different_output.json | 269 +++++++++++++++++++++++++ build/hint.json | 172 ++++++++++++++-- build/test.zip | Bin 1108 -> 0 bytes tests/common.rs | 25 +++ tests/contracts/different_output.cairo | 12 ++ tests/contracts/hint.cairo | 14 +- tests/sharp.rs | 1 + tests/snos.rs | 39 +++- 12 files changed, 537 insertions(+), 22 deletions(-) create mode 100644 build/different_output.json delete mode 100644 build/test.zip create mode 100644 tests/contracts/different_output.cairo diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81a19275..1e12634c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,4 +28,4 @@ jobs: run: rustup show - uses: Swatinem/rust-cache@v2 - name: Run tests - run: cargo test + run: pip install cairo-lang; cargo test diff --git a/.gitignore b/.gitignore index ea8c4bf7..0b4d0f85 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /target +build/test.zip \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index dba91171..33e40959 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -536,6 +536,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "digest" version = "0.10.7" @@ -1478,6 +1484,16 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "pretty_assertions" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +dependencies = [ + "diff", + "yansi", +] + [[package]] name = "primitive-types" version = "0.12.1" @@ -2003,6 +2019,7 @@ dependencies = [ "log", "num-bigint", "num-traits", + "pretty_assertions", "reqwest", "rstest", "serde", @@ -2813,6 +2830,12 @@ dependencies = [ "tap", ] +[[package]] +name = "yansi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" + [[package]] name = "zeroize" version = "1.6.0" diff --git a/Cargo.toml b/Cargo.toml index d2f983d2..4fb1384c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ hex = "0.4.3" starknet = { git = "https://github.com/xJonathanLEI/starknet-rs" } uuid = { version = "1.4.0", features = ["v4", "serde"] } reqwest = { version = "0.11.18", features = ["blocking", "json"] } +pretty_assertions = "1.4.0" [dev-dependencies] rstest = "0.18.2" diff --git a/build/different_output.json b/build/different_output.json new file mode 100644 index 00000000..9d57569e --- /dev/null +++ b/build/different_output.json @@ -0,0 +1,269 @@ +{ + "attributes": [], + "builtins": [ + "output" + ], + "compiler_version": "0.12.2", + "data": [ + "0x480680017fff8000", + "0x12", + "0x480280007ffd7fff", + "0x482680017ffd8000", + "0x1", + "0x208b7fff7fff7ffe" + ], + "debug_info": { + "file_contents": {}, + "instruction_locations": { + "0": { + "accessible_scopes": [ + "__main__", + "__main__.main" + ], + "flow_tracking_data": { + "ap_tracking": { + "group": 0, + "offset": 0 + }, + "reference_ids": { + "__main__.main.output_ptr": 0 + } + }, + "hints": [], + "inst": { + "end_col": 19, + "end_line": 4, + "input_file": { + "filename": "tests/contracts/different_output.cairo" + }, + "start_col": 17, + "start_line": 4 + } + }, + "2": { + "accessible_scopes": [ + "__main__", + "__main__.main" + ], + "flow_tracking_data": { + "ap_tracking": { + "group": 0, + "offset": 1 + }, + "reference_ids": { + "__main__.main.a": 1, + "__main__.main.output_ptr": 0 + } + }, + "hints": [], + "inst": { + "end_col": 27, + "end_line": 5, + "input_file": { + "filename": "tests/contracts/different_output.cairo" + }, + "start_col": 5, + "start_line": 5 + } + }, + "3": { + "accessible_scopes": [ + "__main__", + "__main__.main" + ], + "flow_tracking_data": { + "ap_tracking": { + "group": 0, + "offset": 2 + }, + "reference_ids": { + "__main__.main.a": 1, + "__main__.main.output_ptr": 2 + } + }, + "hints": [ + { + "location": { + "end_col": 23, + "end_line": 8, + "input_file": { + "filename": "tests/contracts/different_output.cairo" + }, + "start_col": 5, + "start_line": 8 + }, + "n_prefix_newlines": 0 + } + ], + "inst": { + "end_col": 36, + "end_line": 9, + "input_file": { + "filename": "tests/contracts/different_output.cairo" + }, + "parent_location": [ + { + "end_col": 35, + "end_line": 11, + "input_file": { + "filename": "tests/contracts/different_output.cairo" + }, + "start_col": 25, + "start_line": 11 + }, + "While expanding the reference 'output_ptr' in:" + ], + "start_col": 22, + "start_line": 9 + } + }, + "5": { + "accessible_scopes": [ + "__main__", + "__main__.main" + ], + "flow_tracking_data": { + "ap_tracking": { + "group": 0, + "offset": 3 + }, + "reference_ids": { + "__main__.main.a": 1, + "__main__.main.output_ptr": 2 + } + }, + "hints": [], + "inst": { + "end_col": 37, + "end_line": 11, + "input_file": { + "filename": "tests/contracts/different_output.cairo" + }, + "start_col": 5, + "start_line": 11 + } + } + } + }, + "hints": { + "3": [ + { + "accessible_scopes": [ + "__main__", + "__main__.main" + ], + "code": "print(ids.a)", + "flow_tracking_data": { + "ap_tracking": { + "group": 0, + "offset": 2 + }, + "reference_ids": { + "__main__.main.a": 1, + "__main__.main.output_ptr": 0 + } + } + } + ] + }, + "identifiers": { + "__main__.main": { + "decorators": [], + "pc": 0, + "type": "function" + }, + "__main__.main.Args": { + "full_name": "__main__.main.Args", + "members": { + "output_ptr": { + "cairo_type": "felt*", + "offset": 0 + } + }, + "size": 1, + "type": "struct" + }, + "__main__.main.ImplicitArgs": { + "full_name": "__main__.main.ImplicitArgs", + "members": {}, + "size": 0, + "type": "struct" + }, + "__main__.main.Return": { + "cairo_type": "(output_ptr: felt*)", + "type": "type_definition" + }, + "__main__.main.SIZEOF_LOCALS": { + "type": "const", + "value": 0 + }, + "__main__.main.a": { + "cairo_type": "felt", + "full_name": "__main__.main.a", + "references": [ + { + "ap_tracking_data": { + "group": 0, + "offset": 1 + }, + "pc": 2, + "value": "[cast(ap + (-1), felt*)]" + } + ], + "type": "reference" + }, + "__main__.main.output_ptr": { + "cairo_type": "felt*", + "full_name": "__main__.main.output_ptr", + "references": [ + { + "ap_tracking_data": { + "group": 0, + "offset": 0 + }, + "pc": 0, + "value": "[cast(fp + (-3), felt**)]" + }, + { + "ap_tracking_data": { + "group": 0, + "offset": 2 + }, + "pc": 3, + "value": "cast([fp + (-3)] + 1, felt*)" + } + ], + "type": "reference" + } + }, + "main_scope": "__main__", + "prime": "0x800000000000011000000000000000000000000000000000000000000000001", + "reference_manager": { + "references": [ + { + "ap_tracking_data": { + "group": 0, + "offset": 0 + }, + "pc": 0, + "value": "[cast(fp + (-3), felt**)]" + }, + { + "ap_tracking_data": { + "group": 0, + "offset": 1 + }, + "pc": 2, + "value": "[cast(ap + (-1), felt*)]" + }, + { + "ap_tracking_data": { + "group": 0, + "offset": 2 + }, + "pc": 3, + "value": "cast([fp + (-3)] + 1, felt*)" + } + ] + } +} diff --git a/build/hint.json b/build/hint.json index 7bd8ba0b..342d0802 100644 --- a/build/hint.json +++ b/build/hint.json @@ -1,8 +1,15 @@ { "attributes": [], - "builtins": [], + "builtins": [ + "output" + ], "compiler_version": "0.12.2", "data": [ + "0x480680017fff8000", + "0x11", + "0x480280007ffd7fff", + "0x482680017ffd8000", + "0x1", "0x208b7fff7fff7ffe" ], "debug_info": { @@ -19,37 +26,127 @@ "offset": 0 }, "reference_ids": { - "__main__.main.a": 0 + "__main__.main.output_ptr": 0 + } + }, + "hints": [], + "inst": { + "end_col": 19, + "end_line": 4, + "input_file": { + "filename": "tests/contracts/hint.cairo" + }, + "start_col": 17, + "start_line": 4 + } + }, + "2": { + "accessible_scopes": [ + "__main__", + "__main__.main" + ], + "flow_tracking_data": { + "ap_tracking": { + "group": 0, + "offset": 1 + }, + "reference_ids": { + "__main__.main.a": 1, + "__main__.main.output_ptr": 0 + } + }, + "hints": [], + "inst": { + "end_col": 27, + "end_line": 5, + "input_file": { + "filename": "tests/contracts/hint.cairo" + }, + "start_col": 5, + "start_line": 5 + } + }, + "3": { + "accessible_scopes": [ + "__main__", + "__main__.main" + ], + "flow_tracking_data": { + "ap_tracking": { + "group": 0, + "offset": 2 + }, + "reference_ids": { + "__main__.main.a": 1, + "__main__.main.output_ptr": 2 } }, "hints": [ { "location": { "end_col": 23, - "end_line": 4, + "end_line": 8, "input_file": { "filename": "tests/contracts/hint.cairo" }, "start_col": 5, - "start_line": 4 + "start_line": 8 }, "n_prefix_newlines": 0 } ], "inst": { - "end_col": 15, - "end_line": 5, + "end_col": 36, + "end_line": 9, + "input_file": { + "filename": "tests/contracts/hint.cairo" + }, + "parent_location": [ + { + "end_col": 35, + "end_line": 11, + "input_file": { + "filename": "tests/contracts/hint.cairo" + }, + "start_col": 25, + "start_line": 11 + }, + "While expanding the reference 'output_ptr' in:" + ], + "start_col": 22, + "start_line": 9 + } + }, + "5": { + "accessible_scopes": [ + "__main__", + "__main__.main" + ], + "flow_tracking_data": { + "ap_tracking": { + "group": 0, + "offset": 3 + }, + "reference_ids": { + "__main__.main.a": 1, + "__main__.main.output_ptr": 2 + } + }, + "hints": [], + "inst": { + "end_col": 37, + "end_line": 11, "input_file": { "filename": "tests/contracts/hint.cairo" }, "start_col": 5, - "start_line": 5 + "start_line": 11 } } } }, "hints": { - "0": [ + "3": [ { "accessible_scopes": [ "__main__", @@ -59,10 +156,11 @@ "flow_tracking_data": { "ap_tracking": { "group": 0, - "offset": 0 + "offset": 2 }, "reference_ids": { - "__main__.main.a": 0 + "__main__.main.a": 1, + "__main__.main.output_ptr": 0 } } } @@ -76,8 +174,13 @@ }, "__main__.main.Args": { "full_name": "__main__.main.Args", - "members": {}, - "size": 0, + "members": { + "output_ptr": { + "cairo_type": "felt*", + "offset": 0 + } + }, + "size": 1, "type": "struct" }, "__main__.main.ImplicitArgs": { @@ -87,7 +190,7 @@ "type": "struct" }, "__main__.main.Return": { - "cairo_type": "()", + "cairo_type": "(output_ptr: felt*)", "type": "type_definition" }, "__main__.main.SIZEOF_LOCALS": { @@ -97,6 +200,21 @@ "__main__.main.a": { "cairo_type": "felt", "full_name": "__main__.main.a", + "references": [ + { + "ap_tracking_data": { + "group": 0, + "offset": 1 + }, + "pc": 2, + "value": "[cast(ap + (-1), felt*)]" + } + ], + "type": "reference" + }, + "__main__.main.output_ptr": { + "cairo_type": "felt*", + "full_name": "__main__.main.output_ptr", "references": [ { "ap_tracking_data": { @@ -104,7 +222,15 @@ "offset": 0 }, "pc": 0, - "value": "cast(17, felt)" + "value": "[cast(fp + (-3), felt**)]" + }, + { + "ap_tracking_data": { + "group": 0, + "offset": 2 + }, + "pc": 3, + "value": "cast([fp + (-3)] + 1, felt*)" } ], "type": "reference" @@ -120,7 +246,23 @@ "offset": 0 }, "pc": 0, - "value": "cast(17, felt)" + "value": "[cast(fp + (-3), felt**)]" + }, + { + "ap_tracking_data": { + "group": 0, + "offset": 1 + }, + "pc": 2, + "value": "[cast(ap + (-1), felt*)]" + }, + { + "ap_tracking_data": { + "group": 0, + "offset": 2 + }, + "pc": 3, + "value": "cast([fp + (-3)] + 1, felt*)" } ] } diff --git a/build/test.zip b/build/test.zip deleted file mode 100644 index a3b55fe07d836d77b4ffcb8cbafdc9c7b6be7acb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1108 zcmWIWW@Zs#U|`^2&`mWBcPJ|6WM^bxkYQqA;0222rj{h8B$g!VWfkY=1;xj1F5Ig__k5c?^wd$p%BcF7bOUs^8z`k~#kce~3DzYN>=%jnAN zY`J$<*9+IZUM;Qn^qthFch~-kT>r4CBK+M`o9pj>Y2EU?lk%&%G^>2>{d;?*m%qM# z|)^2^^*YPXuh3esrvoQsgjr#^cHSL)p7JTOp(CL{Ua% zzNzyzmxN6b*Vf3i22T=L%Ut>Gh@w)4P@+%I!K|FO``)r$Kk)0|x_EunjJFyqyKZ=> zNY4|V>?%>z!om^QAt;oXa%_fckr~rWo#f~&B?XSI)30;b#oOOL@_4o^$bbH0#>c$} zgL|Jdi~hU%i8}x##2VJmQlA10sRCvO1}<=j<>nVv>Lq37^-kI-c*uaqRn=00!#g0* z(8Y@_k$-jiMxCiGGXIWSwyKh&+NyWCvge)9VEev}ValKP(OfI{=;)PJ zu@@9Sur-Y<+q3g>HQV=;`_-9y9#yPwui>98eNZ*t+q@wA=C`jqcYnxR%f4Ci>9pY4 zlMmM>6>-LViDxT_{d~1TPBNy}es}+_!WsPsU4OE~OnHyy_G8j_%e8@S)(2ux8c0k@ z$t=mt&r8gShe!JAuv7ZFni{&D{#s|xXr4S1ylGXKmyV|9DIYCePhGZ8S*26Bgf1(d zxfR8L8ZfIxPfhd$8X5q^5K7U4A&-0A8 zU$6f;{j=Jhxsf>qjRcX|AbHIrXZs$#v!X%f!Z0F^q6sH1)v#V z@U{^|p=1nnt>|e2qLG2&ZKDw|Z{?=KQV6;}^!PyNdk55qEq2gNKo4|;3EsdQjVlV! hO+gPkgej+hrXYu8fHx}}NR|}{O@VIY1m*$;1_0pHeWm~a diff --git a/tests/common.rs b/tests/common.rs index 960d3bf1..f433370d 100644 --- a/tests/common.rs +++ b/tests/common.rs @@ -37,3 +37,28 @@ pub fn setup_pie() -> CairoPie { runner.get_cairo_pie(&vm).unwrap() } + +#[allow(unused)] +pub fn assert_python_and_rust_output_match(program: &str, mut vm: VirtualMachine) { + let mut rs_output = String::new(); + vm.write_output(&mut rs_output).unwrap(); + let rs_output = rs_output.split('\n').filter(|&x| !x.is_empty()); + let python_output = std::process::Command::new("cairo-run") + .arg("--layout=small") + .arg(format!("--program={program:}")) + .arg("--print_output") + .output() + .expect("failed to run python vm"); + let python_output = unsafe { std::str::from_utf8_unchecked(&python_output.stdout) }.to_string(); + let python_output = python_output + .split('\n') + .into_iter() + .skip_while(|&x| x != "Program output:") + .skip(1) + .filter(|&x| !x.trim().is_empty()) + .into_iter(); + for (i, (rs, py)) in rs_output.zip(python_output).enumerate() { + let py = py.to_string().trim().to_string(); + pretty_assertions::assert_eq!(*rs, py, "Output #{i:} is different"); + } +} diff --git a/tests/contracts/different_output.cairo b/tests/contracts/different_output.cairo new file mode 100644 index 00000000..2709e39b --- /dev/null +++ b/tests/contracts/different_output.cairo @@ -0,0 +1,12 @@ +%builtins output + +func main(output_ptr: felt*) -> (output_ptr: felt*) { + tempvar a = 18; + a = [output_ptr], ap++; + + // Use custom hint to print the value of a + %{ print(ids.a) %} + let output_ptr = output_ptr + 1; + + return(output_ptr = output_ptr); +} diff --git a/tests/contracts/hint.cairo b/tests/contracts/hint.cairo index 31b816d0..be24253f 100644 --- a/tests/contracts/hint.cairo +++ b/tests/contracts/hint.cairo @@ -1,6 +1,12 @@ -func main() { - let a = 17; +%builtins output + +func main(output_ptr: felt*) -> (output_ptr: felt*) { + tempvar a = 17; + a = [output_ptr], ap++; + // Use custom hint to print the value of a %{ print(ids.a) %} - return (); -} \ No newline at end of file + let output_ptr = output_ptr + 1; + + return(output_ptr = output_ptr); +} diff --git a/tests/sharp.rs b/tests/sharp.rs index 6c2a10e2..e8ba73d3 100644 --- a/tests/sharp.rs +++ b/tests/sharp.rs @@ -14,6 +14,7 @@ const TEST_CAIRO_JOB_ID: &str = "3a24bbca-ad75-49d5-8ced-12796c6c0738"; // } #[rstest] +#[ignore] fn sharp_client_status() { let sharp_client = SharpClient::default(); let submit_resp = sharp_client.get_status(TEST_CAIRO_JOB_ID).unwrap(); diff --git a/tests/snos.rs b/tests/snos.rs index 07d6eb4c..ace8a8ca 100644 --- a/tests/snos.rs +++ b/tests/snos.rs @@ -1,3 +1,4 @@ +mod common; use cairo_vm::cairo_run::{cairo_run, CairoRunConfig}; use cairo_vm::felt::Felt252; use cairo_vm::hint_processor::builtin_hint_processor::builtin_hint_processor_definition::{ @@ -8,6 +9,7 @@ use cairo_vm::hint_processor::hint_processor_definition::HintReference; use cairo_vm::serde::deserialize_program::ApTracking; use cairo_vm::types::exec_scope::ExecutionScopes; use cairo_vm::vm::{errors::hint_errors::HintError, vm_core::VirtualMachine}; +use common::assert_python_and_rust_output_match; use snos::SnOsRunner; use std::collections::HashMap; use std::fs::File; @@ -30,6 +32,7 @@ fn print_a_hint( #[test] fn custom_hint_ok() { + let program_path = "build/hint.json"; // Wrap the Rust hint implementation in a Box smart pointer inside a HintFunc let hint = HintFunc(Box::new(print_a_hint)); @@ -39,13 +42,13 @@ fn custom_hint_ok() { //Add the custom hint, together with the Python code hint_processor.add_hint(String::from("print(ids.a)"), Rc::new(hint)); - let file = File::open(Path::new("build/hint.json")).expect("Couldn't load file"); + let file = File::open(Path::new(program_path)).expect("Couldn't load file"); let mut reader = BufReader::new(file); let mut buffer = Vec::::new(); reader.read_to_end(&mut buffer).expect("Couldn't read file"); //Run the cairo program - cairo_run( + let (_cairo_runner, virtual_machine) = cairo_run( &buffer, &CairoRunConfig { layout: "all_cairo", @@ -54,6 +57,7 @@ fn custom_hint_ok() { &mut hint_processor, ) .expect("Couldn't run program"); + assert_python_and_rust_output_match(program_path, virtual_machine); } #[test] @@ -62,3 +66,34 @@ fn snos_ok() { let _runner_res = snos_runner.run(); assert_eq!(4, 4); } + +#[test] +#[should_panic(expected = "Output #0 is different")] +fn test_different_outputs() { + let program_path = "build/hint.json"; + // Wrap the Rust hint implementation in a Box smart pointer inside a HintFunc + let hint = HintFunc(Box::new(print_a_hint)); + + //Instantiate the hint processor + let mut hint_processor = BuiltinHintProcessor::new_empty(); + + //Add the custom hint, together with the Python code + hint_processor.add_hint(String::from("print(ids.a)"), Rc::new(hint)); + + let file = File::open(Path::new(program_path)).expect("Couldn't load file"); + let mut reader = BufReader::new(file); + let mut buffer = Vec::::new(); + reader.read_to_end(&mut buffer).expect("Couldn't read file"); + + //Run the cairo program + let (_cairo_runner, virtual_machine) = cairo_run( + &buffer, + &CairoRunConfig { + layout: "all_cairo", + ..Default::default() + }, + &mut hint_processor, + ) + .expect("Couldn't run program"); + assert_python_and_rust_output_match("build/different_output.json", virtual_machine); +}