Skip to content

Commit

Permalink
FEAT : Benchmark for integration tests (#477)
Browse files Browse the repository at this point in the history 
* Added Benchmark for integration tests

* Relocated time measuring code

* nit

* Minor design changes

* Using tabwriter
  • Loading branch information
@Sh0g0-1758
Sh0g0-1758 authored Jun 28, 2024
1 parent 036884a commit 6d939e4
Show file tree
Hide file tree
Showing 2 changed files with 171 additions and 15 deletions.
81 changes: 81 additions & 0 deletions integration_tests/BenchMarks.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
===========================================================================================================================
| File | PythonVM (ms) | GoVM (ms) |
===========================================================================================================================
| get_point_from_x.small.cairo | 971 | 125 |
---------------------------------------------------------------------------------------------------------------------------
| is_quad_residue.small.cairo | 963 | 152 |
---------------------------------------------------------------------------------------------------------------------------
| reduce_v1.small.cairo | 931 | 130 |
---------------------------------------------------------------------------------------------------------------------------
| uint256_unsigned_div_rem.small.cairo | 1061 | 132 |
---------------------------------------------------------------------------------------------------------------------------
| set_add.small.cairo | 749 | 117 |
---------------------------------------------------------------------------------------------------------------------------
| split64.small.cairo | 1014 | 117 |
---------------------------------------------------------------------------------------------------------------------------
| uint256_signedNN.small.cairo | 752 | 114 |
---------------------------------------------------------------------------------------------------------------------------
| split_felt.small.cairo | 841 | 145 |
---------------------------------------------------------------------------------------------------------------------------
| split_int.small.cairo | 1037 | 133 |
---------------------------------------------------------------------------------------------------------------------------
| uint256_add.small.cairo | 921 | 110 |
---------------------------------------------------------------------------------------------------------------------------
| signed_div_rem.small.cairo | 816 | 117 |
---------------------------------------------------------------------------------------------------------------------------
| sqrt.small.cairo | 960 | 119 |
---------------------------------------------------------------------------------------------------------------------------
| uint256_sqrt.small.cairo | 1195 | 115 |
---------------------------------------------------------------------------------------------------------------------------
| assert_not_zero.cairo | 586 | 100 |
---------------------------------------------------------------------------------------------------------------------------
| blake.starknet_with_keccak.cairo | 2498 | 144 |
---------------------------------------------------------------------------------------------------------------------------
| import_secp256R1P.small.cairo | 699 | 123 |
---------------------------------------------------------------------------------------------------------------------------
| memcpy.cairo | 704 | 124 |
---------------------------------------------------------------------------------------------------------------------------
| unsigned_div_rem.small.cairo | 843 | 118 |
---------------------------------------------------------------------------------------------------------------------------
| factorial.cairo | 1161 | 123 |
---------------------------------------------------------------------------------------------------------------------------
| hintrefs.cairo | 676 | 127 |
---------------------------------------------------------------------------------------------------------------------------
| cmp.small.cairo | 897 | 110 |
---------------------------------------------------------------------------------------------------------------------------
| dict_squash.small.cairo | 1240 | 166 |
---------------------------------------------------------------------------------------------------------------------------
| div_mod_n.small.cairo | 921 | 113 |
---------------------------------------------------------------------------------------------------------------------------
| find_element.small.cairo | 939 | 124 |
---------------------------------------------------------------------------------------------------------------------------
| uint256_mul_div_mod.small.cairo | 1032 | 119 |
---------------------------------------------------------------------------------------------------------------------------
| verify_zero.small.cairo | 781 | 112 |
---------------------------------------------------------------------------------------------------------------------------
| ec.small.cairo | 1042 | 129 |
---------------------------------------------------------------------------------------------------------------------------
| memset.cairo | 701 | 128 |
---------------------------------------------------------------------------------------------------------------------------
| pow.small.cairo | 887 | 112 |
---------------------------------------------------------------------------------------------------------------------------
| search_sorted_lower.small.cairo | 836 | 134 |
---------------------------------------------------------------------------------------------------------------------------
| alloc.cairo | 666 | 126 |
---------------------------------------------------------------------------------------------------------------------------
| simple.cairo | 596 | 110 |
---------------------------------------------------------------------------------------------------------------------------
| assert_250_bits.small.cairo | 979 | 120 |
---------------------------------------------------------------------------------------------------------------------------
| assert_not_equal.cairo | 638 | 101 |
---------------------------------------------------------------------------------------------------------------------------
| dict.cairo | 678 | 115 |
---------------------------------------------------------------------------------------------------------------------------
| is_positive.small.cairo | 884 | 128 |
---------------------------------------------------------------------------------------------------------------------------
| usort.small.cairo | 797 | 119 |
---------------------------------------------------------------------------------------------------------------------------
| verify_ecdsa_signature.small.cairo | 758 | 117 |
---------------------------------------------------------------------------------------------------------------------------
| fib.cairo | 634 | 116 |
===========================================================================================================================
105 changes: 90 additions & 15 deletions integration_tests/cairozero_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,11 @@ import (
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"testing"
"text/tabwriter"
"time"

"github.com/NethermindEth/cairo-vm-go/pkg/vm"
"github.com/consensys/gnark-crypto/ecc/stark-curve/fp"
Expand Down Expand Up @@ -69,6 +72,8 @@ func TestCairoZeroFiles(t *testing.T) {
filter := Filter{}
filter.init()

benchmarkMap := make(map[string][2]int)

for _, dirEntry := range testFiles {
if dirEntry.IsDir() || isGeneratedFile(dirEntry.Name()) {
continue
Expand All @@ -91,18 +96,20 @@ func TestCairoZeroFiles(t *testing.T) {
continue
}

pyTraceFile, pyMemoryFile, err := runPythonVm(dirEntry.Name(), compiledOutput)
elapsedPy, pyTraceFile, pyMemoryFile, err := runPythonVm(dirEntry.Name(), compiledOutput)
if err != nil {
t.Error(err)
continue
}

traceFile, memoryFile, _, err := runVm(compiledOutput)
elapsedGo, traceFile, memoryFile, _, err := runVm(compiledOutput)
if err != nil {
t.Error(err)
continue
}

benchmarkMap[dirEntry.Name()] = [2]int{int(elapsedPy.Milliseconds()), int(elapsedGo.Milliseconds())}

pyTrace, pyMemory, err := decodeProof(pyTraceFile, pyMemoryFile)
if err != nil {
t.Error(err)
Expand All @@ -125,10 +132,68 @@ func TestCairoZeroFiles(t *testing.T) {
}
}

WriteBenchMarksToFile(benchmarkMap)

clean(root1)
clean(root2)
}

// Save the Benchmarks for the integration tests in `BenchMarks.txt`
func WriteBenchMarksToFile(benchmarkMap map[string][2]int) {
totalWidth := 123

border := strings.Repeat("=", totalWidth)
separator := strings.Repeat("-", totalWidth)

var sb strings.Builder
w := tabwriter.NewWriter(&sb, 40, 0, 0, ' ', tabwriter.Debug)

sb.WriteString(border + "\n")
fmt.Fprintln(w, "| File \t PythonVM (ms) \t GoVM (ms) \t")
w.Flush()
sb.WriteString(border + "\n")

iterator := 0
totalFiles := len(benchmarkMap)

for key, values := range benchmarkMap {
row := "| " + key + "\t "

for iter, value := range values {
row = row + strconv.Itoa(value) + "\t"
if iter == 0 {
row = row + " "
}
}

fmt.Fprintln(w, row)
w.Flush()

if iterator < totalFiles-1 {
sb.WriteString(separator + "\n")
}

iterator++
}

sb.WriteString(border + "\n")

fileName := "BenchMarks.txt"
file, err := os.Create(fileName)
if err != nil {
fmt.Println("Error creating file: ", err)
return
}
defer file.Close()

_, err = file.WriteString(sb.String())
if err != nil {
fmt.Println("Error writing to file: ", err)
} else {
fmt.Println("Benchmarks successfully written to:", fileName)
}
}

const (
compiledSuffix = "_compiled.json"
pyTraceSuffix = "_py_trace"
Expand Down Expand Up @@ -166,7 +231,7 @@ func compileZeroCode(path string) (string, error) {

// given a path to a compiled cairo zero file, execute it using the
// python vm and returns the trace and memory files location
func runPythonVm(testFilename, path string) (string, string, error) {
func runPythonVm(testFilename, path string) (time.Duration, string, string, error) {
traceOutput := swapExtenstion(path, pyTraceSuffix)
memoryOutput := swapExtenstion(path, pyMemorySuffix)

Expand All @@ -191,19 +256,24 @@ func runPythonVm(testFilename, path string) (string, string, error) {

cmd := exec.Command("cairo-run", args...)

start := time.Now()

res, err := cmd.CombinedOutput()

elapsed := time.Since(start)

if err != nil {
return "", "", fmt.Errorf(
return 0, "", "", fmt.Errorf(
"cairo-run %s: %w\n%s", path, err, string(res),
)
}

return traceOutput, memoryOutput, nil
return elapsed, traceOutput, memoryOutput, nil
}

// given a path to a compiled cairo zero file, execute
// it using our vm
func runVm(path string) (string, string, string, error) {
func runVm(path string) (time.Duration, string, string, string, error) {
traceOutput := swapExtenstion(path, traceSuffix)
memoryOutput := swapExtenstion(path, memorySuffix)

Expand All @@ -230,14 +300,19 @@ func runVm(path string) (string, string, string, error) {
path,
)

start := time.Now()

res, err := cmd.CombinedOutput()

elapsed := time.Since(start)

if err != nil {
return "", "", string(res), fmt.Errorf(
return 0, "", "", string(res), fmt.Errorf(
"cairo-vm run %s: %w\n%s", path, err, string(res),
)
}

return traceOutput, memoryOutput, string(res), nil
return elapsed, traceOutput, memoryOutput, string(res), nil

}

Expand Down Expand Up @@ -316,7 +391,7 @@ func TestFailingRangeCheck(t *testing.T) {
compiledOutput, err := compileZeroCode("./builtin_tests/range_check.small.cairo")
require.NoError(t, err)

_, _, _, err = runVm(compiledOutput)
_, _, _, _, err = runVm(compiledOutput)
require.ErrorContains(t, err, "check write: 2**128 <")

clean("./builtin_tests/")
Expand All @@ -326,7 +401,7 @@ func TestBitwise(t *testing.T) {
compiledOutput, err := compileZeroCode("./builtin_tests/bitwise_builtin_test.starknet_with_keccak.cairo")
require.NoError(t, err)

_, _, _, err = runVm(compiledOutput)
_, _, _, _, err = runVm(compiledOutput)
require.NoError(t, err)

clean("./builtin_tests/")
Expand All @@ -336,7 +411,7 @@ func TestPedersen(t *testing.T) {
compiledOutput, err := compileZeroCode("./builtin_tests/pedersen_test.small.cairo")
require.NoError(t, err)

_, _, output, err := runVm(compiledOutput)
_, _, _, output, err := runVm(compiledOutput)
require.NoError(t, err)
require.Contains(t, output, "Program output:\n 2089986280348253421170679821480865132823066470938446095505822317253594081284")

Expand All @@ -347,7 +422,7 @@ func TestPoseidon(t *testing.T) {
compiledOutput, err := compileZeroCode("./builtin_tests/poseidon_test.starknet_with_keccak.cairo")
require.NoError(t, err)

_, _, output, err := runVm(compiledOutput)
_, _, _, output, err := runVm(compiledOutput)
require.NoError(t, err)
require.Contains(t, output, "Program output:\n 442682200349489646213731521593476982257703159825582578145778919623645026501\n 2233832504250924383748553933071188903279928981104663696710686541536735838182\n 2512222140811166287287541003826449032093371832913959128171347018667852712082\n")
require.Contains(t, output, "3016509350703874362933565866148509373957094754875411937434637891208784994231\n 3015199725895936530535660185611704199044060139852899280809302949374221328865\n 3062378460350040063467318871602229987911299744598148928378797834245039883769\n")
Expand All @@ -358,7 +433,7 @@ func TestECDSA(t *testing.T) {
compiledOutput, err := compileZeroCode("./builtin_tests/ecdsa_test.starknet_with_keccak.cairo")
require.NoError(t, err)

_, _, _, err = runVm(compiledOutput)
_, _, _, _, err = runVm(compiledOutput)
require.NoError(t, err)

clean("./builtin_tests/")
Expand All @@ -368,7 +443,7 @@ func TestEcOp(t *testing.T) {
compiledOutput, err := compileZeroCode("./builtin_tests/ecop.starknet_with_keccak.cairo")
require.NoError(t, err)

_, _, _, err = runVm(compiledOutput)
_, _, _, _, err = runVm(compiledOutput)
// todo(rodro): This test is failing due to the lack of hint processing. It should be address soon
require.Error(t, err)

Expand All @@ -379,7 +454,7 @@ func TestKeccak(t *testing.T) {
compiledOutput, err := compileZeroCode("./builtin_tests/keccak_test.starknet_with_keccak.cairo")
require.NoError(t, err)

_, _, output, err := runVm(compiledOutput)
_, _, _, output, err := runVm(compiledOutput)
require.NoError(t, err)
require.Contains(t, output, "Program output:\n 1304102964824333531548398680304964155037696012322029952943772\n 688749063493959345342507274897412933692859993314608487848187\n 986714560881445649520443980361539218531403996118322524237197\n 1184757872753521629808292433475729390634371625298664050186717\n 719230200744669084408849842242045083289669818920073250264351\n 1543031433416778513637578850638598357854418012971636697855068\n 63644822371671650271181212513090078620238279557402571802224\n 879446821229338092940381117330194802032344024906379963157761\n")

Expand Down

0 comments on commit 6d939e4

Please sign in to comment.