Chore: Add Benchmark scripts (#154)

* Polishing

* Add flags and windows friendly

* Add makefile and readme explanation

* Minor improvements

* Minor fix in readme

.gitignore

@@ -1,5 +1,6 @@
 # Project Specific
 bin/
+benchmarks
 cov.out
 cpu.out
 *.test

Makefile

@@ -3,8 +3,7 @@
 BINARY_DIR := bin
 BINARY_NAME := cairo-vm
 
-MEM_PROFILE := mem.out
-CPU_PROFILE := cpu.out
+TEST := "."
 
 default: help
 
@@ -32,8 +31,6 @@ build:
 clean:
 	@echo "Cleaning up..."
 	@rm -rf $(BINARY_DIR)
-	@rm -f $(MEM_PROFILE)
-	@rm	-f $(CPU_PROFILE)
 
 unit:
 	@echo "Running unit tests..."
@@ -55,4 +52,5 @@ testall:
 	@go test ./...
 
 bench:
-	@go test ./pkg/runners/zero -bench=. -cpuprofile $(CPU_PROFILE) -memprofile $(MEM_PROFILE)
+	@echo "Running benchmarks..."
+	@go run scripts/benchmark.go --pkg=${PKG_NAME} --test=${TEST}

README.md

@@ -22,7 +22,7 @@ Currently, it is only possible to use it by building it from source by following
 1. Clone the repo to your machine: `git clone https://github.com/NethermindEth/cairo-vm-go`.
 2. Install `Go` on your PC, instructions [here](https://go.dev/.doc/install).
 3. Execute on the root folder of the repo: `make build`.
-4. Make sure everything is running smoothly by executing: `make testall`.
+4. Make sure everything is running smoothly by executing: `make unit`.
 
 After completing these steps, you can find the compiled VM in `bin/cairo-vm`.
 
@@ -61,10 +61,11 @@ To learn about all the possible options the VM can be run with, execute the `run
 
 ### Testing
 
-We currently have defined two sets of tests:
+We currently have defined three sets of tests:
 
 * unit tests where we check the correct work of each component individually.
 * integration tests where we compare that the proof of execution of our VM is the same as the proof of execution of the Python VM.
+* benchmark tests to have a baseline performance indicator.
 
 Unit tests can be automatically run with:
 
@@ -90,12 +91,19 @@ To benchmark the project run:
 make bench
 ```
 
-This will calculate the millionth Fibonacci number and store the execution information in a _cpu.out_ and _mem.out_ files. To graphically display them please run this command with either of those files:
+This will run benchmarks for most of the project packages. It will create a _benchmark_ folder and a subfolder named after the current branch git information. Inside this subfolder, each package that was benchmarked will have a _cpu.out_, _mem.out_ and _stdout.text_. The first two files will hold profiling data regarding CPU and memory usage respectively, the last one will hold allocations/operations per second per benchmark test.
+
+To view profiling information with a web UI, run:
 
 ```bash
-go tool pprof -http=:8080 cpu.out
+go tool pprof -http=:8080 benchmarks/<subfolder>/<pkg>/cpu.out
 ```
 
+You may also be interested in benchmarking a specific package or a specific function, you can use the `PKG_NAME` and `TEST` flags for this.
+
+```bash
+make bench PKG_NAME="hintrunner" TEST="AllocSegment"
+```
 
 ### Useful Commands
 

scripts/benchmark.go

@@ -0,0 +1,108 @@
+//go:build ignore
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+func RunBenchmarks(pkgSubstr, testSubstr string) {
+	pkgPrefix := filepath.Join(".", "pkg")
+	benchmarkDirs := []string{
+		filepath.Join(pkgPrefix, "hintrunner"),
+		filepath.Join(pkgPrefix, "vm"),
+		filepath.Join(pkgPrefix, "runners", "zero"),
+	}
+
+	// create benchmark result folder
+	branchName, commitHash, date := getGitInfo()
+	dirName := fmt.Sprintf("%s %s %s", branchName, date, commitHash)
+	dirPath := filepath.Join("benchmarks", dirName)
+
+	err := os.MkdirAll(dirPath, 0755)
+	if err != nil {
+		log.Fatalf("creating parent benchmark folder: %s", err)
+	}
+
+	for _, pkgDir := range benchmarkDirs {
+		pkg := filepath.Base(pkgDir)
+		if !strings.Contains(pkg, pkgSubstr) {
+			continue
+		}
+
+		pkgDir, err := filepath.Abs(pkgDir)
+		if err != nil {
+			log.Fatalf("locating pkg %s", pkg)
+		}
+
+		benchPath := filepath.Join(dirPath, pkg)
+		err = os.MkdirAll(benchPath, 0755)
+		if err != nil {
+			log.Fatalf("creating benchmark subfolder for pkg %s: %s", pkg, err)
+		}
+
+		cpuPath := filepath.Join(benchPath, "cpu.out")
+		memPath := filepath.Join(benchPath, "mem.out")
+
+		cmd := exec.Command(
+			"go", "test", pkgDir, "-bench", testSubstr,
+			"-cpuprofile", cpuPath, "-memprofile", memPath, "-benchmem",
+		)
+		var stdOut bytes.Buffer
+		cmd.Stdout = &stdOut
+		cmd.Stderr = os.Stderr
+
+		err = cmd.Run()
+		if err != nil {
+			log.Fatalf("failed to benchmark %s: %s", pkg, err)
+		}
+
+		stdOutPath := filepath.Join(benchPath, "stdout.txt")
+		err = os.WriteFile(stdOutPath, stdOut.Bytes(), 0644)
+
+		fmt.Printf(" - %s ✔\n", pkg)
+	}
+	fmt.Printf("Done! Results stored in: \"%s\"\n", dirPath)
+}
+
+func getGitInfo() (string, string, string) {
+	branchBytes, err := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD").CombinedOutput()
+	if err != nil {
+		log.Fatalf("branch name parsing: %s\n%s", err, string(branchBytes))
+	}
+
+	commitHashBytes, err := exec.Command("git", "rev-parse", "HEAD").CombinedOutput()
+	if err != nil {
+		log.Fatalf("commit hash parsing: %s\n%s", err, string(commitHashBytes))
+	}
+
+	dateBytes, err := exec.Command("git", "log", "-1", "--format=%cd", "--date=short").CombinedOutput()
+	if err != nil {
+		log.Fatalf("date parsing: %s\n%s", err, string(dateBytes))
+	}
+
+	return strings.TrimSpace(string(branchBytes)),
+		strings.TrimSpace(string(commitHashBytes))[0:8],
+		strings.TrimSpace(string(dateBytes))
+}
+
+func main() {
+	var pkgName string
+	var testName string
+	flag.StringVar(
+		&pkgName, "pkg", "", "package names that contain this substring will be benchmarked",
+	)
+	flag.StringVar(
+		&testName, "test", ".", "test names that contain this substring will be benchmarked",
+	)
+	flag.Parse()
+
+	RunBenchmarks(pkgName, testName)
+}