From 298fa4df0592ee3685ddfcd93d48b76e64c7d081 Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Wed, 16 Oct 2024 16:49:07 -0400 Subject: [PATCH 1/6] Remove checking and parsing from suite, fix windows issues. --- compat/main.py | 68 ++++++++------------------------------------------ 1 file changed, 10 insertions(+), 58 deletions(-) diff --git a/compat/main.py b/compat/main.py index 1da0c95d21..fe6b697a32 100644 --- a/compat/main.py +++ b/compat/main.py @@ -4,7 +4,7 @@ import html import json import logging -import os +import os, stat import re import shlex import shutil @@ -76,17 +76,6 @@ def replace(pattern, replacement): with path.open(mode="w") as f: f.write(source) - @classmethod - def parse(cls, path: Path, use_json: bool, bench: bool) -> (bool, Optional): - return cls._run(PARSER_PATH, "Parsing", path, use_json, bench) - - def check(self, path: Path, use_json: bool, bench: bool) -> (bool, Optional): - extra_args = [ - f"-memberAccountAccess=S.{path}:S.{(path.parent / Path(other_path)).resolve()}" - for other_path in self.member_account_access - ] - return self._run(CHECKER_PATH, "Checking", path, use_json, bench, extra_args) - @staticmethod def _run( tool_path: Path, @@ -126,10 +115,11 @@ def run(self, working_dir: Path, prepare: bool, go_test_ref: Optional[str]) -> b if go_test_ref: replacement = f'github.com/onflow/cadence@{go_test_ref}' else: - replacement = shlex.quote(str(Path.cwd().parent.absolute())) + replacement = f'{Path.cwd().parent.absolute().resolve().as_posix()}' with cwd(working_dir / self.path): if prepare: + logger.info("Loading dependencies") subprocess.run([ "go", "mod", "edit", "-replace", f'github.com/onflow/cadence={replacement}', ]) @@ -200,6 +190,11 @@ def from_dict(cls, data: Dict): def _clone(self, working_dir: Path): if working_dir.exists(): + for root, dirs, files in os.walk(working_dir): + for dir in dirs: + os.chmod(os.path.join(root, dir), stat.S_IRWXU) + for file in files: + os.chmod(os.path.join(root, file), stat.S_IRWXU) shutil.rmtree(working_dir) logger.info(f"Cloning {self.url} ({self.branch})") @@ -223,57 +218,16 @@ def run( self._clone(working_dir) results: List[Result] = [] - check_succeeded = True - if check: - check_succeeded, results = self.check(working_dir, prepare=prepare, use_json=use_json, bench=bench) - go_tests_succeeded = True if go_test: for test in self.go_tests: if not test.run(working_dir, prepare=prepare, go_test_ref=go_test_ref): go_tests_succeeded = False - succeeded = check_succeeded and go_tests_succeeded + succeeded = go_tests_succeeded return succeeded, results - def check(self, working_dir: Path, prepare: bool, use_json: bool, bench: bool) -> (bool, List[Result]): - - run_succeeded = True - - results: List[Result] = [] - - for file in self.files: - path = working_dir.joinpath(file.path) - - if prepare: - file.rewrite(path) - - parse_succeeded, parse_results = \ - File.parse(path, use_json=use_json, bench=bench) - result = Result( - path=str(path), - parse_result=ParseResult.from_dict(parse_results[0]) if parse_results else None - ) - if not parse_succeeded: - run_succeeded = False - if use_json: - results.append(result) - continue - - check_succeeded, check_results = \ - file.check(path, use_json=use_json, bench=bench) - if check_results: - result.check_result = CheckResult.from_dict(check_results[0]) - if use_json: - results.append(result) - if not check_succeeded: - run_succeeded = False - continue - - return run_succeeded, results - - class Git: @staticmethod @@ -693,7 +647,7 @@ def build_all(): @click.option( "--go-test/--no-go-test", is_flag=True, - default=False, + default=True, help="Run the suite Go tests" ) @click.option( @@ -808,8 +762,6 @@ def run( names: Collection[str] ) -> (bool, List[Result]): - build_all() - all_succeeded = True if not names: From 8ce898631ecbf69d0d886f56cd3b65759d65c4f8 Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Thu, 17 Oct 2024 10:43:18 -0400 Subject: [PATCH 2/6] Update flow-go version to pass tests. --- compat/main.py | 50 ++++++++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/compat/main.py b/compat/main.py index fe6b697a32..75eebd3e7f 100644 --- a/compat/main.py +++ b/compat/main.py @@ -111,18 +111,17 @@ class GoTest: path: str command: str - def run(self, working_dir: Path, prepare: bool, go_test_ref: Optional[str]) -> bool: - if go_test_ref: - replacement = f'github.com/onflow/cadence@{go_test_ref}' - else: - replacement = f'{Path.cwd().parent.absolute().resolve().as_posix()}' + def run(self, working_dir: Path, prepare: bool, cadence_version: str, flowgo_version: str) -> bool: + cadence_replacement = f'github.com/onflow/cadence@{cadence_version}' + flowgo_replacement = f'github.com/onflow/flow-go@{flowgo_version}' with cwd(working_dir / self.path): if prepare: - logger.info("Loading dependencies") + logger.info("Editing dependencies") subprocess.run([ - "go", "mod", "edit", "-replace", f'github.com/onflow/cadence={replacement}', + "go", "mod", "edit", "-replace", f'github.com/onflow/flow-go={flowgo_replacement}', ]) + logger.info("Downloading dependencies") subprocess.run([ "go", "get", "-t", ".", ]) @@ -209,7 +208,8 @@ def run( bench: bool, check: bool, go_test: bool, - go_test_ref: Optional[str], + cadence_version: str, + flowgo_version: str, ) -> (bool, List[Result]): working_dir = SUITE_PATH / name @@ -221,7 +221,7 @@ def run( go_tests_succeeded = True if go_test: for test in self.go_tests: - if not test.run(working_dir, prepare=prepare, go_test_ref=go_test_ref): + if not test.run(working_dir, prepare=prepare, cadence_version=cadence_version, flowgo_version=flowgo_version): go_tests_succeeded = False succeeded = go_tests_succeeded @@ -613,12 +613,6 @@ def _category(self) -> BenchCategory: def json_serialize(data) -> str: return json.dumps(data, indent=4, cls=EnhancedJSONEncoder) - -def build_all(): - for name in ("parse", "check"): - build(name) - - @click.command() @click.option( "--rerun", @@ -651,9 +645,14 @@ def build_all(): help="Run the suite Go tests" ) @click.option( - "--go-test-ref", - default=None, - help="Git ref of Cadence for Go tests" + "--cadence-version", + default="v1.0.0", + help="version of Cadence for Go tests" +) +@click.option( + "--flowgo-version", + default="v0.37.16", + help="version of flow-go for Go tests" ) @click.option( "--output", @@ -682,7 +681,8 @@ def main( bench: bool, check: bool, go_test: bool, - go_test_ref: Optional[str], + cadence_version: str, + flowgo_version: str, output: Optional[LazyFile], other_ref: Optional[str], delta_threshold: float, @@ -707,7 +707,8 @@ def main( bench=bench, check=check, go_test=go_test, - go_test_ref=go_test_ref, + cadence_version=cadence_version, + flowgo_version=flowgo_version, names=names ) @@ -758,12 +759,16 @@ def run( bench: bool, check: bool, go_test: bool, - go_test_ref: Optional[str], + cadence_version: str, + flowgo_version: str, names: Collection[str] ) -> (bool, List[Result]): all_succeeded = True + # only flow-go version has an impact + logger.info(f'Chosen versions: cadence@{cadence_version}, flow-go@{flowgo_version}') + if not names: names = load_index(SUITE_PATH / "index.yaml") @@ -780,7 +785,8 @@ def run( bench=bench, check=check, go_test=go_test, - go_test_ref=go_test_ref, + cadence_version=cadence_version, + flowgo_version=flowgo_version, ) if not run_succeeded: From 314bca6cd478c05ce57f6526fcaaf5696afe192a Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Thu, 17 Oct 2024 10:54:56 -0400 Subject: [PATCH 3/6] Update to latest cadence version. --- compat/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compat/main.py b/compat/main.py index 75eebd3e7f..1a8dea80e9 100644 --- a/compat/main.py +++ b/compat/main.py @@ -646,12 +646,12 @@ def json_serialize(data) -> str: ) @click.option( "--cadence-version", - default="v1.0.0", + default="v1.0.1", help="version of Cadence for Go tests" ) @click.option( "--flowgo-version", - default="v0.37.16", + default="v0.37.17", help="version of flow-go for Go tests" ) @click.option( @@ -766,7 +766,7 @@ def run( all_succeeded = True - # only flow-go version has an impact + # only flow-go version has an impact, cadence version will be upgraded to match flow-go version logger.info(f'Chosen versions: cadence@{cadence_version}, flow-go@{flowgo_version}') if not names: From 8800bfe7216c643ca58d15033b8c8d3a22e990cf Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Thu, 17 Oct 2024 14:13:09 -0400 Subject: [PATCH 4/6] Fix formatting, add versioning back. --- compat/main.py | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/compat/main.py b/compat/main.py index 1a8dea80e9..a428a0af89 100644 --- a/compat/main.py +++ b/compat/main.py @@ -4,10 +4,11 @@ import html import json import logging -import os, stat +import os import re import shlex import shutil +import stat import subprocess import sys import textwrap @@ -111,9 +112,17 @@ class GoTest: path: str command: str - def run(self, working_dir: Path, prepare: bool, cadence_version: str, flowgo_version: str) -> bool: - cadence_replacement = f'github.com/onflow/cadence@{cadence_version}' - flowgo_replacement = f'github.com/onflow/flow-go@{flowgo_version}' + def run(self, working_dir: Path, prepare: bool, cadence_version: Optional[str], flowgo_version: Optional[str]) -> bool: + if cadence_version: + cadence_replacement = f'github.com/onflow/cadence@{cadence_version}' + else: + cadence_replacement = Path.cwd().parent.absolute().resolve().as_posix() + + if flowgo_version: + flowgo_replacement = f'github.com/onflow/flow-go@{flowgo_version}' + else: + # default flow-go version is 0.37.16 the one corresponding to cadence v1.0.0 + flowgo_replacement = f'github.com/onflow/flow-go@v0.37.16' with cwd(working_dir / self.path): if prepare: @@ -121,6 +130,9 @@ def run(self, working_dir: Path, prepare: bool, cadence_version: str, flowgo_ver subprocess.run([ "go", "mod", "edit", "-replace", f'github.com/onflow/flow-go={flowgo_replacement}', ]) + subprocess.run([ + "go", "mod", "edit", "-replace", f'github.com/onflow/cadence={cadence_replacement}', + ]) logger.info("Downloading dependencies") subprocess.run([ "go", "get", "-t", ".", @@ -191,9 +203,9 @@ def _clone(self, working_dir: Path): if working_dir.exists(): for root, dirs, files in os.walk(working_dir): for dir in dirs: - os.chmod(os.path.join(root, dir), stat.S_IRWXU) + os.chmod(os.path.join(root, dir), stat.S_IRUSR | stat.S_IWUSR) for file in files: - os.chmod(os.path.join(root, file), stat.S_IRWXU) + os.chmod(os.path.join(root, file), stat.S_IRUSR | stat.S_IWUSR) shutil.rmtree(working_dir) logger.info(f"Cloning {self.url} ({self.branch})") @@ -208,8 +220,8 @@ def run( bench: bool, check: bool, go_test: bool, - cadence_version: str, - flowgo_version: str, + cadence_version: Optional[str], + flowgo_version: Optional[str], ) -> (bool, List[Result]): working_dir = SUITE_PATH / name @@ -646,12 +658,12 @@ def json_serialize(data) -> str: ) @click.option( "--cadence-version", - default="v1.0.1", + default=None, help="version of Cadence for Go tests" ) @click.option( "--flowgo-version", - default="v0.37.17", + default=None, help="version of flow-go for Go tests" ) @click.option( @@ -766,8 +778,7 @@ def run( all_succeeded = True - # only flow-go version has an impact, cadence version will be upgraded to match flow-go version - logger.info(f'Chosen versions: cadence@{cadence_version}, flow-go@{flowgo_version}') + logger.info(f'Chosen versions: cadence@{ cadence_version if cadence_version else "local version" }, flow-go@{flowgo_version if flowgo_version else "v0.37.16"}') if not names: names = load_index(SUITE_PATH / "index.yaml") From 4f3e6fa5b74dfe5a63723928dc9b0f72125c5453 Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Fri, 18 Oct 2024 10:21:59 -0400 Subject: [PATCH 5/6] Remove unused code, always use latest flow-go version. --- compat/main.py | 602 +--------------------------------------- compat/requirements.txt | 1 - 2 files changed, 13 insertions(+), 590 deletions(-) diff --git a/compat/main.py b/compat/main.py index a428a0af89..9a973e9a80 100644 --- a/compat/main.py +++ b/compat/main.py @@ -1,44 +1,22 @@ from __future__ import annotations -import dataclasses -import html -import json import logging import os -import re import shlex import shutil import stat import subprocess -import sys -import textwrap -from collections import defaultdict from contextlib import contextmanager from dataclasses import dataclass, field -from enum import Enum, unique from pathlib import Path -from typing import List, Optional, Collection, Dict, IO, Any -from typing_extensions import Protocol, Literal +from typing import List, Optional, Collection, Dict import click as click import coloredlogs -import tabulate import yaml -from click.utils import LazyFile from dacite import from_dict SUITE_PATH = Path("suite").resolve() -CMD_PATH = Path("../runtime/cmd").resolve() -PARSER_PATH = Path("parse").resolve() -CHECKER_PATH = Path("check").resolve() - -ansi_escape_pattern = re.compile(r'\x1b[^m]*m') - - -class Openable(Protocol): - def open(self) -> IO: - pass - @contextmanager def cwd(path): @@ -49,64 +27,6 @@ def cwd(path): finally: os.chdir(oldpwd) - -@dataclass -class File: - path: str - prepare: Optional[str] - member_account_access: List[str] = field(default_factory=list) - - def rewrite(self, path: Path): - if not isinstance(self.prepare, str): - return - - logger.info(f"Preparing {path}") - - source: str - with path.open(mode="r") as f: - source = f.read() - - def replace(pattern, replacement): - nonlocal source - source = re.sub(pattern, replacement, source) - - variables = {"replace": replace} - with cwd(str(path.parent.absolute())): - exec(self.prepare, variables) - - with path.open(mode="w") as f: - f.write(source) - - @staticmethod - def _run( - tool_path: Path, - verb: str, - path: Path, - use_json: bool, - bench: bool, - extra_args: List[str] = None - ) -> (bool, Optional): - logger.info(f"{verb} {path}") - json_args = ["-json"] if use_json else [] - bench_args = ["-bench"] if bench else [] - args = json_args + bench_args + (extra_args or []) - - completed_process = subprocess.run( - [tool_path, *args, path], - cwd=str(path.parent), - capture_output=use_json - ) - result = None - if use_json: - result = json.loads(completed_process.stdout) - - if completed_process.returncode != 0: - logger.error(f"{verb} failed: {path}") - return False, result - - return True, result - - @dataclass class GoTest: path: str @@ -116,19 +36,20 @@ def run(self, working_dir: Path, prepare: bool, cadence_version: Optional[str], if cadence_version: cadence_replacement = f'github.com/onflow/cadence@{cadence_version}' else: - cadence_replacement = Path.cwd().parent.absolute().resolve().as_posix() + # default: point to local cadence repo + cadence_replacement = shlex.quote(Path.cwd().parent.absolute().resolve().as_posix()) if flowgo_version: flowgo_replacement = f'github.com/onflow/flow-go@{flowgo_version}' else: - # default flow-go version is 0.37.16 the one corresponding to cadence v1.0.0 - flowgo_replacement = f'github.com/onflow/flow-go@v0.37.16' + # default: use the newest version of flow-go available + flowgo_replacement = f'github.com/onflow/flow-go@latest' with cwd(working_dir / self.path): if prepare: logger.info("Editing dependencies") subprocess.run([ - "go", "mod", "edit", "-replace", f'github.com/onflow/flow-go={flowgo_replacement}', + "go", "get", flowgo_replacement, ]) subprocess.run([ "go", "mod", "edit", "-replace", f'github.com/onflow/cadence={cadence_replacement}', @@ -141,40 +62,6 @@ def run(self, working_dir: Path, prepare: bool, cadence_version: Optional[str], result = subprocess.run(shlex.split(self.command)) return result.returncode == 0 - -@dataclass -class BenchmarkResult: - iterations: int - time: int - - -@dataclass -class ParseResult: - error: Optional[Any] = field(default=None) - bench: Optional[BenchmarkResult] = field(default=None) - - @classmethod - def from_dict(cls, data: Dict): - return from_dict(data_class=cls, data=data) - - -@dataclass -class CheckResult: - error: Optional[Any] = field(default=None) - bench: Optional[BenchmarkResult] = field(default=None) - - @classmethod - def from_dict(cls, data: Dict): - return from_dict(data_class=cls, data=data) - - -@dataclass -class Result: - path: str - parse_result: ParseResult - check_result: Optional[CheckResult] = field(default=None) - - def load_index(path: Path) -> List[str]: logger.info(f"Loading suite index from {path} ...") with path.open(mode="r") as f: @@ -185,7 +72,6 @@ class Description: description: str url: str branch: str - files: List[File] = field(default_factory=list) go_tests: List[GoTest] = field(default_factory=list) @staticmethod @@ -216,20 +102,16 @@ def run( self, name: str, prepare: bool, - use_json: bool, - bench: bool, - check: bool, go_test: bool, cadence_version: Optional[str], flowgo_version: Optional[str], - ) -> (bool, List[Result]): + ) -> (bool): working_dir = SUITE_PATH / name if prepare: self._clone(working_dir) - results: List[Result] = [] go_tests_succeeded = True if go_test: for test in self.go_tests: @@ -238,7 +120,7 @@ def run( succeeded = go_tests_succeeded - return succeeded, results + return succeeded class Git: @@ -266,365 +148,6 @@ def checkout(ref: str): raise Exception(f'failed to checkout ref {ref}') -def build(name): - logger.info(f"Building {name}") - subprocess.run(["go", "build", CMD_PATH / name]) - - -class EnhancedJSONEncoder(json.JSONEncoder): - def default(self, o): - if dataclasses.is_dataclass(o): - return dataclasses.asdict(o) - if isinstance(o, Enum): - return o.value - return super().default(o) - - -def indent(text: str) -> str: - return textwrap.indent(text, " ") - - -def format_markdown_details(details: str) -> str: - stripped = ansi_escape_pattern.sub('', details) - escaped = html.escape(stripped).replace("\n", "
") - return f"
Details
{escaped}
" - - -@dataclass -class Comparisons: - parse_error_comparisons: Dict[str, ErrorComparison] - parse_bench_comparisons: Optional[Dict[str, BenchComparison]] - check_error_comparisons: Dict[str, ErrorComparison] - check_bench_comparisons: Optional[Dict[str, BenchComparison]] - - @classmethod - def from_results(cls, - other_results: List[Result], - current_results: List[Result], - bench: bool, - delta_threshold: float - ) -> Comparisons: - - parse_error_comparisons: Dict[str, ErrorComparison] = {} - check_error_comparisons: Dict[str, ErrorComparison] = {} - - parse_bench_comparisons: Optional[Dict[str, BenchComparison]] = None - check_bench_comparisons: Optional[Dict[str, BenchComparison]] = None - if bench: - parse_bench_comparisons: Dict[str, BenchComparison] = {} - check_bench_comparisons: Dict[str, BenchComparison] = {} - - other_results.sort(key=lambda result: result.path) - current_results.sort(key=lambda result: result.path) - - for other_result, current_result in zip(other_results, current_results): - path = other_result.path - assert current_result.path == path - - parse_error_comparisons[path] = ErrorComparison( - other_result.parse_result.error, - current_result.parse_result.error - ) - check_error_comparisons[path] = ErrorComparison( - other_result.check_result.error, - current_result.check_result.error - ) - - if bench: - if other_result.parse_result.bench and current_result.parse_result.bench: - parse_bench_comparisons[path] = BenchComparison( - other_result.parse_result.bench, - current_result.parse_result.bench, - delta_threshold=delta_threshold - ) - - if other_result.check_result.bench and current_result.check_result.bench: - check_bench_comparisons[path] = BenchComparison( - other_result.check_result.bench, - current_result.check_result.bench, - delta_threshold=delta_threshold - ) - - return Comparisons( - parse_error_comparisons, - parse_bench_comparisons, - check_error_comparisons, - check_bench_comparisons - ) - - def write(self, output: IO, format: Format): - - result: Optional[str] = None - if format in ("pretty", "markdown"): - - output.write("\n## Parser Errors\n\n") - self._write_error_comparisons(self.parse_error_comparisons, output, format) - if self.parse_bench_comparisons: - output.write("\n## Parser Benchmarks\n\n") - self._write_bench_comparisons(self.parse_bench_comparisons, output, format) - - output.write("\n## Checker Errors\n\n") - self._write_error_comparisons(self.check_error_comparisons, output, format) - if self.check_bench_comparisons: - output.write("\n## Checker Benchmarks\n\n") - self._write_bench_comparisons(self.check_bench_comparisons, output, format) - - if format == "json": - result = json_serialize(self) - - if result: - output.write(result) - - @staticmethod - def _write_error_comparisons(comparisons: Dict[str, ErrorComparison], output: IO, format: Format): - - def write_table(data, headers): - output.write(tabulate.tabulate(data, headers, tablefmt="pipe")) - output.write("\n\n") - - groups = defaultdict(list) - - for path, comparison in comparisons.items(): - relative_path = Path(path).relative_to(SUITE_PATH) - groups[comparison.category].append((relative_path, comparison)) - - for category in ( - ErrorCategory.REGRESSION, - ErrorCategory.STILL_FAIL, - ErrorCategory.CHANGE, - ErrorCategory.IMPROVEMENT, - ErrorCategory.STILL_SUCCESS, - ): - - category_comparisons = groups.get(category, []) - - if not len(category_comparisons): - continue - - if format == "pretty": - - if category == ErrorCategory.REGRESSION: - output.write("😭 Regressions\n") - output.write("-------------\n") - - for path, comparison in category_comparisons: - output.write(f"- {path}:\n") - output.write(indent(comparison.current)) - output.write("\n\n") - - elif category == ErrorCategory.STILL_FAIL: - output.write("😢 Still failing\n") - output.write("---------------\n") - - for path, comparison in category_comparisons: - output.write(f"- {path}:\n") - output.write(indent(comparison.current)) - output.write("\n\n") - - elif category == ErrorCategory.CHANGE: - output.write("😕 Changed\n") - output.write("---------\n") - - for path, comparison in category_comparisons: - output.write(f"- {path}:\n") - output.write(" Before:\n") - output.write(indent(comparison.other)) - output.write("\n") - output.write(" Now:\n") - output.write(indent(comparison.current)) - output.write("\n\n") - - elif category == ErrorCategory.IMPROVEMENT: - output.write("🎉 Improvements\n") - output.write("--------------\n") - - for path, comparison in category_comparisons: - output.write(f"- {path}\n") - - elif category == ErrorCategory.STILL_SUCCESS: - output.write("🙂 Still succeeding\n") - output.write("------------------\n") - - for path, comparison in category_comparisons: - output.write(f"- {path}\n") - - output.write("\n") - - elif format == "markdown": - - if category == ErrorCategory.REGRESSION: - data = [] - headers = ["😭 Regressions", "Details"] - for path, comparison in category_comparisons: - data.append([path, format_markdown_details(comparison.current)]) - - write_table(data, headers) - - elif category == ErrorCategory.STILL_FAIL: - data = [] - headers = ["😢 Still failing", "Details"] - for path, comparison in category_comparisons: - data.append([path, format_markdown_details(comparison.current)]) - - write_table(data, headers) - - elif category == ErrorCategory.CHANGE: - data = [] - headers = ["😕 Changed", "Before", "Now"] - for path, comparison in category_comparisons: - data.append([ - path, - format_markdown_details(comparison.other), - format_markdown_details(comparison.current) - ]) - - write_table(data, headers) - - elif category == ErrorCategory.IMPROVEMENT: - data = [] - headers = ["🎉 Improvements"] - for path, comparison in category_comparisons: - data.append([path]) - - write_table(data, headers) - - elif category == ErrorCategory.STILL_SUCCESS: - data = [] - headers = ["🙂 Still succeeding"] - for path, comparison in category_comparisons: - data.append([path]) - - write_table(data, headers) - - @classmethod - def _write_bench_comparisons(cls, comparisons: Dict[str, BenchComparison], output: IO, format: Format): - - def write_table(data, headers): - output.write(tabulate.tabulate(data, headers, tablefmt="pipe")) - output.write("\n\n") - - groups = defaultdict(list) - - for path, comparison in comparisons.items(): - relative_path = Path(path).relative_to(SUITE_PATH) - groups[comparison.category].append((relative_path, comparison)) - - for category in ( - BenchCategory.REGRESSION, - BenchCategory.IMPROVEMENT, - BenchCategory.SAME, - ): - - category_comparisons = groups.get(category, []) - - if not len(category_comparisons): - continue - - title = "" - if category == BenchCategory.REGRESSION: - title = "😭 Regressions" - elif category == BenchCategory.IMPROVEMENT: - title = "🎉 Improvements" - elif category == BenchCategory.SAME: - title = "🙂 Same" - - data = [] - headers = [title, "OLD", "NEW", "DELTA", "RATIO"] - for path, comparison in category_comparisons: - data.append([ - path, - cls._time_markdown(comparison.other.time), - cls._time_markdown(comparison.current.time), - cls._delta_markdown(comparison.delta), - cls._ratio_markdown(comparison.ratio) - ]) - - write_table(data, headers) - - @staticmethod - def _time_markdown(time: int) -> str: - return f'{time / 1000000:.2f}' - - @staticmethod - def _delta_markdown(delta: float) -> str: - result = f'{delta:.2f}%' - if result[0] != '-': - result = '+' + result - return result - - @staticmethod - def _ratio_markdown(ratio: float) -> str: - return f'**{ratio:.2f}x**' - - -@unique -class ErrorCategory(Enum): - REGRESSION = "Regression" - IMPROVEMENT = "Improvement" - CHANGE = "Change" - STILL_SUCCESS = "Still succeeding" - STILL_FAIL = "Still failing" - - -@dataclass -class ErrorComparison: - other: Optional[str] - current: Optional[str] - category: ErrorCategory = field(init=False) - - def __post_init__(self): - self.category = self._category() - - def _category(self) -> ErrorCategory: - if self.other != self.current: - return ErrorCategory.CHANGE - elif not self.other and self.current: - return ErrorCategory.REGRESSION - elif self.other and not self.current: - return ErrorCategory.IMPROVEMENT - elif self.other: - return ErrorCategory.STILL_FAIL - else: - return ErrorCategory.STILL_SUCCESS - - -@unique -class BenchCategory(Enum): - REGRESSION = "Regression" - IMPROVEMENT = "Improvement" - SAME = "Same" - - -@dataclass -class BenchComparison: - other: BenchmarkResult - current: BenchmarkResult - delta_threshold: float - ratio: float = field(init=False) - delta: float = field(init=False) - category: BenchCategory = field(init=False) - - def __post_init__(self): - self.ratio = self.other.time / self.current.time - self.delta = -(((self.current.time * 100) / self.other.time) - 100) - self.category = self._category() - - def _category(self) -> BenchCategory: - if abs(self.delta) > self.delta_threshold: - if self.delta > 0: - return BenchCategory.IMPROVEMENT - if self.delta < 0: - return BenchCategory.REGRESSION - - return BenchCategory.SAME - - -Format = Literal["pretty", "json", "markdown"] - - -def json_serialize(data) -> str: - return json.dumps(data, indent=4, cls=EnhancedJSONEncoder) - @click.command() @click.option( "--rerun", @@ -632,24 +155,6 @@ def json_serialize(data) -> str: default=False, help="Rerun without cloning and preparing the suites" ) -@click.option( - "--format", - type=click.Choice(["pretty", "json", "markdown"], case_sensitive=False), - default="pretty", - help="output format", -) -@click.option( - "--bench/--no-bench", - is_flag=True, - default=False, - help="Run benchmarks" -) -@click.option( - "--check/--no-check", - is_flag=True, - default=True, - help="Parse and check the suite files" -) @click.option( "--go-test/--no-go-test", is_flag=True, @@ -666,135 +171,56 @@ def json_serialize(data) -> str: default=None, help="version of flow-go for Go tests" ) -@click.option( - "--output", - default=None, - type=click.File("w"), - help="Write output to given path. Standard output by default" -) -@click.option( - "--compare-ref", - "other_ref", - help="Compare with another Git ref (e.g. commit or branch)" -) -@click.option( - "--delta-threshold", - default=4, - type=float, - help="Delta threshold to consider a benchmark result a change" -) @click.argument( "names", nargs=-1, ) def main( rerun: bool, - format: Format, - bench: bool, - check: bool, go_test: bool, cadence_version: str, flowgo_version: str, - output: Optional[LazyFile], - other_ref: Optional[str], - delta_threshold: float, names: Collection[str] ): - if other_ref is None and format not in ("pretty", "json"): - raise Exception(f"unsupported format: {format}") prepare = not rerun - output: IO = output.open() if output else sys.stdout - - # Comparison of different runs is only possible when using JSON - - use_json_for_run = format != "pretty" or (other_ref is not None) - # Run for the current checkout - current_success, current_results = run( + current_success = run( prepare=prepare, - use_json=use_json_for_run, - bench=bench, - check=check, go_test=go_test, cadence_version=cadence_version, flowgo_version=flowgo_version, names=names ) - # Run for the other checkout, if any - - if other_ref: - current_ref = Git.get_head_ref() - try: - Git.checkout(other_ref) - - _, other_results = run( - # suite repositories were already cloned in the previous run - prepare=False, - use_json=use_json_for_run, - bench=bench, - check=check, - go_test=go_test, - names=names - ) - - comparisons = Comparisons.from_results( - other_results, - current_results, - bench=bench, - delta_threshold=delta_threshold, - ) - - comparisons.write(output, format) - - finally: - Git.checkout(current_ref) - - else: - - if format == "json": - results_json = json_serialize(current_results) - output.write(results_json) - - output.close() - if not current_success: exit(1) def run( prepare: bool, - use_json: bool, - bench: bool, - check: bool, go_test: bool, cadence_version: str, flowgo_version: str, names: Collection[str] -) -> (bool, List[Result]): +) -> (bool): all_succeeded = True - logger.info(f'Chosen versions: cadence@{ cadence_version if cadence_version else "local version" }, flow-go@{flowgo_version if flowgo_version else "v0.37.16"}') + logger.info(f'Chosen versions: cadence@{ cadence_version if cadence_version else "local version" }, flow-go@{flowgo_version if flowgo_version else "latest"}') if not names: names = load_index(SUITE_PATH / "index.yaml") - all_results: List[Result] = [] - for name in names: description = Description.load(name) - run_succeeded, results = description.run( + run_succeeded = description.run( name, prepare=prepare, - use_json=use_json, - bench=bench, - check=check, go_test=go_test, cadence_version=cadence_version, flowgo_version=flowgo_version, @@ -803,9 +229,7 @@ def run( if not run_succeeded: all_succeeded = False - all_results.extend(results) - - return all_succeeded, all_results + return all_succeeded if __name__ == "__main__": diff --git a/compat/requirements.txt b/compat/requirements.txt index c83c2d82a8..83eccf7636 100644 --- a/compat/requirements.txt +++ b/compat/requirements.txt @@ -3,4 +3,3 @@ coloredlogs==14.0 dacite==1.5.1 PyYAML==5.4 typing-extensions==3.7.4.3 -tabulate==0.8.7 From 8805722f95b1b0f1feb0e2149ef0def3c7a45ea3 Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Fri, 18 Oct 2024 13:13:49 -0400 Subject: [PATCH 6/6] Remove unused files property. --- compat/suite/flow-core-contracts.yaml | 130 -------------------------- compat/suite/flow-ft.yaml | 79 ---------------- compat/suite/flow-nft.yaml | 54 ----------- compat/suite/nba-smart-contracts.yaml | 81 ---------------- 4 files changed, 344 deletions(-) diff --git a/compat/suite/flow-core-contracts.yaml b/compat/suite/flow-core-contracts.yaml index e7607d8573..2b3cdcaa10 100644 --- a/compat/suite/flow-core-contracts.yaml +++ b/compat/suite/flow-core-contracts.yaml @@ -7,133 +7,3 @@ branch: master go_tests: - path: lib/go/test command: make test -files: -- path: contracts/FlowToken.cdc - # language=Python prefix="replace: Callable[[str, str], None]\n" - prepare: | - from pathlib import Path - - replace( - 'import FungibleToken from 0xFUNGIBLETOKENADDRESS', - f'import FungibleToken from "{Path("../../flow-ft/contracts/FungibleToken.cdc").resolve()}"' - ) - -- path: contracts/FlowFees.cdc - # language=Python prefix="replace: Callable[[str, str], None]\n" - prepare: | - from pathlib import Path - - replace( - 'import FungibleToken from 0xFUNGIBLETOKENADDRESS', - f'import FungibleToken from "{Path("../../flow-ft/contracts/FungibleToken.cdc").resolve()}"' - ) - replace( - 'import FlowToken from 0xFLOWTOKENADDRESS', - f'import FlowToken from "{Path("FlowToken.cdc").resolve()}"' - ) - -- path: contracts/FlowIDTableStaking.cdc - # language=Python prefix="replace: Callable[[str, str], None]\n" - prepare: | - from pathlib import Path - - replace( - 'import FungibleToken from 0xFUNGIBLETOKENADDRESS', - f'import FungibleToken from "{Path("../../flow-ft/contracts/FungibleToken.cdc").resolve()}"' - ) - replace( - 'import FlowToken from 0xFLOWTOKENADDRESS', - f'import FlowToken from "{Path("FlowToken.cdc").resolve()}"' - ) - -- path: contracts/FlowStorageFees.cdc - # language=Python prefix="replace: Callable[[str, str], None]\n" - prepare: | - from pathlib import Path - - replace( - 'import FungibleToken from 0xFUNGIBLETOKENADDRESS', - f'import FungibleToken from "{Path("../../flow-ft/contracts/FungibleToken.cdc").resolve()}"' - ) - replace( - 'import FlowToken from 0xFLOWTOKENADDRESS', - f'import FlowToken from "{Path("FlowToken.cdc").resolve()}"' - ) - -- path: contracts/FlowServiceAccount.cdc - # language=Python prefix="replace: Callable[[str, str], None]\n" - prepare: | - from pathlib import Path - - replace( - 'import FungibleToken from 0xFUNGIBLETOKENADDRESS', - f'import FungibleToken from "{Path("../../flow-ft/contracts/FungibleToken.cdc").resolve()}"' - ) - replace( - 'import FlowToken from 0xFLOWTOKENADDRESS', - f'import FlowToken from "{Path("FlowToken.cdc").resolve()}"' - ) - replace( - 'import FlowFees from 0xFLOWFEESADDRESS', - f'import FlowFees from "{Path("FlowFees.cdc").resolve()}"' - ) - replace( - 'import FlowStorageFees from 0xFLOWSTORAGEFEESADDRESS', - f'import FlowStorageFees from "{Path("FlowStorageFees.cdc").resolve()}"' - ) - -- path: contracts/StakingProxy.cdc - -- path: contracts/LockedTokens.cdc - # language=Python prefix="replace: Callable[[str, str], None]\n" - prepare: | - from pathlib import Path - - replace( - 'import FlowToken from 0xFLOWTOKENADDRESS', - f'import FlowToken from "{Path("FlowToken.cdc").resolve()}"' - ) - replace( - 'import FungibleToken from 0xFUNGIBLETOKENADDRESS', - f'import FungibleToken from "{Path("../../flow-ft/contracts/FungibleToken.cdc").resolve()}"' - ) - replace( - 'import FlowIDTableStaking from 0xFLOWIDTABLESTAKINGADDRESS', - f'import FlowIDTableStaking from "{Path("FlowIDTableStaking.cdc").resolve()}"' - ) - replace( - 'import FlowStorageFees from 0xFLOWSTORAGEFEESADDRESS', - f'import FlowStorageFees from "{Path("FlowStorageFees.cdc").resolve()}"' - ) - replace( - 'import StakingProxy from 0xSTAKINGPROXYADDRESS', - f'import StakingProxy from "{Path("StakingProxy.cdc").resolve()}"' - ) - -- path: contracts/FlowStakingCollection.cdc - member_account_access: - - 'LockedTokens.cdc' - # language=Python prefix="replace: Callable[[str, str], None]\n" - prepare: | - from pathlib import Path - - replace( - 'import FungibleToken from 0xFUNGIBLETOKENADDRESS', - f'import FungibleToken from "{Path("../../flow-ft/contracts/FungibleToken.cdc").resolve()}"' - ) - replace( - 'import FlowToken from 0xFLOWTOKENADDRESS', - f'import FlowToken from "{Path("FlowToken.cdc").resolve()}"' - ) - replace( - 'import FlowIDTableStaking from 0xFLOWIDTABLESTAKINGADDRESS', - f'import FlowIDTableStaking from "{Path("FlowIDTableStaking.cdc").resolve()}"' - ) - replace( - 'import LockedTokens from 0xLOCKEDTOKENSADDRESS', - f'import LockedTokens from "{Path("LockedTokens.cdc").resolve()}"' - ) - replace( - 'import FlowStorageFees from 0xFLOWSTORAGEFEESADDRESS', - f'import FlowStorageFees from "{Path("FlowStorageFees.cdc").resolve()}"' - ) diff --git a/compat/suite/flow-ft.yaml b/compat/suite/flow-ft.yaml index 1fd621a3d5..3b872d692d 100644 --- a/compat/suite/flow-ft.yaml +++ b/compat/suite/flow-ft.yaml @@ -7,82 +7,3 @@ branch: master go_tests: - path: lib/go/test command: make test -files: -- path: contracts/FungibleToken.cdc -- path: contracts/utilityContracts/TokenForwarding.cdc - # language=Python prefix="replace: Callable[[str, str], None]\n" - prepare: | - from pathlib import Path - - replace( - 'import FungibleToken from 0xFUNGIBLETOKENADDRESS', - f'import FungibleToken from "{Path("../FungibleToken.cdc").resolve()}"' - ) - -- path: contracts/ExampleToken.cdc - # language=Python prefix="replace: Callable[[str, str], None]\n" - prepare: | - from pathlib import Path - - replace( - 'import FungibleToken from 0xFUNGIBLETOKENADDRESS', - f'import FungibleToken from "{Path("FungibleToken.cdc").resolve()}"' - ) - -- path: transactions/transfer_tokens.cdc - # language=Python prefix="replace: Callable[[str, str], None]\n" - prepare: | - from pathlib import Path - - replace( - 'import FungibleToken from 0xFUNGIBLETOKENADDRESS', - f'import FungibleToken from "{Path("../contracts/FungibleToken.cdc").resolve()}"' - ) - replace( - 'import ExampleToken from 0xTOKENADDRESS', - f'import ExampleToken from "{Path("../contracts/ExampleToken.cdc").resolve()}"' - ) - -- path: transactions/setup_account.cdc - # language=Python prefix="replace: Callable[[str, str], None]\n" - prepare: | - from pathlib import Path - - replace( - 'import FungibleToken from 0xFUNGIBLETOKENADDRESS', - f'import FungibleToken from "{Path("../contracts/FungibleToken.cdc").resolve()}"' - ) - replace( - 'import ExampleToken from 0xTOKENADDRESS', - f'import ExampleToken from "{Path("../contracts/ExampleToken.cdc").resolve()}"' - ) - -- path: transactions/mint_tokens.cdc - # language=Python prefix="replace: Callable[[str, str], None]\n" - prepare: | - from pathlib import Path - - replace( - 'import FungibleToken from 0xFUNGIBLETOKENADDRESS', - f'import FungibleToken from "{Path("../contracts/FungibleToken.cdc").resolve()}"' - ) - replace( - 'import ExampleToken from 0xTOKENADDRESS', - f'import ExampleToken from "{Path("../contracts/ExampleToken.cdc").resolve()}"' - ) - - -- path: transactions/burn_tokens.cdc - # language=Python prefix="replace: Callable[[str, str], None]\n" - prepare: | - from pathlib import Path - - replace( - 'import FungibleToken from 0xFUNGIBLETOKENADDRESS', - f'import FungibleToken from "{Path("../contracts/FungibleToken.cdc").resolve()}"' - ) - replace( - 'import ExampleToken from 0xTOKENADDRESS', - f'import ExampleToken from "{Path("../contracts/ExampleToken.cdc").resolve()}"' - ) - diff --git a/compat/suite/flow-nft.yaml b/compat/suite/flow-nft.yaml index c6d93cc7bd..958f72f5bb 100644 --- a/compat/suite/flow-nft.yaml +++ b/compat/suite/flow-nft.yaml @@ -7,57 +7,3 @@ branch: master go_tests: - path: lib/go/test command: make test -files: -- path: contracts/NonFungibleToken.cdc -- path: contracts/ExampleNFT.cdc - # language=Python prefix="replace: Callable[[str, str], None]\n" - prepare: | - from pathlib import Path - - replace( - 'import NonFungibleToken from 0x02', - f'import NonFungibleToken from "{Path("NonFungibleToken.cdc").resolve()}"' - ) - - -- path: transactions/mint_nft.cdc - # language=Python prefix="replace: Callable[[str, str], None]\n" - prepare: | - from pathlib import Path - - replace( - 'import NonFungibleToken from 0xNFTADDRESS', - f'import NonFungibleToken from "{Path("../contracts/NonFungibleToken.cdc").resolve()}"' - ) - replace( - 'import ExampleNFT from 0xNFTCONTRACTADDRESS', - f'import ExampleNFT from "{Path("../contracts/ExampleNFT.cdc").resolve()}"' - ) - -- path: transactions/setup_account.cdc - # language=Python prefix="replace: Callable[[str, str], None]\n" - prepare: | - from pathlib import Path - - replace( - 'import NonFungibleToken from 0xNFTADDRESS', - f'import NonFungibleToken from "{Path("../contracts/NonFungibleToken.cdc").resolve()}"' - ) - replace( - 'import ExampleNFT from 0xNFTCONTRACTADDRESS', - f'import ExampleNFT from "{Path("../contracts/ExampleNFT.cdc").resolve()}"' - ) - -- path: transactions/transfer_nft.cdc - # language=Python prefix="replace: Callable[[str, str], None]\n" - prepare: | - from pathlib import Path - - replace( - 'import NonFungibleToken from 0xNFTADDRESS', - f'import NonFungibleToken from "{Path("../contracts/NonFungibleToken.cdc").resolve()}"' - ) - replace( - 'import ExampleNFT from 0xNFTCONTRACTADDRESS', - f'import ExampleNFT from "{Path("../contracts/ExampleNFT.cdc").resolve()}"' - ) \ No newline at end of file diff --git a/compat/suite/nba-smart-contracts.yaml b/compat/suite/nba-smart-contracts.yaml index f56ebbfcee..8e44b443f3 100644 --- a/compat/suite/nba-smart-contracts.yaml +++ b/compat/suite/nba-smart-contracts.yaml @@ -7,84 +7,3 @@ branch: master go_tests: - path: lib/go/test command: make test -files: -- path: contracts/TopShot.cdc - # language=Python prefix="replace: Callable[[str, str], None]\n" - prepare: | - from pathlib import Path - - replace( - 'import NonFungibleToken from 0xNFTADDRESS', - f'import NonFungibleToken from "{Path("../../flow-nft/contracts/NonFungibleToken.cdc").resolve()}"' - ) - -- path: contracts/MarketTopShot.cdc - # language=Python prefix="replace: Callable[[str, str], None]\n" - prepare: | - from pathlib import Path - - replace( - 'import FungibleToken from 0xFUNGIBLETOKENADDRESS', - f'import FungibleToken from "{Path("../../flow-ft/contracts/FungibleToken.cdc").resolve()}"' - ) - - replace( - 'import NonFungibleToken from 0xNFTADDRESS', - f'import NonFungibleToken from "{Path("../../flow-nft/contracts/NonFungibleToken.cdc").resolve()}"' - ) - - replace( - 'import TopShot from 0xTOPSHOTADDRESS', - f'import TopShot from "{Path("TopShot.cdc").resolve()}"' - ) - -- path: contracts/TopShotMarketV2.cdc - # language=Python prefix="replace: Callable[[str, str], None]\n" - prepare: | - from pathlib import Path - - replace( - 'import FungibleToken from 0xFUNGIBLETOKENADDRESS', - f'import FungibleToken from "{Path("../../flow-ft/contracts/FungibleToken.cdc").resolve()}"' - ) - - replace( - 'import NonFungibleToken from 0xNFTADDRESS', - f'import NonFungibleToken from "{Path("../../flow-nft/contracts/NonFungibleToken.cdc").resolve()}"' - ) - - replace( - 'import TopShot from 0xTOPSHOTADDRESS', - f'import TopShot from "{Path("TopShot.cdc").resolve()}"' - ) - -- path: contracts/TopShotShardedCollection.cdc - # language=Python prefix="replace: Callable[[str, str], None]\n" - prepare: | - from pathlib import Path - - replace( - 'import NonFungibleToken from 0xNFTADDRESS', - f'import NonFungibleToken from "{Path("../../flow-nft/contracts/NonFungibleToken.cdc").resolve()}"' - ) - - replace( - 'import TopShot from 0xTOPSHOTADDRESS', - f'import TopShot from "{Path("TopShot.cdc").resolve()}"' - ) - -- path: contracts/TopshotAdminReceiver.cdc - # language=Python prefix="replace: Callable[[str, str], None]\n" - prepare: | - from pathlib import Path - - replace( - 'import TopShotShardedCollection from 0xSHARDEDADDRESS', - f'import TopShotShardedCollection from "{Path("TopShotShardedCollection.cdc").resolve()}"' - ) - - replace( - 'import TopShot from 0xTOPSHOTADDRESS', - f'import TopShot from "{Path("TopShot.cdc").resolve()}"' - ) -