From b8f3c25ec8e3c5b4eeefaff1192d97ba9b790654 Mon Sep 17 00:00:00 2001 From: Ulrich Kautz Date: Tue, 12 Nov 2024 11:29:58 +0000 Subject: [PATCH] fbshipit-source-id: d82bc9705c35 --- cogwheel/runner.py | 231 ---------------------------- cogwheel/tests/append/test-main.md | 5 - cogwheel/tests/append/test.md | 6 - cogwheel/tests/base/cram.t | 43 ------ cogwheel/tests/base/envvars.md | 22 --- cogwheel/tests/base/markdown.md | 57 ------- cogwheel/tests/package/setup.sh | 6 - cogwheel/tests/package/test.md | 31 ---- cogwheel/tests/prepend/test-main.md | 6 - cogwheel/tests/prepend/test.md | 5 - 10 files changed, 412 deletions(-) delete mode 100644 cogwheel/runner.py delete mode 100644 cogwheel/tests/append/test-main.md delete mode 100644 cogwheel/tests/append/test.md delete mode 100644 cogwheel/tests/base/cram.t delete mode 100644 cogwheel/tests/base/envvars.md delete mode 100644 cogwheel/tests/base/markdown.md delete mode 100644 cogwheel/tests/package/setup.sh delete mode 100644 cogwheel/tests/package/test.md delete mode 100644 cogwheel/tests/prepend/test-main.md delete mode 100644 cogwheel/tests/prepend/test.md diff --git a/cogwheel/runner.py b/cogwheel/runner.py deleted file mode 100644 index 9fee048..0000000 --- a/cogwheel/runner.py +++ /dev/null @@ -1,231 +0,0 @@ -# (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -# pyre-strict -from __future__ import annotations - -import dataclasses -import json -import os -import subprocess -import sys -from argparse import ArgumentParser -from dataclasses import dataclass -from pathlib import Path - -from clifoundation.scrut.buck.scruttest import generate_test_method_name - -from windtunnel.cogwheel.failure import ( - AssertionFailure, - FailureHandler, - InfrastructureFailure, - UserSetupFailure, - UserSkippedFailure, -) -from windtunnel.cogwheel.lib.logging import FrameworkLogger, TestLogger -from windtunnel.cogwheel.result import CogwheelTestSuiteResult -from windtunnel.cogwheel.result_if.ttypes import Result -from windtunnel.cogwheel.test import cogwheel_test, CogwheelTest - - -SCRUT_BINARY: str = "/usr/local/bin/scrut" -SCRUT_EXTENSIONS: set[str] = {".md", ".markdown", ".t", ".cram"} -WORKLOAD_ENVVAR: str = "SERVICELAB_WORKLOAD" -SCRUT_CONFIG_FILE: str = "scrut_config.json" - - -@dataclass -class CogwheelScrutTestConfig: - args: list[str] - srcs: list[str] - prepend_srcs: list[str] - append_srcs: list[str] - env: dict[str, str] - - -class CogwheelScrutTestFailureHandler(FailureHandler): - - def handleTestFailure( - self, e: Exception, name: str, result: CogwheelTestSuiteResult - ) -> None: - if isinstance(e, AssertionFailure): - FrameworkLogger.error(f"An assertion failed: {e}") - - if isinstance(e, InfrastructureFailure): - status = Result.INFRA_FAILURE - elif isinstance(e, UserSkippedFailure): - result.skipped.append(name) - status = Result.SKIPPED - else: - status = Result.FAILURE - - result.setTestStatus( - test_name=name, - status=status, - type=type(e).__name__, - message=str(e), - stacktrace=None, - ) - - def handleTestsuiteFailure( - self, e: Exception, result: CogwheelTestSuiteResult - ) -> None: - if isinstance(e, InfrastructureFailure): - status = Result.INFRA_FAILURE - elif isinstance(e, UserSetupFailure): - status = Result.FAILURE - elif isinstance(e, UserSkippedFailure): - status = Result.SKIPPED - else: - status = Result.SETUP_FAILURE - - result.setTestsuiteStatus( - status=status, - type=type(e).__name__, - message=None, - stacktrace=None, - is_status_for_main_test=True, - ) - result.setMainTestStatus(status) - - -class CogwheelScrutTest(CogwheelTest): - - def __init__(self, config: CogwheelScrutTestConfig) -> None: - super().__init__(handler=CogwheelScrutTestFailureHandler()) - self._config = config - TestLogger.info( - f"Initialize workload {_workload_name()} with {json.dumps(dataclasses.asdict(config))}" - ) - self._register_tests() - - def _register_tests(self) -> None: - """ - Iterate all test files and register them as a Python test method - """ - - for src in sorted(self._config.srcs): - ext = Path(src).suffix - if ext not in SCRUT_EXTENSIONS: - continue - self._setup_test(src) - - pass - - def _setup_test(self, path: str) -> None: - """ - Create a callback test and register with `cogwheel_test` decorator - """ - - def call_scrut_test(self: CogwheelScrutTest) -> None: - self._run_test(path) - - name = generate_test_method_name(Path(path)) - TestLogger.info(f"Setup test {path} as {name}") - # pyre-ignore[16] - call_scrut_test.__name__ = name - cogwheel_test(call_scrut_test) - - def _run_test(self, path: str) -> None: - """ - Execute a test runnig `scrut test ... - """ - - args = self._build_args() - TestLogger.info(f"Run test {path} with args {json.dumps(args)}") - stdout, stderr, code = self._run( - [ - SCRUT_BINARY, - "test", - "--log-level=debug", - *args, - path, - ] - ) - if code == 0: - TestLogger.info( - f"Test {path} succeded with exit code {code}\n\nSTDOUT:\n{stdout}\n\nSTDERR:\n{stderr}\n" - ) - self.assertEqual( - 0, - code, - f"Test {path} failed with exit code {code}\n\nSTDOUT:\n{stdout}\n\nSTDERR:\n{stderr}\n", - ) - - def _build_args(self) -> list[str]: - """ - Create list of parameters for `scrut test` execution - """ - args = self._config.args.copy() - for param, srcs in { - "--prepend-test-file-paths": self._config.prepend_srcs, - "--append-test-file-paths": self._config.append_srcs, - }.items(): - if not srcs: - continue - args.extend([param, " ".join(srcs)]) - return args - - def _run(self, cmd: str | list[str]) -> tuple[str, str, int | None]: - """ - Excecute a command and return the output - """ - TestLogger.info(f"Run command {json.dumps(cmd)}") - result = subprocess.run( - cmd, - capture_output=True, - env=self._config.env, - cwd=self._test_srcs_directory(), - ) - return ( - result.stdout.decode("utf-8"), - result.stderr.decode("utf-8"), - result.returncode, - ) - - def _test_srcs_directory(self) -> str: - """ - Path to the directory where the test files are located - """ - return f"{_harness_directory(self.get_package_path())}/test_srcs" - - -def main() -> None: - parser = ArgumentParser("CogwheelScrutTest harness", add_help=False) - parser.add_argument("--package-path") - args, _ = parser.parse_known_args() - - config_file = os.path.join(_harness_directory(args.package_path), SCRUT_CONFIG_FILE) - TestLogger.info(f"Loading scrut config from file {config_file}") - with open(config_file) as f: - config = json.load(f) - TestLogger.info(f"Loaded scrut config {json.dumps(config)}") - CogwheelScrutTest( - config=CogwheelScrutTestConfig(**config), - ).main() - - -def _harness_directory(package_path: str | None) -> str: - """ - Returns the path to the directory where the test harness is located - """ - if not package_path: - package_path = "/packages" - return f"{package_path}/{_workload_name()}_test_harness" - - -def _workload_name() -> str: - """ - Returns the name of the workload that has the shape `cogwheel_scrut__`. - """ - try: - # environment variable is only available in remote executions - return os.environ[WORKLOAD_ENVVAR] - except KeyError: - # fallback to using the name of the par file that has the format: - # /packages/_test_harness/.par - # this is needed in local runs where the environment variable is notset - return os.path.basename(sys.argv[0]).removesuffix(".par") - - -if __name__ == "__main__": - main() # pragma: no cover diff --git a/cogwheel/tests/append/test-main.md b/cogwheel/tests/append/test-main.md deleted file mode 100644 index 340de03..0000000 --- a/cogwheel/tests/append/test-main.md +++ /dev/null @@ -1,5 +0,0 @@ -# Setups an env variable for the other appended file to consume - -```scrut -$ export APPEND_TEST_VAR='I am still here' -``` diff --git a/cogwheel/tests/append/test.md b/cogwheel/tests/append/test.md deleted file mode 100644 index e123204..0000000 --- a/cogwheel/tests/append/test.md +++ /dev/null @@ -1,6 +0,0 @@ -# See whether the env variable was set properly before - -```scrut -$ echo $APPEND_TEST_VAR -I am still here -``` diff --git a/cogwheel/tests/base/cram.t b/cogwheel/tests/base/cram.t deleted file mode 100644 index aec60f1..0000000 --- a/cogwheel/tests/base/cram.t +++ /dev/null @@ -1,43 +0,0 @@ -A simple string without new line - $ echo -n hello - hello (no-eol) - - -A simple string with newline - $ echo this should be working - this should be working - - -Using expanding regular expressions - $ echo -ne "foo is\nbar1\nbar2\nbar3\nbaz" - foo is - bar\d+ (re+) - baz (no-eol) - - -Using expanding globs - $ echo -e "foo is\nbar1\nbar2\nbar3\nbaz" - foo is - bar* (glob+) - baz - - -Setting shell state - $ shopt -s expand_aliases - > SOME_VAR1=foo1 - > export SOME_VAR2=foo2 - > some_function() { - > echo foo3 - > } - > alias some_alias='echo foo4' - - -Using shell state - $ echo "shell var: $SOME_VAR1" - > echo "env var: $SOME_VAR2" - > echo "function: $(some_function)" - > echo "alias: $(some_alias)" - shell var: foo1 - env var: foo2 - function: foo3 - alias: foo4 diff --git a/cogwheel/tests/base/envvars.md b/cogwheel/tests/base/envvars.md deleted file mode 100644 index ad1b544..0000000 --- a/cogwheel/tests/base/envvars.md +++ /dev/null @@ -1,22 +0,0 @@ -# Test whether standard environment variables are available - -See: https://www.internalfb.com/intern/wiki/CLI_Foundation/Tools/Scrut/Advanced/Specifics/#common-linux-environment - -## Common (linux) environment variables - -```scrut -$ echo "LANG = '$LANG'" -> echo "LANGUAGE = '$LANGUAGE'" -> echo "LC_ALL = '$LC_ALL'" -> echo "TZ = '$TZ'" -> echo "COLUMNS = '$COLUMNS'" -> echo "CDPATH = '$CDPATH'" -> echo "GREP_OPTIONS = '$GREP_OPTIONS'" -LANG = 'C' -LANGUAGE = 'C' -LC_ALL = 'C' -TZ = 'GMT' -COLUMNS = '80' -CDPATH = '' -GREP_OPTIONS = '' -``` diff --git a/cogwheel/tests/base/markdown.md b/cogwheel/tests/base/markdown.md deleted file mode 100644 index d636c31..0000000 --- a/cogwheel/tests/base/markdown.md +++ /dev/null @@ -1,57 +0,0 @@ -# A bunch of test - -## A simple string without new line - -```scrut -$ echo -n hello -hello (no-eol) -``` - -## A simple string with newline - -```scrut -$ echo this should be working -this should be working -``` - -## Using expanding regular expressions - -```scrut -$ echo -ne "foo is\nbar1\nbar2\nbar3\nbaz" -foo is -bar\d+ (re+) -baz (no-eol) -``` - -## Using expanding globs - -```scrut -$ echo -e "foo is\nbar1\nbar2\nbar3\nbaz" -foo is -bar* (glob+) -baz -``` - -## Setting shell state - -```scrut -$ SOME_VAR1=foo1 -> export SOME_VAR2=foo2 -> some_function() { -> echo foo3 -> } -> alias some_alias='echo foo4' -``` - -## Using shell state - -```scrut -$ echo "shell var: $SOME_VAR1" -> echo "env var: $SOME_VAR2" -> echo "function: $(some_function)" -> echo "alias: $(some_alias)" -shell var: foo1 -env var: foo2 -function: foo3 -alias: foo4 -``` diff --git a/cogwheel/tests/package/setup.sh b/cogwheel/tests/package/setup.sh deleted file mode 100644 index cad82a6..0000000 --- a/cogwheel/tests/package/setup.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -shopt -s expand_aliases - -alias cli='$CLI_BIN' diff --git a/cogwheel/tests/package/test.md b/cogwheel/tests/package/test.md deleted file mode 100644 index 8d092d2..0000000 --- a/cogwheel/tests/package/test.md +++ /dev/null @@ -1,31 +0,0 @@ -# A test that use user-provided packages - -This test validates that provided RPMs and FPBKGs are indeed available. - -Load setup - -```scrut -$ source "$TESTDIR/setup.sh" -``` - -## Is RPM available? - -```scrut -$ cli --help -* (glob+) -Usage: ohno -* (glob+) -``` - -## Is New-Style FBPKG available? - -```scrut -$ test -d /packages/biggrep.client -``` - - -## Is Old-Style FBPKG available? - -```scrut -$ test -d /packages/hwcontrol.cli -``` diff --git a/cogwheel/tests/prepend/test-main.md b/cogwheel/tests/prepend/test-main.md deleted file mode 100644 index 8bc3f05..0000000 --- a/cogwheel/tests/prepend/test-main.md +++ /dev/null @@ -1,6 +0,0 @@ -# Tests for the existence of the env vra - -```scrut -$ echo $PREPEND_TEST -I am here -``` diff --git a/cogwheel/tests/prepend/test.md b/cogwheel/tests/prepend/test.md deleted file mode 100644 index afeff76..0000000 --- a/cogwheel/tests/prepend/test.md +++ /dev/null @@ -1,5 +0,0 @@ -# Ensures some file gets created so that we can test it's contents later - -```scrut -$ export PREPEND_TEST="I am here" -```