diff --git a/airbyte-ci/connectors/live-tests/.gitignore b/airbyte-ci/connectors/live-tests/.gitignore index 692dc1a1022d..452eecef73dd 100644 --- a/airbyte-ci/connectors/live-tests/.gitignore +++ b/airbyte-ci/connectors/live-tests/.gitignore @@ -1,2 +1 @@ regression_tests_artifacts -live_tests_debug_reports diff --git a/airbyte-ci/connectors/live-tests/README.md b/airbyte-ci/connectors/live-tests/README.md index 87a9951a145b..ab6a5540a4d2 100644 --- a/airbyte-ci/connectors/live-tests/README.md +++ b/airbyte-ci/connectors/live-tests/README.md @@ -16,106 +16,6 @@ This project contains utilities for running connector tests against live data. poetry install ``` -## Commands - -### `debug` - -``` -Usage: poetry run live-tests debug [OPTIONS] {check|discover|read|read-with-state|spec} - - Run a specific command on one or multiple connectors and persists the - outputs to local storage. - -Options: - --connection-id TEXT - --config-path FILE - --catalog-path FILE - --state-path FILE - -c, --connector-image TEXT Docker image name of the connector to debug - (e.g. `airbyte/source-faker:latest`, - `airbyte/source-faker:dev`) [required] - -hc, --http-cache Use the HTTP cache for the connector. - --help Show this message and exit. -``` - -This command is made to run any of the following connector commands against one or multiple connector images. - -**Available connector commands:** - -- `spec` -- `check` -- `discover` -- `read` or `read_with_state` (requires a `--state-path` to be passed) - -It will write artifacts to an output directory: - -- `stdout.log`: The collected standard output following the command execution -- `stderr.log`: The collected standard error following the c -- `http_dump.txt`: An `mitmproxy` http stream log. Can be consumed with `mitmweb` (version `9.0.1`) for debugging. -- `airbyte_messages.db`: A DuckDB database containing the messages produced by the connector. -- `airbyte_messages`: A directory containing `.jsonl` files for each message type (logs, records, traces, controls, states etc.) produced by the connector. - -#### Example - -Let's run `debug` to check the output of `read` on two different versions of the same connector: - -```bash -poetry run live-tests debug read \ ---connection-id=d3bd39cd-6fec-4691-a661-d52c466d8554 ---connector-image=airbyte/source-pokeapi:dev \ ---connector-image=airbyte/source-pokeapi:latest \ ---config-path=poke_config.json \ ---catalog-path=configured_catalog.json -``` - -It will store the results in a `live_test_debug_reports` directory under the current working directory: - -``` -live_tests_debug_reports -└── 1709547771 - └── source-pokeapi - └── read - ├── dev - │   ├── airbyte_messages - | │ ├── duck.db # DuckDB database - │   │   ├── logs.jsonl - │   │   ├── records.jsonl - │   │   └── traces.jsonl - │   ├── stderr.log - │   └── stdout.log - └── latest - ├── airbyte_messages - │ ├── duck.db # DuckDB database - │   ├── logs.jsonl - │   ├── records.jsonl - │   └── traces.jsonl - ├── stderr.log - └── stdout.log - -``` - -You can also run the `debug` command on a live connection by passing the `--connection-id` option: - -```bash -poetry run live-tests debug read \ ---connector-image=airbyte/source-pokeapi:dev \ ---connector-image=airbyte/source-pokeapi:latest \ ---connection-id= -``` - -##### Consuming `http_dump.mitm` - -You can install [`mitmproxy`](https://mitmproxy.org/): - -```bash -pipx install mitmproxy -``` - -And run: - -```bash -mitmweb --rfile=http_dump.mitm -``` ## Regression tests @@ -279,6 +179,10 @@ The traffic recorded on the control connector is passed to the target connector ## Changelog +### 0.19.0 + +Delete the `debug`command. + ### 0.18.8 Improve error message when failing to retrieve connection. diff --git a/airbyte-ci/connectors/live-tests/pyproject.toml b/airbyte-ci/connectors/live-tests/pyproject.toml index 411025b8042f..e5504fdd58ae 100644 --- a/airbyte-ci/connectors/live-tests/pyproject.toml +++ b/airbyte-ci/connectors/live-tests/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "live-tests" -version = "0.18.8" +version = "0.19.0" description = "Contains utilities for testing connectors against live data." authors = ["Airbyte "] license = "MIT" @@ -44,8 +44,6 @@ genson = "^1.2.2" segment-analytics-python = "^2.3.2" python-slugify = ">=8.0.4" -[tool.poetry.scripts] -live-tests = "live_tests.cli:live_tests" [tool.poetry.group.dev.dependencies] ruff = "^0.3.0" diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/cli.py b/airbyte-ci/connectors/live-tests/src/live_tests/cli.py deleted file mode 100644 index f23da702fe0b..000000000000 --- a/airbyte-ci/connectors/live-tests/src/live_tests/cli.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -from __future__ import annotations - -import asyncclick as click -from live_tests.debug.cli import debug_cmd - - -@click.group() -@click.pass_context -async def live_tests(ctx: click.Context) -> None: - pass - - -live_tests.add_command(debug_cmd) diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/debug/__init__.py b/airbyte-ci/connectors/live-tests/src/live_tests/debug/__init__.py deleted file mode 100644 index d3a001370f73..000000000000 --- a/airbyte-ci/connectors/live-tests/src/live_tests/debug/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -from __future__ import annotations - -import os -import sys - -import dagger - -DAGGER_EXEC_TIMEOUT = dagger.Timeout( - int(os.environ.get("DAGGER_EXEC_TIMEOUT", "3600")) -) # One hour by default -DAGGER_CONFIG = dagger.Config(timeout=DAGGER_EXEC_TIMEOUT, log_output=sys.stderr) diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/debug/cli.py b/airbyte-ci/connectors/live-tests/src/live_tests/debug/cli.py deleted file mode 100644 index 47f24e3db645..000000000000 --- a/airbyte-ci/connectors/live-tests/src/live_tests/debug/cli.py +++ /dev/null @@ -1,137 +0,0 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -from __future__ import annotations - -import logging -import textwrap -from pathlib import Path -from typing import Optional - -import asyncclick as click -import dagger -from live_tests.commons.connection_objects_retrieval import COMMAND_TO_REQUIRED_OBJECT_TYPES, get_connection_objects -from live_tests.commons.connector_runner import ConnectorRunner -from live_tests.commons.models import ActorType, Command, ConnectionObjects, ConnectorUnderTest, ExecutionInputs, TargetOrControl -from live_tests.commons.utils import clean_up_artifacts -from live_tests.debug import DAGGER_CONFIG -from rich.prompt import Prompt - -from .consts import MAIN_OUTPUT_DIRECTORY - -LOGGER = logging.getLogger("debug_command") - - -@click.command( - "debug", - help="Run a specific command on one or multiple connectors and persists the outputs to local storage.", -) -@click.argument( - "command", - type=click.Choice([c.value for c in Command]), - callback=lambda _, __, value: Command(value), -) -@click.option("--connection-id", type=str, required=False, default=None) -@click.option( - "--config-path", - type=click.Path(file_okay=True, readable=True, dir_okay=False, resolve_path=True, path_type=Path), - required=False, - default=None, -) -@click.option( - "--catalog-path", - type=click.Path(file_okay=True, readable=True, dir_okay=False, resolve_path=True, path_type=Path), - required=False, - default=None, -) -@click.option( - "--state-path", - type=click.Path(file_okay=True, readable=True, dir_okay=False, resolve_path=True, path_type=Path), - required=False, - default=None, -) -@click.option( - "-c", - "--connector-image", - "connector_images", - help="Docker image name of the connector to debug (e.g. `airbyte/source-faker:latest`, `airbyte/source-faker:dev`)", - multiple=True, - type=str, - required=True, -) -# TODO add an env var options to pass to the connector -@click.pass_context -async def debug_cmd( - ctx: click.Context, - command: Command, - connection_id: Optional[str], - config_path: Optional[Path], - catalog_path: Optional[Path], - state_path: Optional[Path], - connector_images: list[str], -) -> None: - if connection_id: - retrieval_reason = click.prompt("👮‍♂️ Please provide a reason for accessing the connection objects. This will be logged") - else: - retrieval_reason = None - - try: - connection_objects = get_connection_objects( - COMMAND_TO_REQUIRED_OBJECT_TYPES[command], - connection_id, - config_path, - catalog_path, - state_path, - retrieval_reason, - ) - except ValueError as e: - raise click.UsageError(str(e)) - async with dagger.Connection(config=DAGGER_CONFIG) as dagger_client: - MAIN_OUTPUT_DIRECTORY.mkdir(parents=True, exist_ok=True) - try: - for connector_image in connector_images: - await _execute_command_and_save_artifacts( - dagger_client, - connector_image, - command, - connection_objects, - ) - - Prompt.ask( - textwrap.dedent( - """ - Debug artifacts will be destroyed after this prompt. - Press enter when you're done reading them. - 🚨 Do not copy them elsewhere on your disk!!! 🚨 - """ - ) - ) - finally: - clean_up_artifacts(MAIN_OUTPUT_DIRECTORY, LOGGER) - - -async def _execute_command_and_save_artifacts( - dagger_client: dagger.Client, - connector_image: str, - command: Command, - connection_objects: ConnectionObjects, -) -> None: - try: - connector_under_test = await ConnectorUnderTest.from_image_name(dagger_client, connector_image, TargetOrControl.CONTROL) - if connector_under_test.actor_type is ActorType.SOURCE: - actor_id = connection_objects.source_id - else: - actor_id = connection_objects.destination_id - assert actor_id is not None - execution_inputs = ExecutionInputs( - global_output_dir=MAIN_OUTPUT_DIRECTORY, - connector_under_test=connector_under_test, - command=command, - config=connection_objects.source_config, - configured_catalog=connection_objects.configured_catalog, - state=connection_objects.state, - environment_variables=None, - actor_id=actor_id, - ) - except ValueError as e: - raise click.UsageError(str(e)) - execution_result = await ConnectorRunner(dagger_client, execution_inputs).run() - await execution_result.save_artifacts(MAIN_OUTPUT_DIRECTORY) diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/debug/consts.py b/airbyte-ci/connectors/live-tests/src/live_tests/debug/consts.py deleted file mode 100644 index 192881be5915..000000000000 --- a/airbyte-ci/connectors/live-tests/src/live_tests/debug/consts.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2024 Airbyte, Inc., all rights reserved. -from __future__ import annotations - -from pathlib import Path - -MAIN_OUTPUT_DIRECTORY = Path("/tmp/debug_artifacts")