diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b9efe8f..cf7e5fe 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,7 +13,7 @@ We provide a shortcut to standard commands using [just][just] in our `justfile`. Now that you have a local environment, you can make edits and run our tests. -``` +```bash uv run pytest tests -m "not integration" ``` @@ -34,6 +34,24 @@ uv run pytest tests -m integration # or `just integration` ``` +### Integration tests with OpenTelemetry + +Exchange primarily uses http to access model providers. If you are receiving failures, it can be helpful to see traces +of the underlying HTTP requests. For example, a 404 could be indicative of an incorrect URL or a missing model. + +First, ensure you have an OpenTelemetry compatible collector listening on port 4318, such as +[otel-tui](https://github.com/ymtdzzz/otel-tui). + +```bash +brew install ymtdzzz/tap/otel-tui +otel-tui +``` + +Then, trace your integration tests like this: +```bash +uv run dotenv -f ./tests/otel.env run -- opentelemetry-instrument pytest tests -m integration +# or `just integration-otel` + ### Integration testing with Ollama To run integration tests against Ollama, you need the model that tests expect available locally. @@ -50,8 +68,13 @@ Finally, run ollama integration tests. ```bash uv run pytest tests -m integration -k ollama # or `just integration -k ollama` + ``` +Now, you can see failure details like this: + +otel-tui + ## Pull Requests When opening a pull request, please ensure that your PR title adheres to the [Conventional Commits specification](https://www.conventionalcommits.org/). diff --git a/justfile b/justfile index b2e7b72..288569d 100644 --- a/justfile +++ b/justfile @@ -9,6 +9,9 @@ test *FLAGS: integration *FLAGS: uv run pytest tests -m integration {{FLAGS}} +integration-otel *FLAGS: + uv run dotenv -f ./tests/otel.env run -- opentelemetry-instrument pytest tests -m integration {{FLAGS}} + format: ruff check --fix && ruff format diff --git a/pyproject.toml b/pyproject.toml index 5504ce2..2211e5f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,15 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.uv] -dev-dependencies = ["pytest>=8.3.2", "codecov>=2.1.13"] +dev-dependencies = [ + "pytest>=8.3.2", + "codecov>=2.1.13", + "opentelemetry-sdk>=1.27.0", + "opentelemetry-exporter-otlp-proto-http>=1.27.0", + "opentelemetry-distro>=0.48b0", + "opentelemetry-instrumentation-httpx>=0.48b0", + "python-dotenv[cli]>=1.0.1" +] [project.entry-points."exchange.provider"] openai = "exchange.providers.openai:OpenAiProvider" diff --git a/tests/otel.env b/tests/otel.env new file mode 100644 index 0000000..75091c8 --- /dev/null +++ b/tests/otel.env @@ -0,0 +1,5 @@ +OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 +OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf +OTEL_RESOURCE_ATTRIBUTES=service.name=exchange-ai +OTEL_METRIC_EXPORT_INTERVAL=100 +OTEL_BSP_SCHEDULE_DELAY=100