From f52fe6c79b849e4922be12315194eac1f7cec855 Mon Sep 17 00:00:00 2001 From: Emmanuel Evbuomwan Date: Wed, 27 Nov 2024 12:40:59 +0100 Subject: [PATCH] feat: e2e tests within docker - we split out e2e tests into own folder - e2e tests will run against the service in docker compose - we start by splitting out the prometheus and kafka e2e tests which do not need local running services --- .github/workflows/tests.yml | 39 +---- .gitignore | 4 +- GNUmakefile | 8 + container/karapace.registry.env | 3 + container/karapace.rest.env | 1 + karapace.config.env | 9 +- mypy.ini | 2 +- pyproject.toml | 10 ++ requirements/requirements-dev.txt | 147 +++++++++++++++--- requirements/requirements-typing.txt | 144 +++++++++++++++-- requirements/requirements.txt | 141 +++++++++++++++-- src/karapace.egg-info/top_level.txt | 3 + src/karapace/instrumentation/prometheus.py | 1 + src/karapace/protobuf/io.py | 2 +- src/schema_registry/schema_registry_apis.py | 5 - src/schema_registry/user.py | 5 - tests/e2e/__init__.py | 4 + tests/e2e/conftest.py | 131 ++++++++++++++++ .../instrumentation/__init__.py | 0 .../instrumentation/test_prometheus.py | 0 tests/{integration => e2e}/kafka/__init__.py | 0 .../{integration => e2e}/kafka/test_admin.py | 0 .../kafka/test_consumer.py | 0 .../kafka/test_producer.py | 0 .../schema_registry/__init__.py | 0 .../schema_registry/test_jsonschema.py | 0 tests/{integration => e2e}/test_karapace.py | 0 27 files changed, 568 insertions(+), 91 deletions(-) create mode 100644 src/karapace.egg-info/top_level.txt create mode 100644 tests/e2e/__init__.py create mode 100644 tests/e2e/conftest.py rename tests/{integration => e2e}/instrumentation/__init__.py (100%) rename tests/{integration => e2e}/instrumentation/test_prometheus.py (100%) rename tests/{integration => e2e}/kafka/__init__.py (100%) rename tests/{integration => e2e}/kafka/test_admin.py (100%) rename tests/{integration => e2e}/kafka/test_consumer.py (100%) rename tests/{integration => e2e}/kafka/test_producer.py (100%) rename tests/{integration => e2e}/schema_registry/__init__.py (100%) rename tests/{integration => e2e}/schema_registry/test_jsonschema.py (100%) rename tests/{integration => e2e}/test_karapace.py (100%) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1ce27c993..c801cd1e7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,10 +14,9 @@ env: FORCE_COLOR: 1 PIP_PROGRESS_BAR: off PYTHONUNBUFFERED: 1 - KARAPACE_DOTENV: ${{ github.workspace }}/karapace.config.env jobs: - unit-tests: + tests: runs-on: ubuntu-latest strategy: matrix: @@ -41,8 +40,8 @@ jobs: with: go-version: '1.21.0' - - name: Install requirements - run: make install + - name: Create virtual environment & install requirements + run: make install-dev - name: Resolve Karapace version run: | @@ -53,41 +52,20 @@ jobs: - name: Run containers run: KARAPACE_VERSION=${{ env.KARAPACE_VERSION }} docker compose --file=container/compose.yml up --build --wait --detach - - run: make install-dev - run: make unit-tests-in-docker env: COVERAGE_FILE: ".coverage.${{ matrix.python-version }}" PYTEST_ARGS: "--cov=src --cov-append --numprocesses 4" - KARAPACE_VERSION=: ${{ env.KARAPACE_VERSION }} - - integration-tests: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [ '3.10', '3.11', '3.12' ] - env: - PYTEST_ADDOPTS: >- - --log-dir=/tmp/ci-logs - --log-file=/tmp/ci-logs/pytest.log - --showlocals - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - cache: pip - python-version: ${{ matrix.python-version }} - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: '1.21.0' + - run: make e2e-tests-in-docker + env: + COVERAGE_FILE: ".coverage.${{ matrix.python-version }}" + PYTEST_ARGS: "--cov=src --cov-append --numprocesses 4" - run: make integration-tests env: COVERAGE_FILE: ".coverage.${{ matrix.python-version }}" - PYTEST_ARGS: "--cov=src --cov-append --random-order --numprocesses 4" + PYTEST_ARGS: "--cov=karapace --cov-append --random-order --numprocesses 4" - name: Archive logs uses: actions/upload-artifact@v4 @@ -95,7 +73,6 @@ jobs: with: name: karapace-integration-test-logs-${{ matrix.python-version }} path: /tmp/ci-logs - - name: Archive coverage file uses: actions/upload-artifact@v4 with: diff --git a/.gitignore b/.gitignore index 612ad46b2..73ce34070 100644 --- a/.gitignore +++ b/.gitignore @@ -10,12 +10,12 @@ __pycache__/ /build/ /dist/ -/karapace.egg-info/ +src/karapace.egg-info/ /karapace-rpm-src.tar /kafka_*.tgz /kafka_*/ venv -/karapace/version.py +src/karapace/version.py .run .python-version .hypothesis/ diff --git a/GNUmakefile b/GNUmakefile index b8305d65a..a4873115c 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -111,3 +111,11 @@ unit-tests-in-docker: rm -fr runtime/* $(KARAPACE-CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/unit/ rm -fr runtime/* + +.PHONY: e2e-tests-in-docker +e2e-tests-in-docker: export PYTEST_ARGS ?= +e2e-tests-in-docker: + rm -fr runtime/* + sleep 10 + $(KARAPACE-CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/e2e/ + rm -fr runtime/* diff --git a/container/karapace.registry.env b/container/karapace.registry.env index cd757a99b..8004c56d8 100644 --- a/container/karapace.registry.env +++ b/container/karapace.registry.env @@ -45,3 +45,6 @@ STATSD_PORT=8125 KAFKA_SCHEMA_READER_STRICT_MODE=False KAFKA_RETRIABLE_ERRORS_SILENCED=True USE_PROTOBUF_FORMATTER=False +HTTP_REQUEST_MAX_SIZE=1048576 +REST_BASE_URI=http://karapace-rest-proxy:8082 +TAGS='{ "app": "karapace-schema-registry" }' diff --git a/container/karapace.rest.env b/container/karapace.rest.env index 3df13f3b2..cf0a854f0 100644 --- a/container/karapace.rest.env +++ b/container/karapace.rest.env @@ -48,4 +48,5 @@ KAFKA_SCHEMA_READER_STRICT_MODE=False KAFKA_RETRIABLE_ERRORS_SILENCED=True USE_PROTOBUF_FORMATTER=False HTTP_REQUEST_MAX_SIZE=1048576 +REST_BASE_URI=http://karapace-rest-proxy:8082 TAGS='{ "app": "karapace-rest-proxy" }' diff --git a/karapace.config.env b/karapace.config.env index 70cf0c616..89cc63764 100644 --- a/karapace.config.env +++ b/karapace.config.env @@ -1,8 +1,9 @@ ACCESS_LOGS_DEBUG=False +ACCESS_LOG_CLASS=aiohttp.web_log.AccessLogger ADVERTISED_HOSTNAME=127.0.0.1 ADVERTISED_PORT=8081 ADVERTISED_PROTOCOL=http -BOOTSTRAP_URI=127.0.0.1:9092 +BOOTSTRAP_URI=kafka:29092 CLIENT_ID=sr-1 COMPATIBILITY=BACKWARD CONNECTIONS_MAX_IDLE_MS=15000 @@ -14,7 +15,7 @@ FETCH_MIN_BYTES=1 GROUP_ID=group_id8357e932 HOST=127.0.0.1 PORT=8081 -REGISTRY_HOST=127.0.0.1 +REGISTRY_HOST=karapace-schema-registry REGISTRY_PORT=8081 REST_AUTHORIZATION=False LOG_HANDLER=stdout @@ -38,8 +39,10 @@ NAME_STRATEGY=topic_name NAME_STRATEGY_VALIDATION=True MASTER_ELECTION_STRATEGY=lowest PROTOBUF_RUNTIME_DIRECTORY=runtime -STATSD_HOST=127.0.0.1 +STATSD_HOST=statsd-exporter STATSD_PORT=8125 KAFKA_SCHEMA_READER_STRICT_MODE=False KAFKA_RETRIABLE_ERRORS_SILENCED=True USE_PROTOBUF_FORMATTER=False +REST_BASE_URI=http://karapace-rest-proxy:8082 +TAGS='{ "app": "karapace" }' diff --git a/mypy.ini b/mypy.ini index c4ef8efd1..30d56b0bc 100644 --- a/mypy.ini +++ b/mypy.ini @@ -15,7 +15,7 @@ warn_no_return = True warn_unreachable = True strict_equality = True -[mypy-karapace.schema_registry_apis] +[mypy-schema_registry.schema_registry_apis] ignore_errors = True [mypy-karapace.compatibility.jsonschema.checks] diff --git a/pyproject.toml b/pyproject.toml index f1f9016cb..6512d9d23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,10 @@ dependencies = [ "zstandard", "prometheus-client == 0.20.0", "yarl == 1.12.1", + "opentelemetry-api == 1.28.2", + "opentelemetry-sdk == 1.28.2", + "opentelemetry-instrumentation-fastapi == 0.49b2", + "dependency-injector == 4.43.0", # Patched dependencies # @@ -105,6 +109,12 @@ typing = [ [tool.setuptools] include-package-data = true +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +karapace = ["*.yaml"] + [tool.setuptools_scm] version_file = "src/karapace/version.py" diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt index 9848f80e0..6c6e4a9e7 100644 --- a/requirements/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make pin-requirements @@ -8,15 +8,20 @@ accept-types==0.4.1 # via karapace (/karapace/pyproject.toml) aiohappyeyeballs==2.4.3 # via aiohttp -aiohttp==3.10.10 +aiohttp==3.10.11 # via karapace (/karapace/pyproject.toml) aiokafka==0.10.0 # via karapace (/karapace/pyproject.toml) aiosignal==1.3.1 # via aiohttp anyio==4.6.2.post1 - # via watchfiles -async-timeout==4.0.3 + # via + # httpx + # starlette + # watchfiles +asgiref==3.8.1 + # via opentelemetry-instrumentation-asgi +async-timeout==5.0.1 # via # aiohttp # aiokafka @@ -38,20 +43,35 @@ cachetools==5.3.3 certifi==2024.8.30 # via # geventhttpclient + # httpcore + # httpx # requests # sentry-sdk charset-normalizer==3.4.0 # via requests click==8.1.7 - # via flask + # via + # flask + # typer + # uvicorn configargparse==1.7 # via locust confluent-kafka==2.4.0 # via karapace (/karapace/pyproject.toml) -coverage[toml]==7.6.4 +coverage[toml]==7.6.8 # via pytest-cov cramjam==2.9.0 # via python-snappy +dependency-injector==4.43.0 + # via karapace (/karapace/pyproject.toml) +deprecated==1.2.15 + # via + # opentelemetry-api + # opentelemetry-semantic-conventions +dnspython==2.7.0 + # via email-validator +email-validator==2.2.0 + # via fastapi exceptiongroup==1.2.2 # via # anyio @@ -61,9 +81,13 @@ execnet==2.1.1 # via pytest-xdist fancycompleter==0.9.1 # via pdbpp +fastapi[standard]==0.115.5 + # via karapace (/karapace/pyproject.toml) +fastapi-cli[standard]==0.0.5 + # via fastapi filelock==3.16.1 # via karapace (/karapace/pyproject.toml) -flask==3.0.3 +flask==3.1.0 # via # flask-cors # flask-login @@ -80,19 +104,31 @@ gevent==24.11.1 # via # geventhttpclient # locust -geventhttpclient==2.3.1 +geventhttpclient==2.3.3 # via locust greenlet==3.1.1 # via gevent -hypothesis==6.118.8 +h11==0.14.0 + # via + # httpcore + # uvicorn +httpcore==1.0.7 + # via httpx +httptools==0.6.4 + # via uvicorn +httpx==0.27.2 + # via fastapi +hypothesis==6.119.4 # via karapace (/karapace/pyproject.toml) idna==3.10 # via # anyio + # email-validator + # httpx # requests # yarl importlib-metadata==8.5.0 - # via flask + # via opentelemetry-api iniconfig==2.0.0 # via pytest isodate==0.7.2 @@ -100,12 +136,14 @@ isodate==0.7.2 itsdangerous==2.2.0 # via flask jinja2==3.1.4 - # via flask + # via + # fastapi + # flask jsonschema==4.23.0 # via karapace (/karapace/pyproject.toml) jsonschema-specifications==2024.10.1 # via jsonschema -locust==2.32.2 +locust==2.32.3 # via karapace (/karapace/pyproject.toml) lz4==4.3.3 # via karapace (/karapace/pyproject.toml) @@ -123,11 +161,40 @@ multidict==6.1.0 # via # aiohttp # yarl -networkx==3.2.1 +networkx==3.4.2 + # via karapace (/karapace/pyproject.toml) +opentelemetry-api==1.28.2 + # via + # karapace (/karapace/pyproject.toml) + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-instrumentation==0.49b2 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-asgi==0.49b2 + # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-fastapi==0.49b2 # via karapace (/karapace/pyproject.toml) +opentelemetry-sdk==1.28.2 + # via karapace (/karapace/pyproject.toml) +opentelemetry-semantic-conventions==0.49b2 + # via + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-sdk +opentelemetry-util-http==0.49b2 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi packaging==24.2 # via # aiokafka + # opentelemetry-instrumentation # pytest pdbpp==0.10.3 # via karapace (/karapace/pyproject.toml) @@ -142,11 +209,15 @@ psutil==6.1.0 # karapace (/karapace/pyproject.toml) # locust # pytest-xdist +pydantic==1.10.17 + # via + # fastapi + # karapace (/karapace/pyproject.toml) pygments==2.18.0 # via # pdbpp # rich -pyjwt==2.9.0 +pyjwt==2.10.0 # via karapace (/karapace/pyproject.toml) pyrepl==0.9.0 # via fancycompleter @@ -167,8 +238,14 @@ pytest-xdist[psutil]==3.6.1 # via karapace (/karapace/pyproject.toml) python-dateutil==2.9.0.post0 # via karapace (/karapace/pyproject.toml) +python-dotenv==1.0.1 + # via uvicorn +python-multipart==0.0.17 + # via fastapi python-snappy==0.7.3 # via karapace (/karapace/pyproject.toml) +pyyaml==6.0.2 + # via uvicorn pyzmq==26.2.0 # via locust referencing==0.35.1 @@ -180,19 +257,29 @@ requests==2.32.3 # karapace (/karapace/pyproject.toml) # locust rich==13.7.1 - # via karapace (/karapace/pyproject.toml) + # via + # karapace (/karapace/pyproject.toml) + # typer rpds-py==0.21.0 # via # jsonschema # referencing -sentry-sdk==2.18.0 +sentry-sdk==2.19.0 # via karapace (/karapace/pyproject.toml) +shellingham==1.5.4 + # via typer six==1.16.0 - # via python-dateutil + # via + # dependency-injector + # python-dateutil sniffio==1.3.1 - # via anyio + # via + # anyio + # httpx sortedcontainers==2.4.0 # via hypothesis +starlette==0.41.3 + # via fastapi tenacity==9.0.0 # via karapace (/karapace/pyproject.toml) tomli==2.1.0 @@ -200,12 +287,20 @@ tomli==2.1.0 # coverage # locust # pytest +typer==0.13.1 + # via fastapi-cli typing-extensions==4.12.2 # via # anyio + # asgiref + # fastapi # karapace (/karapace/pyproject.toml) # locust # multidict + # opentelemetry-sdk + # pydantic + # typer + # uvicorn ujson==5.10.0 # via karapace (/karapace/pyproject.toml) urllib3==2.2.3 @@ -213,8 +308,18 @@ urllib3==2.2.3 # geventhttpclient # requests # sentry-sdk +uvicorn[standard]==0.32.1 + # via + # fastapi + # fastapi-cli +uvloop==0.21.0 + # via uvicorn watchfiles==0.24.0 - # via karapace (/karapace/pyproject.toml) + # via + # karapace (/karapace/pyproject.toml) + # uvicorn +websockets==14.1 + # via uvicorn werkzeug==3.1.3 # via # flask @@ -222,6 +327,10 @@ werkzeug==3.1.3 # locust wmctrl==0.5 # via pdbpp +wrapt==1.17.0 + # via + # deprecated + # opentelemetry-instrumentation xxhash==3.5.0 # via karapace (/karapace/pyproject.toml) yarl==1.12.1 diff --git a/requirements/requirements-typing.txt b/requirements/requirements-typing.txt index aef63ee86..beab6babb 100644 --- a/requirements/requirements-typing.txt +++ b/requirements/requirements-typing.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make pin-requirements @@ -8,15 +8,20 @@ accept-types==0.4.1 # via karapace (/karapace/pyproject.toml) aiohappyeyeballs==2.4.3 # via aiohttp -aiohttp==3.10.10 +aiohttp==3.10.11 # via karapace (/karapace/pyproject.toml) aiokafka==0.10.0 # via karapace (/karapace/pyproject.toml) aiosignal==1.3.1 # via aiohttp anyio==4.6.2.post1 - # via watchfiles -async-timeout==4.0.3 + # via + # httpx + # starlette + # watchfiles +asgiref==3.8.1 + # via opentelemetry-instrumentation-asgi +async-timeout==5.0.1 # via # aiohttp # aiokafka @@ -30,23 +35,60 @@ avro @ https://github.com/aiven/avro/archive/5a82d57f2a650fd87c819a30e433f1abb2c cachetools==5.3.3 # via karapace (/karapace/pyproject.toml) certifi==2024.8.30 - # via sentry-sdk + # via + # httpcore + # httpx + # sentry-sdk +click==8.1.7 + # via + # typer + # uvicorn confluent-kafka==2.4.0 # via karapace (/karapace/pyproject.toml) cramjam==2.9.0 # via python-snappy +dependency-injector==4.43.0 + # via karapace (/karapace/pyproject.toml) +deprecated==1.2.15 + # via + # opentelemetry-api + # opentelemetry-semantic-conventions +dnspython==2.7.0 + # via email-validator +email-validator==2.2.0 + # via fastapi exceptiongroup==1.2.2 # via anyio +fastapi[standard]==0.115.5 + # via karapace (/karapace/pyproject.toml) +fastapi-cli[standard]==0.0.5 + # via fastapi frozenlist==1.5.0 # via # aiohttp # aiosignal +h11==0.14.0 + # via + # httpcore + # uvicorn +httpcore==1.0.7 + # via httpx +httptools==0.6.4 + # via uvicorn +httpx==0.27.2 + # via fastapi idna==3.10 # via # anyio + # email-validator + # httpx # yarl +importlib-metadata==8.5.0 + # via opentelemetry-api isodate==0.7.2 # via karapace (/karapace/pyproject.toml) +jinja2==3.1.4 + # via fastapi jsonschema==4.23.0 # via karapace (/karapace/pyproject.toml) jsonschema-specifications==2024.10.1 @@ -55,6 +97,8 @@ lz4==4.3.3 # via karapace (/karapace/pyproject.toml) markdown-it-py==3.0.0 # via rich +markupsafe==3.0.2 + # via jinja2 mdurl==0.1.2 # via markdown-it-py multidict==6.1.0 @@ -65,43 +109,95 @@ mypy==1.13.0 # via karapace (/karapace/pyproject.toml) mypy-extensions==1.0.0 # via mypy -networkx==3.2.1 +networkx==3.4.2 # via karapace (/karapace/pyproject.toml) +opentelemetry-api==1.28.2 + # via + # karapace (/karapace/pyproject.toml) + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-instrumentation==0.49b2 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-asgi==0.49b2 + # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-fastapi==0.49b2 + # via karapace (/karapace/pyproject.toml) +opentelemetry-sdk==1.28.2 + # via karapace (/karapace/pyproject.toml) +opentelemetry-semantic-conventions==0.49b2 + # via + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-sdk +opentelemetry-util-http==0.49b2 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi packaging==24.2 - # via aiokafka + # via + # aiokafka + # opentelemetry-instrumentation prometheus-client==0.20.0 # via karapace (/karapace/pyproject.toml) protobuf==3.20.3 # via karapace (/karapace/pyproject.toml) +pydantic==1.10.17 + # via + # fastapi + # karapace (/karapace/pyproject.toml) pygments==2.18.0 # via rich -pyjwt==2.9.0 +pyjwt==2.10.0 # via karapace (/karapace/pyproject.toml) python-dateutil==2.9.0.post0 # via karapace (/karapace/pyproject.toml) +python-dotenv==1.0.1 + # via uvicorn +python-multipart==0.0.17 + # via fastapi python-snappy==0.7.3 # via karapace (/karapace/pyproject.toml) +pyyaml==6.0.2 + # via uvicorn referencing==0.35.1 # via # jsonschema # jsonschema-specifications # types-jsonschema rich==13.7.1 - # via karapace (/karapace/pyproject.toml) + # via + # karapace (/karapace/pyproject.toml) + # typer rpds-py==0.21.0 # via # jsonschema # referencing -sentry-sdk==2.18.0 +sentry-sdk==2.19.0 # via karapace (/karapace/pyproject.toml) +shellingham==1.5.4 + # via typer six==1.16.0 - # via python-dateutil + # via + # dependency-injector + # python-dateutil sniffio==1.3.1 - # via anyio + # via + # anyio + # httpx +starlette==0.41.3 + # via fastapi tenacity==9.0.0 # via karapace (/karapace/pyproject.toml) tomli==2.1.0 # via mypy +typer==0.13.1 + # via fastapi-cli types-cachetools==5.5.0.20240820 # via karapace (/karapace/pyproject.toml) types-jsonschema==4.23.0.20240813 @@ -111,20 +207,42 @@ types-protobuf==3.20.4.6 typing-extensions==4.12.2 # via # anyio + # asgiref + # fastapi # karapace (/karapace/pyproject.toml) # multidict # mypy + # opentelemetry-sdk + # pydantic + # typer + # uvicorn ujson==5.10.0 # via karapace (/karapace/pyproject.toml) urllib3==2.2.3 # via sentry-sdk +uvicorn[standard]==0.32.1 + # via + # fastapi + # fastapi-cli +uvloop==0.21.0 + # via uvicorn watchfiles==0.24.0 - # via karapace (/karapace/pyproject.toml) + # via + # karapace (/karapace/pyproject.toml) + # uvicorn +websockets==14.1 + # via uvicorn +wrapt==1.17.0 + # via + # deprecated + # opentelemetry-instrumentation xxhash==3.5.0 # via karapace (/karapace/pyproject.toml) yarl==1.12.1 # via # aiohttp # karapace (/karapace/pyproject.toml) +zipp==3.21.0 + # via importlib-metadata zstandard==0.23.0 # via karapace (/karapace/pyproject.toml) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 5bb9cf22e..ccdc011c4 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make pin-requirements @@ -8,15 +8,20 @@ accept-types==0.4.1 # via karapace (/karapace/pyproject.toml) aiohappyeyeballs==2.4.3 # via aiohttp -aiohttp==3.10.10 +aiohttp==3.10.11 # via karapace (/karapace/pyproject.toml) aiokafka==0.10.0 # via karapace (/karapace/pyproject.toml) aiosignal==1.3.1 # via aiohttp anyio==4.6.2.post1 - # via watchfiles -async-timeout==4.0.3 + # via + # httpx + # starlette + # watchfiles +asgiref==3.8.1 + # via opentelemetry-instrumentation-asgi +async-timeout==5.0.1 # via # aiohttp # aiokafka @@ -29,22 +34,60 @@ avro @ https://github.com/aiven/avro/archive/5a82d57f2a650fd87c819a30e433f1abb2c # via karapace (/karapace/pyproject.toml) cachetools==5.3.3 # via karapace (/karapace/pyproject.toml) +certifi==2024.8.30 + # via + # httpcore + # httpx +click==8.1.7 + # via + # typer + # uvicorn confluent-kafka==2.4.0 # via karapace (/karapace/pyproject.toml) cramjam==2.9.0 # via python-snappy +dependency-injector==4.43.0 + # via karapace (/karapace/pyproject.toml) +deprecated==1.2.15 + # via + # opentelemetry-api + # opentelemetry-semantic-conventions +dnspython==2.7.0 + # via email-validator +email-validator==2.2.0 + # via fastapi exceptiongroup==1.2.2 # via anyio +fastapi[standard]==0.115.5 + # via karapace (/karapace/pyproject.toml) +fastapi-cli[standard]==0.0.5 + # via fastapi frozenlist==1.5.0 # via # aiohttp # aiosignal +h11==0.14.0 + # via + # httpcore + # uvicorn +httpcore==1.0.7 + # via httpx +httptools==0.6.4 + # via uvicorn +httpx==0.27.2 + # via fastapi idna==3.10 # via # anyio + # email-validator + # httpx # yarl +importlib-metadata==8.5.0 + # via opentelemetry-api isodate==0.7.2 # via karapace (/karapace/pyproject.toml) +jinja2==3.1.4 + # via fastapi jsonschema==4.23.0 # via karapace (/karapace/pyproject.toml) jsonschema-specifications==2024.10.1 @@ -53,58 +96,134 @@ lz4==4.3.3 # via karapace (/karapace/pyproject.toml) markdown-it-py==3.0.0 # via rich +markupsafe==3.0.2 + # via jinja2 mdurl==0.1.2 # via markdown-it-py multidict==6.1.0 # via # aiohttp # yarl -networkx==3.2.1 +networkx==3.4.2 # via karapace (/karapace/pyproject.toml) +opentelemetry-api==1.28.2 + # via + # karapace (/karapace/pyproject.toml) + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-instrumentation==0.49b2 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-asgi==0.49b2 + # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-fastapi==0.49b2 + # via karapace (/karapace/pyproject.toml) +opentelemetry-sdk==1.28.2 + # via karapace (/karapace/pyproject.toml) +opentelemetry-semantic-conventions==0.49b2 + # via + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-sdk +opentelemetry-util-http==0.49b2 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi packaging==24.2 - # via aiokafka + # via + # aiokafka + # opentelemetry-instrumentation prometheus-client==0.20.0 # via karapace (/karapace/pyproject.toml) protobuf==3.20.3 # via karapace (/karapace/pyproject.toml) +pydantic==1.10.17 + # via + # fastapi + # karapace (/karapace/pyproject.toml) pygments==2.18.0 # via rich -pyjwt==2.9.0 +pyjwt==2.10.0 # via karapace (/karapace/pyproject.toml) python-dateutil==2.9.0.post0 # via karapace (/karapace/pyproject.toml) +python-dotenv==1.0.1 + # via uvicorn +python-multipart==0.0.17 + # via fastapi python-snappy==0.7.3 # via karapace (/karapace/pyproject.toml) +pyyaml==6.0.2 + # via uvicorn referencing==0.35.1 # via # jsonschema # jsonschema-specifications rich==13.7.1 - # via karapace (/karapace/pyproject.toml) + # via + # karapace (/karapace/pyproject.toml) + # typer rpds-py==0.21.0 # via # jsonschema # referencing +shellingham==1.5.4 + # via typer six==1.16.0 - # via python-dateutil + # via + # dependency-injector + # python-dateutil sniffio==1.3.1 - # via anyio + # via + # anyio + # httpx +starlette==0.41.3 + # via fastapi tenacity==9.0.0 # via karapace (/karapace/pyproject.toml) +typer==0.13.1 + # via fastapi-cli typing-extensions==4.12.2 # via # anyio + # asgiref + # fastapi # karapace (/karapace/pyproject.toml) # multidict + # opentelemetry-sdk + # pydantic + # typer + # uvicorn ujson==5.10.0 # via karapace (/karapace/pyproject.toml) +uvicorn[standard]==0.32.1 + # via + # fastapi + # fastapi-cli +uvloop==0.21.0 + # via uvicorn watchfiles==0.24.0 - # via karapace (/karapace/pyproject.toml) + # via + # karapace (/karapace/pyproject.toml) + # uvicorn +websockets==14.1 + # via uvicorn +wrapt==1.17.0 + # via + # deprecated + # opentelemetry-instrumentation xxhash==3.5.0 # via karapace (/karapace/pyproject.toml) yarl==1.12.1 # via # aiohttp # karapace (/karapace/pyproject.toml) +zipp==3.21.0 + # via importlib-metadata zstandard==0.23.0 # via karapace (/karapace/pyproject.toml) diff --git a/src/karapace.egg-info/top_level.txt b/src/karapace.egg-info/top_level.txt new file mode 100644 index 000000000..a3a1d0ab2 --- /dev/null +++ b/src/karapace.egg-info/top_level.txt @@ -0,0 +1,3 @@ +karapace +protopacelib +schema_registry diff --git a/src/karapace/instrumentation/prometheus.py b/src/karapace/instrumentation/prometheus.py index 1336b4ab0..90d260057 100644 --- a/src/karapace/instrumentation/prometheus.py +++ b/src/karapace/instrumentation/prometheus.py @@ -22,6 +22,7 @@ class PrometheusInstrumentation: METRICS_ENDPOINT_PATH: Final[str] = "/metrics" + CONTENT_TYPE_LATEST: Final[str] = "text/plain; version=0.0.4; charset=utf-8" START_TIME_REQUEST_KEY: Final[str] = "start_time" registry: Final[CollectorRegistry] = CollectorRegistry() diff --git a/src/karapace/protobuf/io.py b/src/karapace/protobuf/io.py index 36c76e491..89cdd26f1 100644 --- a/src/karapace/protobuf/io.py +++ b/src/karapace/protobuf/io.py @@ -97,7 +97,7 @@ def get_protobuf_class_instance( class_name: str, cfg: Config, ) -> _ProtobufModel: - directory = Path(cfg["protobuf_runtime_directory"]) + directory = Path(cfg.protobuf_runtime_directory) deps_list = crawl_dependencies(schema) root_class_name = "" for value in deps_list.values(): diff --git a/src/schema_registry/schema_registry_apis.py b/src/schema_registry/schema_registry_apis.py index cc9a01bb2..13f6bb8f2 100644 --- a/src/schema_registry/schema_registry_apis.py +++ b/src/schema_registry/schema_registry_apis.py @@ -65,9 +65,6 @@ class KarapaceSchemaRegistryController: def __init__(self, config: Config, schema_registry: KarapaceSchemaRegistry, stats: StatsClient) -> None: # super().__init__(config=config, not_ready_handler=self._forward_if_not_ready_to_serve) - print("+++++++++========") - print(schema_registry) - self.config = config self._process_start_time = time.monotonic() self.stats = stats @@ -219,8 +216,6 @@ def _has_subject_with_id() -> bool: ) schema = self.schema_registry.schemas_get(parsed_schema_id, fetch_max_id=fetch_max_id) - print("+++++++++========") - print(schema) if not schema: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, diff --git a/src/schema_registry/user.py b/src/schema_registry/user.py index 16cd55705..101cb36bb 100644 --- a/src/schema_registry/user.py +++ b/src/schema_registry/user.py @@ -16,11 +16,6 @@ async def get_current_user( credentials: Annotated[HTTPBasicCredentials, Depends(HTTPBasic())], authorizer: AuthenticatorAndAuthorizer = Depends(Provide[SchemaRegistryContainer.karapace_container.authorizer]), ) -> User: - import logging - - logging.info("get_current_user ++++++++++++=============") - logging.info(f"credentials: {credentials}") - logging.info(f"authorizer: {authorizer}") if authorizer and not credentials: raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 000000000..f53be7121 --- /dev/null +++ b/tests/e2e/__init__.py @@ -0,0 +1,4 @@ +""" +Copyright (c) 2024 Aiven Ltd +See LICENSE for details +""" diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py new file mode 100644 index 000000000..050ced269 --- /dev/null +++ b/tests/e2e/conftest.py @@ -0,0 +1,131 @@ +""" +karapace - conftest + +Copyright (c) 2023 Aiven Ltd +See LICENSE for details +""" +from __future__ import annotations + +from _pytest.fixtures import SubRequest +from aiohttp import BasicAuth +from collections.abc import AsyncGenerator, Iterator +from confluent_kafka.admin import NewTopic +from karapace.client import Client +from karapace.config import KARAPACE_BASE_CONFIG_YAML_PATH +from karapace.container import KarapaceContainer +from karapace.kafka.admin import KafkaAdminClient +from karapace.kafka.consumer import AsyncKafkaConsumer, KafkaConsumer +from karapace.kafka.producer import AsyncKafkaProducer, KafkaProducer +from tests.integration.utils.cluster import RegistryDescription, RegistryEndpoint +from tests.integration.utils.kafka_server import KafkaServers + +import asyncio +import pytest +import secrets + + +@pytest.fixture(scope="session", name="basic_auth") +def fixture_basic_auth() -> BasicAuth: + return BasicAuth("test", "test") + + +@pytest.fixture(name="karapace_container", scope="session") +def fixture_karapace_container() -> KarapaceContainer: + container = KarapaceContainer() + container.base_config.from_yaml(KARAPACE_BASE_CONFIG_YAML_PATH, envs_required=True, required=True) + return container + + +@pytest.fixture(scope="session", name="kafka_servers") +def fixture_kafka_server(karapace_container: KarapaceContainer) -> Iterator[KafkaServers]: + yield KafkaServers([karapace_container.config().bootstrap_uri]) + + +@pytest.fixture(scope="function", name="producer") +def fixture_producer(kafka_servers: KafkaServers) -> Iterator[KafkaProducer]: + yield KafkaProducer(bootstrap_servers=kafka_servers.bootstrap_servers) + + +@pytest.fixture(scope="function", name="admin_client") +def fixture_admin(kafka_servers: KafkaServers) -> Iterator[KafkaAdminClient]: + yield KafkaAdminClient(bootstrap_servers=kafka_servers.bootstrap_servers) + + +@pytest.fixture(scope="function", name="consumer") +def fixture_consumer( + kafka_servers: KafkaServers, +) -> Iterator[KafkaConsumer]: + consumer = KafkaConsumer( + bootstrap_servers=kafka_servers.bootstrap_servers, + auto_offset_reset="earliest", + enable_auto_commit=False, + topic_metadata_refresh_interval_ms=200, # Speed things up for consumer tests to discover topics, etc. + ) + try: + yield consumer + finally: + consumer.close() + + +@pytest.fixture(scope="function", name="asyncproducer") +async def fixture_asyncproducer( + kafka_servers: KafkaServers, + loop: asyncio.AbstractEventLoop, +) -> AsyncGenerator[AsyncKafkaProducer, None]: + asyncproducer = AsyncKafkaProducer(bootstrap_servers=kafka_servers.bootstrap_servers, loop=loop) + await asyncproducer.start() + yield asyncproducer + await asyncproducer.stop() + + +@pytest.fixture(scope="function", name="asyncconsumer") +async def fixture_asyncconsumer( + kafka_servers: KafkaServers, + loop: asyncio.AbstractEventLoop, +) -> AsyncGenerator[AsyncKafkaConsumer, None]: + asyncconsumer = AsyncKafkaConsumer( + bootstrap_servers=kafka_servers.bootstrap_servers, + loop=loop, + auto_offset_reset="earliest", + enable_auto_commit=False, + topic_metadata_refresh_interval_ms=200, # Speed things up for consumer tests to discover topics, etc. + ) + await asyncconsumer.start() + yield asyncconsumer + await asyncconsumer.stop() + + +@pytest.fixture(scope="function", name="registry_cluster") +async def fixture_registry_cluster( + karapace_container: KarapaceContainer, + loop: asyncio.AbstractEventLoop, # pylint: disable=unused-argument +) -> RegistryDescription: + protocol = "http" + endpoint = RegistryEndpoint( + protocol, karapace_container.config().registry_host, karapace_container.config().registry_port + ) + return RegistryDescription(endpoint, karapace_container.config().topic_name) + + +@pytest.fixture(scope="function", name="registry_async_client") +async def fixture_registry_async_client( + request: SubRequest, + basic_auth: BasicAuth, + registry_cluster: RegistryDescription, + loop: asyncio.AbstractEventLoop, # pylint: disable=unused-argument +) -> AsyncGenerator[Client, None]: + client = Client( + server_uri=registry_cluster.endpoint.to_url(), + server_ca=request.config.getoption("server_ca"), + session_auth=basic_auth, + ) + try: + yield client + finally: + await client.close() + + +@pytest.fixture(scope="function", name="new_topic") +def topic_fixture(admin_client: KafkaAdminClient) -> NewTopic: + topic_name = secrets.token_hex(4) + return admin_client.new_topic(topic_name, num_partitions=1, replication_factor=1) diff --git a/tests/integration/instrumentation/__init__.py b/tests/e2e/instrumentation/__init__.py similarity index 100% rename from tests/integration/instrumentation/__init__.py rename to tests/e2e/instrumentation/__init__.py diff --git a/tests/integration/instrumentation/test_prometheus.py b/tests/e2e/instrumentation/test_prometheus.py similarity index 100% rename from tests/integration/instrumentation/test_prometheus.py rename to tests/e2e/instrumentation/test_prometheus.py diff --git a/tests/integration/kafka/__init__.py b/tests/e2e/kafka/__init__.py similarity index 100% rename from tests/integration/kafka/__init__.py rename to tests/e2e/kafka/__init__.py diff --git a/tests/integration/kafka/test_admin.py b/tests/e2e/kafka/test_admin.py similarity index 100% rename from tests/integration/kafka/test_admin.py rename to tests/e2e/kafka/test_admin.py diff --git a/tests/integration/kafka/test_consumer.py b/tests/e2e/kafka/test_consumer.py similarity index 100% rename from tests/integration/kafka/test_consumer.py rename to tests/e2e/kafka/test_consumer.py diff --git a/tests/integration/kafka/test_producer.py b/tests/e2e/kafka/test_producer.py similarity index 100% rename from tests/integration/kafka/test_producer.py rename to tests/e2e/kafka/test_producer.py diff --git a/tests/integration/schema_registry/__init__.py b/tests/e2e/schema_registry/__init__.py similarity index 100% rename from tests/integration/schema_registry/__init__.py rename to tests/e2e/schema_registry/__init__.py diff --git a/tests/integration/schema_registry/test_jsonschema.py b/tests/e2e/schema_registry/test_jsonschema.py similarity index 100% rename from tests/integration/schema_registry/test_jsonschema.py rename to tests/e2e/schema_registry/test_jsonschema.py diff --git a/tests/integration/test_karapace.py b/tests/e2e/test_karapace.py similarity index 100% rename from tests/integration/test_karapace.py rename to tests/e2e/test_karapace.py