From 82396f17c516048ad96eb0df098ded4c6132aa0f Mon Sep 17 00:00:00 2001 From: Emmanuel Evbuomwan Date: Wed, 27 Nov 2024 12:40:59 +0100 Subject: [PATCH] feat: e2e tests within docker - we split out e2e tests into own folder - e2e tests will run against the service in docker compose - we start by splitting out the prometheus and kafka e2e tests which do not need local running services --- .coveragerc | 6 +- .dockerignore | 1 - .github/workflows/container-smoke-test.yml | 20 +- .github/workflows/tests.yml | 65 +- .gitignore | 5 +- GNUmakefile | 31 +- ...-test-rest.sh => smoke-test-rest-proxy.sh} | 2 +- ...istry.sh => smoke-test-schema-registry.sh} | 3 +- container/Dockerfile | 4 - container/Dockerfile.dev | 59 +- container/compose.yml | 49 +- container/karapace.registry.env | 47 - container/karapace.rest.env | 51 - container/start.sh | 56 - karapace.config.env | 9 +- mypy.ini | 2 +- pyproject.toml | 7 +- requirements/requirements-dev.txt | 168 ++- requirements/requirements-typing.txt | 161 ++- requirements/requirements.txt | 156 ++- src/karapace/auth.py | 10 +- src/karapace/base_config.yaml | 3 - src/karapace/config.py | 21 +- src/karapace/container.py | 9 +- .../dependencies/config_dependency.py | 23 - .../dependencies/controller_dependency.py | 23 - .../dependencies/forward_client_dependency.py | 20 - .../schema_registry_dependency.py | 24 - src/karapace/dependencies/stats_dependeny.py | 23 - src/karapace/instrumentation/prometheus.py | 1 + .../kafka_rest_apis/consumer_manager.py | 2 +- src/karapace/karapace_all.py | 3 +- src/karapace/protobuf/io.py | 2 +- src/karapace/routers/compatibility_router.py | 33 - src/karapace/routers/config_router.py | 113 -- src/karapace/routers/errors.py | 56 - src/karapace/routers/health_router.py | 64 -- src/karapace/routers/mode_router.py | 42 - src/karapace/routers/requests.py | 101 -- src/karapace/routers/root_router.py | 16 - src/karapace/routers/schemas_router.py | 83 -- src/karapace/routers/subjects_router.py | 189 ---- src/karapace/schema_registry_apis.py | 982 ------------------ src/karapace/typing.py | 5 +- src/schema_registry/__main__.py | 4 +- src/schema_registry/routers/health.py | 8 +- .../routers/master_availability.py} | 20 +- src/schema_registry/routers/requests.py | 15 +- src/schema_registry/routers/setup.py | 2 + src/schema_registry/schema_registry_apis.py | 5 - src/schema_registry/user.py | 15 +- tests/conftest.py | 5 +- tests/e2e/__init__.py | 4 + tests/e2e/conftest.py | 128 +++ .../e2e/instrumentation}/__init__.py | 0 .../instrumentation/test_prometheus.py | 0 .../instrumentation => e2e/kafka}/__init__.py | 0 .../{integration => e2e}/kafka/test_admin.py | 0 .../kafka/test_consumer.py | 0 .../kafka/test_producer.py | 0 .../kafka => e2e/schema_registry}/__init__.py | 0 .../schema_registry/test_jsonschema.py | 0 tests/e2e/test_karapace.py | 64 ++ tests/integration/conftest.py | 14 +- tests/integration/schema_registry/__init__.py | 0 ...est_dependencies_compatibility_protobuf.py | 8 +- tests/integration/test_karapace.py | 54 - tests/integration/test_schema.py | 16 +- tests/integration/utils/cluster.py | 36 +- tests/integration/utils/zookeeper.py | 2 +- tests/utils.py | 8 +- 71 files changed, 880 insertions(+), 2278 deletions(-) rename bin/{smoke-test-rest.sh => smoke-test-rest-proxy.sh} (80%) rename bin/{smoke-test-registry.sh => smoke-test-schema-registry.sh} (84%) delete mode 100644 container/karapace.registry.env delete mode 100644 container/karapace.rest.env delete mode 100755 container/start.sh delete mode 100644 src/karapace/base_config.yaml delete mode 100644 src/karapace/dependencies/config_dependency.py delete mode 100644 src/karapace/dependencies/controller_dependency.py delete mode 100644 src/karapace/dependencies/forward_client_dependency.py delete mode 100644 src/karapace/dependencies/schema_registry_dependency.py delete mode 100644 src/karapace/dependencies/stats_dependeny.py delete mode 100644 src/karapace/routers/compatibility_router.py delete mode 100644 src/karapace/routers/config_router.py delete mode 100644 src/karapace/routers/errors.py delete mode 100644 src/karapace/routers/health_router.py delete mode 100644 src/karapace/routers/mode_router.py delete mode 100644 src/karapace/routers/requests.py delete mode 100644 src/karapace/routers/root_router.py delete mode 100644 src/karapace/routers/schemas_router.py delete mode 100644 src/karapace/routers/subjects_router.py delete mode 100644 src/karapace/schema_registry_apis.py rename src/{karapace/routers/master_available_router.py => schema_registry/routers/master_availability.py} (71%) create mode 100644 tests/e2e/__init__.py create mode 100644 tests/e2e/conftest.py rename {src/karapace/routers => tests/e2e/instrumentation}/__init__.py (100%) rename tests/{integration => e2e}/instrumentation/test_prometheus.py (100%) rename tests/{integration/instrumentation => e2e/kafka}/__init__.py (100%) rename tests/{integration => e2e}/kafka/test_admin.py (100%) rename tests/{integration => e2e}/kafka/test_consumer.py (100%) rename tests/{integration => e2e}/kafka/test_producer.py (100%) rename tests/{integration/kafka => e2e/schema_registry}/__init__.py (100%) rename tests/{integration => e2e}/schema_registry/test_jsonschema.py (100%) create mode 100644 tests/e2e/test_karapace.py delete mode 100644 tests/integration/schema_registry/__init__.py delete mode 100644 tests/integration/test_karapace.py diff --git a/.coveragerc b/.coveragerc index 2a6a5d055..b473a4c14 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,4 +1,8 @@ [run] branch = true relative_files = true -source = src/karapace +source = src +disable_warnings = module-not-measured, no-data-collected + +[report] +skip_empty = true diff --git a/.dockerignore b/.dockerignore index 57efb59ad..4b946a334 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,7 +10,6 @@ !LICENSE !pyproject.toml !setup.py -!container/start.sh !container/healthcheck.py # Ignore some files in source directories. diff --git a/.github/workflows/container-smoke-test.yml b/.github/workflows/container-smoke-test.yml index cced926bd..25cdc08c9 100644 --- a/.github/workflows/container-smoke-test.yml +++ b/.github/workflows/container-smoke-test.yml @@ -26,14 +26,22 @@ jobs: KARAPACE_VERSION=$(python -c "from karapace import version; print(version.__version__)") echo KARAPACE_VERSION=$KARAPACE_VERSION >> $GITHUB_ENV - - name: Build container - run: docker build --build-arg KARAPACE_VERSION=${{ env.KARAPACE_VERSION }} --file=container/Dockerfile . + - run: echo "RUNNER_UID=$(id -u)" >> $GITHUB_ENV + - run: echo "RUNNER_GID=$(id -g)" >> $GITHUB_ENV - name: Run container - run: docker compose --file=container/compose.yml up --build --wait --detach + run: make start-karapace-docker-resources + env: + KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }} + RUNNER_UID: ${{ env.RUNNER_UID }} + RUNNER_GID: ${{ env.RUNNER_GID }} - - name: Smoke test registry - run: bin/smoke-test-registry.sh + - name: Smoke test schema registry + run: bin/smoke-test-schema-registry.sh + env: + KARAPACE_PORT: 8081 - name: Smoke test REST proxy - run: bin/smoke-test-rest.sh + run: bin/smoke-test-rest-proxy.sh + env: + KARAPACE_PORT: 8082 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1ce27c993..60df28286 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,14 +14,13 @@ env: FORCE_COLOR: 1 PIP_PROGRESS_BAR: off PYTHONUNBUFFERED: 1 - KARAPACE_DOTENV: ${{ github.workspace }}/karapace.config.env jobs: - unit-tests: + tests: runs-on: ubuntu-latest strategy: matrix: - python-version: [ '3.10', '3.11', '3.12' ] + python-version: [ '3.9', '3.10', '3.11', '3.12' ] env: PYTEST_ADDOPTS: >- --log-dir=/tmp/ci-logs @@ -41,53 +40,35 @@ jobs: with: go-version: '1.21.0' - - name: Install requirements - run: make install - - name: Resolve Karapace version - run: | - source ./venv/bin/activate - KARAPACE_VERSION=$(python -c "from karapace import version; print(version.__version__)") - echo KARAPACE_VERSION=$KARAPACE_VERSION >> $GITHUB_ENV + run: echo KARAPACE_VERSION=4.1.1.dev44+gac20eeed.d20241205 >> $GITHUB_ENV - - name: Run containers - run: KARAPACE_VERSION=${{ env.KARAPACE_VERSION }} docker compose --file=container/compose.yml up --build --wait --detach + - run: echo "RUNNER_UID=$(id -u)" >> $GITHUB_ENV + - run: echo "RUNNER_GID=$(id -g)" >> $GITHUB_ENV - - run: make install-dev - run: make unit-tests-in-docker env: - COVERAGE_FILE: ".coverage.${{ matrix.python-version }}" - PYTEST_ARGS: "--cov=src --cov-append --numprocesses 4" - KARAPACE_VERSION=: ${{ env.KARAPACE_VERSION }} - - integration-tests: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [ '3.10', '3.11', '3.12' ] - env: - PYTEST_ADDOPTS: >- - --log-dir=/tmp/ci-logs - --log-file=/tmp/ci-logs/pytest.log - --showlocals - steps: - - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - cache: pip - python-version: ${{ matrix.python-version }} + KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }} + RUNNER_UID: ${{ env.RUNNER_UID }} + RUNNER_GID: ${{ env.RUNNER_GID }} + COVERAGE_FILE: "/opt/karapace/coverage/.coverage.${{ matrix.python-version }}" + PYTEST_ARGS: "--cov=karapace --cov=schema_registry --cov-append --numprocesses 4" - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: '1.21.0' + - run: make e2e-tests-in-docker + env: + KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }} + RUNNER_UID: ${{ env.RUNNER_UID }} + RUNNER_GID: ${{ env.RUNNER_GID }} + COVERAGE_FILE: "/opt/karapace/coverage/.coverage.${{ matrix.python-version }}" + PYTEST_ARGS: "--cov=karapace --cov=schema_registry --cov-append --numprocesses 4" - - run: make integration-tests + - run: make integration-tests-in-docker env: - COVERAGE_FILE: ".coverage.${{ matrix.python-version }}" - PYTEST_ARGS: "--cov=src --cov-append --random-order --numprocesses 4" + KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }} + RUNNER_UID: ${{ env.RUNNER_UID }} + RUNNER_GID: ${{ env.RUNNER_GID }} + COVERAGE_FILE: "/opt/karapace/coverage/.coverage.${{ matrix.python-version }}" + PYTEST_ARGS: "--cov=karapace --cov=schema_registry --cov-append --random-order --numprocesses 4" - name: Archive logs uses: actions/upload-artifact@v4 diff --git a/.gitignore b/.gitignore index 612ad46b2..59077267f 100644 --- a/.gitignore +++ b/.gitignore @@ -10,12 +10,13 @@ __pycache__/ /build/ /dist/ -/karapace.egg-info/ +src/karapace.egg-info/ /karapace-rpm-src.tar /kafka_*.tgz /kafka_*/ venv -/karapace/version.py +*.so +src/karapace/version.py .run .python-version .hypothesis/ diff --git a/GNUmakefile b/GNUmakefile index b8305d65a..9ad120cbb 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -5,7 +5,7 @@ PIP ?= pip3 --disable-pip-version-check --no-input --require-virtualenv PYTHON ?= python3 PYTHON_VERSION ?= 3.10 DOCKER_COMPOSE ?= docker compose -KARAPACE-CLI ?= $(DOCKER_COMPOSE) -f container/compose.yml run karapace-cli +KARAPACE-CLI ?= $(DOCKER_COMPOSE) -f container/compose.yml run --rm karapace-cli define PIN_VERSIONS_COMMAND pip install pip-tools && \ @@ -105,9 +105,36 @@ schema: pin-requirements: docker run -e CUSTOM_COMPILE_COMMAND='make pin-requirements' -it -v .:/karapace --security-opt label=disable python:$(PYTHON_VERSION)-bullseye /bin/bash -c "$(PIN_VERSIONS_COMMAND)" +.PHONY: start-karapace-docker-resources +start-karapace-docker-resources: export KARAPACE_VERSION ?= 4.1.1.dev44+gac20eeed.d20241205 +start-karapace-docker-resources: + sudo touch .coverage.3.9 .coverage.3.10 .coverage.3.11 .coverage.3.12 + sudo chown ${RUNNER_UID} .coverage.3.9 .coverage.3.10 .coverage.3.11 .coverage.3.12 + $(DOCKER_COMPOSE) -f container/compose.yml up -d --build --wait --detach + .PHONY: unit-tests-in-docker unit-tests-in-docker: export PYTEST_ARGS ?= -unit-tests-in-docker: +unit-tests-in-docker: start-karapace-docker-resources rm -fr runtime/* $(KARAPACE-CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/unit/ rm -fr runtime/* + +.PHONY: e2e-tests-in-docker +e2e-tests-in-docker: export PYTEST_ARGS ?= +e2e-tests-in-docker: start-karapace-docker-resources + rm -fr runtime/* + sleep 10 + $(KARAPACE-CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/e2e/test_karapace.py + rm -fr runtime/* + +.PHONY: integration-tests-in-docker +integration-tests-in-docker: export PYTEST_ARGS ?= +integration-tests-in-docker: start-karapace-docker-resources + rm -fr runtime/* + sleep 10 + $(KARAPACE-CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/integration/ + rm -fr runtime/* + +.PHONY: cli +cli: start-karapace-docker-resources + $(KARAPACE-CLI) bash diff --git a/bin/smoke-test-rest.sh b/bin/smoke-test-rest-proxy.sh similarity index 80% rename from bin/smoke-test-rest.sh rename to bin/smoke-test-rest-proxy.sh index 665ab08e5..58ea1b594 100755 --- a/bin/smoke-test-rest.sh +++ b/bin/smoke-test-rest-proxy.sh @@ -3,7 +3,7 @@ retries=5 for ((i = 0; i <= retries; i++)); do - response=$(curl --silent --verbose --fail http://localhost:8082/topics) + response=$(curl --silent --verbose --fail "http://localhost:$KARAPACE_PORT/topics") if [[ $response == '["_schemas","__consumer_offsets"]' ]]; then echo "Ok!" diff --git a/bin/smoke-test-registry.sh b/bin/smoke-test-schema-registry.sh similarity index 84% rename from bin/smoke-test-registry.sh rename to bin/smoke-test-schema-registry.sh index aab54ac76..0e3295dae 100755 --- a/bin/smoke-test-registry.sh +++ b/bin/smoke-test-schema-registry.sh @@ -6,9 +6,8 @@ for ((i = 0; i <= retries; i++)); do response=$( curl --silent --verbose --fail --request POST \ --header 'Content-Type: application/vnd.schemaregistry.v1+json' \ - --header 'Authorization: Basic Og==' \ --data '{"schema": "{\"type\": \"record\", \"name\": \"Obj\", \"fields\":[{\"name\": \"age\", \"type\": \"int\"}]}"}' \ - http://localhost:8081/subjects/test-key/versions + "http://localhost:$KARAPACE_PORT/subjects/test-key/versions" ) if [[ $response == '{"id":1}' ]]; then diff --git a/container/Dockerfile b/container/Dockerfile index 2e1544319..55ca06e1c 100644 --- a/container/Dockerfile +++ b/container/Dockerfile @@ -55,10 +55,6 @@ RUN apt-get update \ COPY --from=builder /venv /venv ENV PATH="/venv/bin:$PATH" -COPY ./container/start.sh /opt/karapace -RUN chmod 500 /opt/karapace/start.sh \ - && chown karapace:karapace /opt/karapace/start.sh - COPY ./container/healthcheck.py /opt/karapace WORKDIR /opt/karapace diff --git a/container/Dockerfile.dev b/container/Dockerfile.dev index 2ba1c0250..a2f380307 100644 --- a/container/Dockerfile.dev +++ b/container/Dockerfile.dev @@ -2,10 +2,17 @@ FROM python:3.10.11-bullseye AS builder ARG KARAPACE_VERSION +ARG RUNNER_UID +ARG RUNNER_GID + +# Setup files and directories. +RUN mkdir /opt/karapace /opt/karapace/runtime /var/log/karapace /opt/karapace/coverage \ +&& touch /opt/karapace/coverage/.coverage.3.9 /opt/karapace/coverage/.coverage.3.10 /opt/karapace/coverage/.coverage.3.11 /opt/karapace/coverage/.coverage.3.12 \ +&& chown --recursive "$RUNNER_UID:$RUNNER_GID" /opt/karapace /opt/karapace/coverage /var/log/karapace # Create, activate, and enforce usage of virtualenv. -RUN python3 -m venv /venv -ENV PATH="/venv/bin:$PATH" +RUN python3 -m venv /opt/karapace/venv +ENV PATH="/opt/karapace/venv/bin:$PATH" ENV PIP_REQUIRE_VIRTUALENV=true # Install golang needed by extensions @@ -15,36 +22,6 @@ RUN wget --progress=dot:giga "https://go.dev/dl/go${GO_VERSION}.linux-$(dpkg --p && tar -C /usr/local -xzf "go${GO_VERSION}.linux-$(dpkg --print-architecture).tar.gz" \ && rm "go${GO_VERSION}.linux-$(dpkg --print-architecture).tar.gz" -# Copy the requirements.txt and install dependencies in venv. Using a separate -# command to use layer caching. -# -# Note: the requirements.txt is pinned, if any of the dependencies is updated -# the cache will be invalidated and the image regenerated, which is the -# intended behavior. -COPY ./requirements/requirements.txt /build/ -COPY ./requirements/requirements-dev.txt /build/ -RUN --mount=type=cache,target=/root/.cache/pip \ - python3 -m pip install -r /build/requirements.txt -r /build/requirements-dev.txt - -COPY . /build/karapace-repo -WORKDIR /build/karapace-repo -RUN --mount=type=cache,target=/root/.cache/pip \ - if [ -z "${KARAPACE_VERSION}" ]; then \ - PRETEND_VERSION="$(python -c 'from src.karapace import version; print(version.__version__)')"; \ - else \ - PRETEND_VERSION=$KARAPACE_VERSION; \ - fi; \ - SETUPTOOLS_SCM_PRETEND_VERSION=$PRETEND_VERSION python3 -m pip install --no-deps . - -# Karapace image, i.e. production. -FROM python:3.10.11-slim-bullseye AS karapace - -# Setup user and directories. -RUN groupadd --system karapace \ - && useradd --system --gid karapace karapace \ - && mkdir /opt/karapace /opt/karapace/runtime /var/log/karapace \ - && chown --recursive karapace:karapace /opt/karapace /var/log/karapace - # Install protobuf compiler. ARG PROTOBUF_COMPILER_VERSION="3.12.4-1+deb11u1" RUN apt-get update \ @@ -52,11 +29,17 @@ RUN apt-get update \ protobuf-compiler=$PROTOBUF_COMPILER_VERSION \ && rm -rf /var/lib/apt/lists/* -# Copy virtualenv from builder and activate it. -COPY --from=builder /venv /venv -ENV PATH="/venv/bin:$PATH" - -COPY ./container/healthcheck.py /opt/karapace +# Install Java via openjdk-11 +COPY --from=openjdk:11 /usr/local/openjdk-11 /usr/local/openjdk-11 +ENV JAVA_HOME /usr/local/openjdk-11 +RUN update-alternatives --install /usr/bin/java java /usr/local/openjdk-11/bin/java 1 WORKDIR /opt/karapace -USER karapace + +COPY ./requirements /opt/karapace/requirements +RUN python3 -m pip install -r /opt/karapace/requirements/requirements.txt -r /opt/karapace/requirements/requirements-dev.txt + +COPY . . +RUN SETUPTOOLS_SCM_PRETEND_VERSION=$KARAPACE_VERSION python3 -m pip install . + +ENV PYTHONPATH="/opt/karapace/src:$PYTHONPATH" diff --git a/container/compose.yml b/container/compose.yml index 87106ee90..ed0e2a920 100644 --- a/container/compose.yml +++ b/container/compose.yml @@ -59,6 +59,8 @@ services: build: context: .. dockerfile: container/Dockerfile + args: + KARAPACE_VERSION: $KARAPACE_VERSION entrypoint: - python3 - -m @@ -67,17 +69,30 @@ services: - kafka ports: - 8081:8081 - volumes: - - ./karapace.registry.env:/opt/karapace/karapace.env environment: - KARAPACE_DOTENV: /opt/karapace/karapace.env + KARAPACE_KARAPACE_REGISTRY: true + KARAPACE_ADVERTISED_HOSTNAME: karapace-schema-registry + KARAPACE_BOOTSTRAP_URI: kafka:29092 KARAPACE_PORT: 8081 + KARAPACE_HOST: 0.0.0.0 + KARAPACE_CLIENT_ID: karapace-schema-registry + KARAPACE_GROUP_ID: karapace-schema-registry + KARAPACE_MASTER_ELIGIBILITY: true + KARAPACE_TOPIC_NAME: _schemas + KARAPACE_LOG_LEVEL: DEBUG + KARAPACE_COMPATIBILITY: FULL + KARAPACE_STATSD_HOST: statsd-exporter + KARAPACE_STATSD_PORT: 8125 + KARAPACE_KAFKA_SCHEMA_READER_STRICT_MODE: false + KARAPACE_KAFKA_RETRIABLE_ERRORS_SILENCED: true karapace-rest-proxy: image: ghcr.io/aiven-open/karapace:develop build: context: .. dockerfile: container/Dockerfile + args: + KARAPACE_VERSION: $KARAPACE_VERSION entrypoint: - python3 - -m @@ -87,27 +102,45 @@ services: - karapace-schema-registry ports: - 8082:8082 - volumes: - - ./karapace.rest.env:/opt/karapace/karapace.env environment: - KARAPACE_DOTENV: /opt/karapace/karapace.env + KARAPACE_KARAPACE_REST: true KARAPACE_PORT: 8082 + KARAPACE_HOST: 0.0.0.0 + KARAPACE_ADVERTISED_HOSTNAME: karapace-rest-proxy + KARAPACE_BOOTSTRAP_URI: kafka:29092 + KARAPACE_REGISTRY_HOST: karapace-schema-registry + KARAPACE_REGISTRY_PORT: 8081 + KARAPACE_ADMIN_METADATA_MAX_AGE: 0 + KARAPACE_LOG_LEVEL: DEBUG + KARAPACE_STATSD_HOST: statsd-exporter + KARAPACE_STATSD_PORT: 8125 + KARAPACE_KAFKA_SCHEMA_READER_STRICT_MODE: false + KARAPACE_KAFKA_RETRIABLE_ERRORS_SILENCED: true karapace-cli: image: ghcr.io/aiven-open/karapace:cli build: context: .. dockerfile: container/Dockerfile.dev + args: + KARAPACE_VERSION: $KARAPACE_VERSION + RUNNER_UID: $RUNNER_UID + RUNNER_GID: $RUNNER_GID tty: true depends_on: - kafka - karapace-schema-registry - karapace-rest-proxy volumes: + - ../.coverage.3.9:/opt/karapace/coverage/.coverage.3.9 + - ../.coverage.3.10:/opt/karapace/coverage/.coverage.3.10 + - ../.coverage.3.11:/opt/karapace/coverage/.coverage.3.11 + - ../.coverage.3.12:/opt/karapace/coverage/.coverage.3.12 + - ../.coveragerc:/opt/karapace/.coveragerc - ../tests:/opt/karapace/tests - - ../karapace.config.env:/opt/karapace/karapace.env environment: - KARAPACE_DOTENV: /opt/karapace/karapace.env + - COVERAGE_FILE + - COVERAGE_RCFILE=/opt/karapace/.coveragerc prometheus: image: prom/prometheus diff --git a/container/karapace.registry.env b/container/karapace.registry.env deleted file mode 100644 index cd757a99b..000000000 --- a/container/karapace.registry.env +++ /dev/null @@ -1,47 +0,0 @@ -KARAPACE_DOTENV=/opt/karapace/karapace.env -ACCESS_LOGS_DEBUG=False -ADVERTISED_HOSTNAME=karapace-schema-registry -ADVERTISED_PORT=8081 -ADVERTISED_PROTOCOL=http -BOOTSTRAP_URI=kafka:29092 -CLIENT_ID=karapace-schema-registry -COMPATIBILITY=BACKWARD -CONNECTIONS_MAX_IDLE_MS=15000 -CONSUMER_ENABLE_AUTO_COMMIT=True -CONSUMER_REQUEST_TIMEOUT_MS=11000 -CONSUMER_REQUEST_MAX_BYTES=67108864 -CONSUMER_IDLE_DISCONNECT_TIMEOUT=0 -FETCH_MIN_BYTES=1 -GROUP_ID=karapace-schema-registry -HOST=0.0.0.0 -PORT=8081 -REGISTRY_HOST=karapace-schema-registry -REGISTRY_PORT=8081 -REST_AUTHORIZATION=False -LOG_HANDLER=stdout -LOG_LEVEL=WARNING -LOG_FORMAT=%(asctime)s [%(threadName)s] %(filename)s:%(funcName)s:%(lineno)d %(message)s -MASTER_ELIGIBILITY=True -REPLICATION_FACTOR=1 -SECURITY_PROTOCOL=PLAINTEXT -SSL_CHECK_HOSTNAME=True -TOPIC_NAME=_schemas -METADATA_MAX_AGE_MS=60000 -ADMIN_METADATA_MAX_AGE=5 -PRODUCER_ACKS=1 -PRODUCER_COUNT=5 -PRODUCER_LINGER_MS=100 -PRODUCER_MAX_REQUEST_SIZE=1048576 -SESSION_TIMEOUT_MS=10000 -KARAPACE_REST=False -KARAPACE_REGISTRY=True -KARAPACE_PORT=8081 -NAME_STRATEGY=topic_name -NAME_STRATEGY_VALIDATION=True -MASTER_ELECTION_STRATEGY=lowest -PROTOBUF_RUNTIME_DIRECTORY=runtime -STATSD_HOST=statsd-exporter -STATSD_PORT=8125 -KAFKA_SCHEMA_READER_STRICT_MODE=False -KAFKA_RETRIABLE_ERRORS_SILENCED=True -USE_PROTOBUF_FORMATTER=False diff --git a/container/karapace.rest.env b/container/karapace.rest.env deleted file mode 100644 index 3df13f3b2..000000000 --- a/container/karapace.rest.env +++ /dev/null @@ -1,51 +0,0 @@ -KARAPACE_DOTENV=/opt/karapace/karapace.env -ACCESS_LOGS_DEBUG=False -# ACCESS_LOG_CLASS=karapace.utils.DebugAccessLogger -ACCESS_LOG_CLASS=aiohttp.web_log.AccessLogger -ADVERTISED_HOSTNAME=karapace-rest-proxy -ADVERTISED_PORT=8082 -ADVERTISED_PROTOCOL=http -BOOTSTRAP_URI=kafka:29092 -CLIENT_ID=karapace-rest-proxy -COMPATIBILITY=BACKWARD -CONNECTIONS_MAX_IDLE_MS=15000 -CONSUMER_ENABLE_AUTO_COMMIT=True -CONSUMER_REQUEST_TIMEOUT_MS=11000 -CONSUMER_REQUEST_MAX_BYTES=67108864 -CONSUMER_IDLE_DISCONNECT_TIMEOUT=0 -FETCH_MIN_BYTES=1 -GROUP_ID=karapace-rest-proxy -HOST=0.0.0.0 -PORT=8082 -REGISTRY_HOST=karapace-schema-registry -REGISTRY_PORT=8081 -REST_AUTHORIZATION=False -LOG_HANDLER=stdout -LOG_LEVEL=WARNING -LOG_FORMAT=%(asctime)s [%(threadName)s] %(filename)s:%(funcName)s:%(lineno)d %(message)s -MASTER_ELIGIBILITY=True -REPLICATION_FACTOR=1 -SECURITY_PROTOCOL=PLAINTEXT -SSL_CHECK_HOSTNAME=True -TOPIC_NAME=_schemas -METADATA_MAX_AGE_MS=60000 -ADMIN_METADATA_MAX_AGE=5 -PRODUCER_ACKS=1 -PRODUCER_COUNT=5 -PRODUCER_LINGER_MS=100 -PRODUCER_MAX_REQUEST_SIZE=1048576 -SESSION_TIMEOUT_MS=10000 -KARAPACE_REST=True -KARAPACE_REGISTRY=False -KARAPACE_PORT=8082 -NAME_STRATEGY=topic_name -NAME_STRATEGY_VALIDATION=True -MASTER_ELECTION_STRATEGY=lowest -PROTOBUF_RUNTIME_DIRECTORY=runtime -STATSD_HOST=statsd-exporter -STATSD_PORT=8125 -KAFKA_SCHEMA_READER_STRICT_MODE=False -KAFKA_RETRIABLE_ERRORS_SILENCED=True -USE_PROTOBUF_FORMATTER=False -HTTP_REQUEST_MAX_SIZE=1048576 -TAGS='{ "app": "karapace-rest-proxy" }' diff --git a/container/start.sh b/container/start.sh deleted file mode 100755 index 95ac86aa2..000000000 --- a/container/start.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -# Configuration is done using environment variables. The environment variable -# names are the same as the configuration keys, all letters in caps, and always -# start with `KARAPACE_`. - -# In the code below the expression ${var+isset} is used to check if the -# variable was defined, and ${var-isunset} if not. -# -# Ref: https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_06_02 - -case $1 in -rest) - # Reexport variables for compatibility - [[ -n ${KARAPACE_REST_ADVERTISED_HOSTNAME+isset} ]] && export KARAPACE_ADVERTISED_HOSTNAME="${KARAPACE_REST_ADVERTISED_HOSTNAME}" - [[ -n ${KARAPACE_REST_BOOTSTRAP_URI+isset} ]] && export KARAPACE_BOOTSTRAP_URI="${KARAPACE_REST_BOOTSTRAP_URI}" - [[ -n ${KARAPACE_REST_REGISTRY_HOST+isset} ]] && export KARAPACE_REGISTRY_HOST="${KARAPACE_REST_REGISTRY_HOST}" - [[ -n ${KARAPACE_REST_REGISTRY_PORT+isset} ]] && export KARAPACE_REGISTRY_PORT="${KARAPACE_REST_REGISTRY_PORT}" - [[ -n ${KARAPACE_REST_HOST+isset} ]] && export KARAPACE_HOST="${KARAPACE_REST_HOST}" - [[ -n ${KARAPACE_REST_PORT+isset} ]] && export KARAPACE_PORT="${KARAPACE_REST_PORT}" - [[ -n ${KARAPACE_REST_ADMIN_METADATA_MAX_AGE+isset} ]] && export KARAPACE_ADMIN_METADATA_MAX_AGE="${KARAPACE_REST_ADMIN_METADATA_MAX_AGE}" - [[ -n ${KARAPACE_REST_LOG_LEVEL+isset} ]] && export KARAPACE_LOG_LEVEL="${KARAPACE_REST_LOG_LEVEL}" - export KARAPACE_REST=1 - echo "{}" >/opt/karapace/rest.config.json - - echo "Starting Karapace REST API" - exec python3 -m karapace.karapace_all /opt/karapace/rest.config.json - ;; -registry) - # Reexport variables for compatibility - [[ -n ${KARAPACE_REGISTRY_ADVERTISED_HOSTNAME+isset} ]] && export KARAPACE_ADVERTISED_HOSTNAME="${KARAPACE_REGISTRY_ADVERTISED_HOSTNAME}" - [[ -n ${KARAPACE_REGISTRY_BOOTSTRAP_URI+isset} ]] && export KARAPACE_BOOTSTRAP_URI="${KARAPACE_REGISTRY_BOOTSTRAP_URI}" - [[ -n ${KARAPACE_REGISTRY_HOST+isset} ]] && export KARAPACE_HOST="${KARAPACE_REGISTRY_HOST}" - [[ -n ${KARAPACE_REGISTRY_PORT+isset} ]] && export KARAPACE_PORT="${KARAPACE_REGISTRY_PORT}" - [[ -n ${KARAPACE_REGISTRY_CLIENT_ID+isset} ]] && export KARAPACE_CLIENT_ID="${KARAPACE_REGISTRY_CLIENT_ID}" - [[ -n ${KARAPACE_REGISTRY_GROUP_ID+isset} ]] && export KARAPACE_GROUP_ID="${KARAPACE_REGISTRY_GROUP_ID}" - # Map misspelled environment variables to correct spelling for backwards compatibility. - [[ -n ${KARAPACE_REGISTRY_MASTER_ELIGIBITY+isset} ]] && export KARAPACE_MASTER_ELIGIBILITY="${KARAPACE_REGISTRY_MASTER_ELIGIBITY}" - [[ -n ${KARAPACE_REGISTRY_MASTER_ELIGIBILITY+isset} ]] && export KARAPACE_MASTER_ELIGIBILITY="${KARAPACE_REGISTRY_MASTER_ELIGIBILITY}" - [[ -n ${KARAPACE_REGISTRY_TOPIC_NAME+isset} ]] && export KARAPACE_TOPIC_NAME="${KARAPACE_REGISTRY_TOPIC_NAME}" - [[ -n ${KARAPACE_REGISTRY_COMPATIBILITY+isset} ]] && export KARAPACE_COMPATIBILITY="${KARAPACE_REGISTRY_COMPATIBILITY}" - [[ -n ${KARAPACE_REGISTRY_LOG_LEVEL+isset} ]] && export KARAPACE_LOG_LEVEL="${KARAPACE_REGISTRY_LOG_LEVEL}" - export KARAPACE_REGISTRY=1 - echo "{}" >/opt/karapace/registry.config.json - - echo "Starting Karapace Schema Registry" - exec python3 -m karapace.karapace_all /opt/karapace/registry.config.json - ;; -*) - echo "usage: start-karapace.sh " - exit 0 - ;; -esac - -wait diff --git a/karapace.config.env b/karapace.config.env index 70cf0c616..89cc63764 100644 --- a/karapace.config.env +++ b/karapace.config.env @@ -1,8 +1,9 @@ ACCESS_LOGS_DEBUG=False +ACCESS_LOG_CLASS=aiohttp.web_log.AccessLogger ADVERTISED_HOSTNAME=127.0.0.1 ADVERTISED_PORT=8081 ADVERTISED_PROTOCOL=http -BOOTSTRAP_URI=127.0.0.1:9092 +BOOTSTRAP_URI=kafka:29092 CLIENT_ID=sr-1 COMPATIBILITY=BACKWARD CONNECTIONS_MAX_IDLE_MS=15000 @@ -14,7 +15,7 @@ FETCH_MIN_BYTES=1 GROUP_ID=group_id8357e932 HOST=127.0.0.1 PORT=8081 -REGISTRY_HOST=127.0.0.1 +REGISTRY_HOST=karapace-schema-registry REGISTRY_PORT=8081 REST_AUTHORIZATION=False LOG_HANDLER=stdout @@ -38,8 +39,10 @@ NAME_STRATEGY=topic_name NAME_STRATEGY_VALIDATION=True MASTER_ELECTION_STRATEGY=lowest PROTOBUF_RUNTIME_DIRECTORY=runtime -STATSD_HOST=127.0.0.1 +STATSD_HOST=statsd-exporter STATSD_PORT=8125 KAFKA_SCHEMA_READER_STRICT_MODE=False KAFKA_RETRIABLE_ERRORS_SILENCED=True USE_PROTOBUF_FORMATTER=False +REST_BASE_URI=http://karapace-rest-proxy:8082 +TAGS='{ "app": "karapace" }' diff --git a/mypy.ini b/mypy.ini index c4ef8efd1..30d56b0bc 100644 --- a/mypy.ini +++ b/mypy.ini @@ -15,7 +15,7 @@ warn_no_return = True warn_unreachable = True strict_equality = True -[mypy-karapace.schema_registry_apis] +[mypy-schema_registry.schema_registry_apis] ignore_errors = True [mypy-karapace.compatibility.jsonschema.checks] diff --git a/pyproject.toml b/pyproject.toml index f1f9016cb..9df5adf5c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,8 @@ dependencies = [ "lz4", "networkx < 4", "protobuf < 4", - "pydantic == 1.10.17", + "pydantic == 2.10.2", + "pydantic-settings == 2.6.1", "pyjwt >= 2.4.0 , < 3", "python-dateutil < 3", "python-snappy", @@ -33,6 +34,10 @@ dependencies = [ "zstandard", "prometheus-client == 0.20.0", "yarl == 1.12.1", + "opentelemetry-api == 1.28.2", + "opentelemetry-sdk == 1.28.2", + "opentelemetry-instrumentation-fastapi == 0.49b2", + "dependency-injector == 4.43.0", # Patched dependencies # diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt index 9848f80e0..e5e7ceb19 100644 --- a/requirements/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -1,22 +1,29 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make pin-requirements # accept-types==0.4.1 # via karapace (/karapace/pyproject.toml) -aiohappyeyeballs==2.4.3 +aiohappyeyeballs==2.4.4 # via aiohttp -aiohttp==3.10.10 +aiohttp==3.10.11 # via karapace (/karapace/pyproject.toml) aiokafka==0.10.0 # via karapace (/karapace/pyproject.toml) aiosignal==1.3.1 # via aiohttp +annotated-types==0.7.0 + # via pydantic anyio==4.6.2.post1 - # via watchfiles -async-timeout==4.0.3 + # via + # httpx + # starlette + # watchfiles +asgiref==3.8.1 + # via opentelemetry-instrumentation-asgi +async-timeout==5.0.1 # via # aiohttp # aiokafka @@ -38,20 +45,36 @@ cachetools==5.3.3 certifi==2024.8.30 # via # geventhttpclient + # httpcore + # httpx # requests # sentry-sdk charset-normalizer==3.4.0 # via requests click==8.1.7 - # via flask + # via + # flask + # rich-toolkit + # typer + # uvicorn configargparse==1.7 # via locust confluent-kafka==2.4.0 # via karapace (/karapace/pyproject.toml) -coverage[toml]==7.6.4 +coverage[toml]==7.6.8 # via pytest-cov cramjam==2.9.0 # via python-snappy +dependency-injector==4.43.0 + # via karapace (/karapace/pyproject.toml) +deprecated==1.2.15 + # via + # opentelemetry-api + # opentelemetry-semantic-conventions +dnspython==2.7.0 + # via email-validator +email-validator==2.2.0 + # via fastapi exceptiongroup==1.2.2 # via # anyio @@ -61,9 +84,13 @@ execnet==2.1.1 # via pytest-xdist fancycompleter==0.9.1 # via pdbpp +fastapi[standard]==0.115.5 + # via karapace (/karapace/pyproject.toml) +fastapi-cli[standard]==0.0.6 + # via fastapi filelock==3.16.1 # via karapace (/karapace/pyproject.toml) -flask==3.0.3 +flask==3.1.0 # via # flask-cors # flask-login @@ -80,19 +107,31 @@ gevent==24.11.1 # via # geventhttpclient # locust -geventhttpclient==2.3.1 +geventhttpclient==2.3.3 # via locust greenlet==3.1.1 # via gevent -hypothesis==6.118.8 +h11==0.14.0 + # via + # httpcore + # uvicorn +httpcore==1.0.7 + # via httpx +httptools==0.6.4 + # via uvicorn +httpx==0.28.0 + # via fastapi +hypothesis==6.122.1 # via karapace (/karapace/pyproject.toml) idna==3.10 # via # anyio + # email-validator + # httpx # requests # yarl importlib-metadata==8.5.0 - # via flask + # via opentelemetry-api iniconfig==2.0.0 # via pytest isodate==0.7.2 @@ -100,12 +139,14 @@ isodate==0.7.2 itsdangerous==2.2.0 # via flask jinja2==3.1.4 - # via flask + # via + # fastapi + # flask jsonschema==4.23.0 # via karapace (/karapace/pyproject.toml) jsonschema-specifications==2024.10.1 # via jsonschema -locust==2.32.2 +locust==2.32.4 # via karapace (/karapace/pyproject.toml) lz4==4.3.3 # via karapace (/karapace/pyproject.toml) @@ -123,11 +164,40 @@ multidict==6.1.0 # via # aiohttp # yarl -networkx==3.2.1 +networkx==3.4.2 + # via karapace (/karapace/pyproject.toml) +opentelemetry-api==1.28.2 + # via + # karapace (/karapace/pyproject.toml) + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-instrumentation==0.49b2 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-asgi==0.49b2 + # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-fastapi==0.49b2 # via karapace (/karapace/pyproject.toml) +opentelemetry-sdk==1.28.2 + # via karapace (/karapace/pyproject.toml) +opentelemetry-semantic-conventions==0.49b2 + # via + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-sdk +opentelemetry-util-http==0.49b2 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi packaging==24.2 # via # aiokafka + # opentelemetry-instrumentation # pytest pdbpp==0.10.3 # via karapace (/karapace/pyproject.toml) @@ -142,15 +212,24 @@ psutil==6.1.0 # karapace (/karapace/pyproject.toml) # locust # pytest-xdist +pydantic==2.10.2 + # via + # fastapi + # karapace (/karapace/pyproject.toml) + # pydantic-settings +pydantic-core==2.27.1 + # via pydantic +pydantic-settings==2.6.1 + # via karapace (/karapace/pyproject.toml) pygments==2.18.0 # via # pdbpp # rich -pyjwt==2.9.0 +pyjwt==2.10.1 # via karapace (/karapace/pyproject.toml) pyrepl==0.9.0 # via fancycompleter -pytest==8.3.3 +pytest==8.3.4 # via # karapace (/karapace/pyproject.toml) # pytest-cov @@ -167,8 +246,16 @@ pytest-xdist[psutil]==3.6.1 # via karapace (/karapace/pyproject.toml) python-dateutil==2.9.0.post0 # via karapace (/karapace/pyproject.toml) +python-dotenv==1.0.1 + # via + # pydantic-settings + # uvicorn +python-multipart==0.0.19 + # via fastapi python-snappy==0.7.3 # via karapace (/karapace/pyproject.toml) +pyyaml==6.0.2 + # via uvicorn pyzmq==26.2.0 # via locust referencing==0.35.1 @@ -180,32 +267,53 @@ requests==2.32.3 # karapace (/karapace/pyproject.toml) # locust rich==13.7.1 - # via karapace (/karapace/pyproject.toml) -rpds-py==0.21.0 + # via + # karapace (/karapace/pyproject.toml) + # rich-toolkit + # typer +rich-toolkit==0.11.3 + # via fastapi-cli +rpds-py==0.22.3 # via # jsonschema # referencing -sentry-sdk==2.18.0 +sentry-sdk==2.19.0 # via karapace (/karapace/pyproject.toml) +shellingham==1.5.4 + # via typer six==1.16.0 - # via python-dateutil + # via + # dependency-injector + # python-dateutil sniffio==1.3.1 # via anyio sortedcontainers==2.4.0 # via hypothesis +starlette==0.41.3 + # via fastapi tenacity==9.0.0 # via karapace (/karapace/pyproject.toml) -tomli==2.1.0 +tomli==2.2.1 # via # coverage # locust # pytest +typer==0.15.1 + # via fastapi-cli typing-extensions==4.12.2 # via # anyio + # asgiref + # fastapi # karapace (/karapace/pyproject.toml) # locust # multidict + # opentelemetry-sdk + # pydantic + # pydantic-core + # rich-toolkit + # typer + # uvicorn ujson==5.10.0 # via karapace (/karapace/pyproject.toml) urllib3==2.2.3 @@ -213,8 +321,18 @@ urllib3==2.2.3 # geventhttpclient # requests # sentry-sdk +uvicorn[standard]==0.32.1 + # via + # fastapi + # fastapi-cli +uvloop==0.21.0 + # via uvicorn watchfiles==0.24.0 - # via karapace (/karapace/pyproject.toml) + # via + # karapace (/karapace/pyproject.toml) + # uvicorn +websockets==14.1 + # via uvicorn werkzeug==3.1.3 # via # flask @@ -222,6 +340,10 @@ werkzeug==3.1.3 # locust wmctrl==0.5 # via pdbpp +wrapt==1.17.0 + # via + # deprecated + # opentelemetry-instrumentation xxhash==3.5.0 # via karapace (/karapace/pyproject.toml) yarl==1.12.1 @@ -232,7 +354,7 @@ zipp==3.21.0 # via importlib-metadata zope-event==5.0 # via gevent -zope-interface==7.1.1 +zope-interface==7.2 # via gevent zstandard==0.23.0 # via karapace (/karapace/pyproject.toml) diff --git a/requirements/requirements-typing.txt b/requirements/requirements-typing.txt index aef63ee86..a27481c35 100644 --- a/requirements/requirements-typing.txt +++ b/requirements/requirements-typing.txt @@ -1,22 +1,29 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make pin-requirements # accept-types==0.4.1 # via karapace (/karapace/pyproject.toml) -aiohappyeyeballs==2.4.3 +aiohappyeyeballs==2.4.4 # via aiohttp -aiohttp==3.10.10 +aiohttp==3.10.11 # via karapace (/karapace/pyproject.toml) aiokafka==0.10.0 # via karapace (/karapace/pyproject.toml) aiosignal==1.3.1 # via aiohttp +annotated-types==0.7.0 + # via pydantic anyio==4.6.2.post1 - # via watchfiles -async-timeout==4.0.3 + # via + # httpx + # starlette + # watchfiles +asgiref==3.8.1 + # via opentelemetry-instrumentation-asgi +async-timeout==5.0.1 # via # aiohttp # aiokafka @@ -30,23 +37,61 @@ avro @ https://github.com/aiven/avro/archive/5a82d57f2a650fd87c819a30e433f1abb2c cachetools==5.3.3 # via karapace (/karapace/pyproject.toml) certifi==2024.8.30 - # via sentry-sdk + # via + # httpcore + # httpx + # sentry-sdk +click==8.1.7 + # via + # rich-toolkit + # typer + # uvicorn confluent-kafka==2.4.0 # via karapace (/karapace/pyproject.toml) cramjam==2.9.0 # via python-snappy +dependency-injector==4.43.0 + # via karapace (/karapace/pyproject.toml) +deprecated==1.2.15 + # via + # opentelemetry-api + # opentelemetry-semantic-conventions +dnspython==2.7.0 + # via email-validator +email-validator==2.2.0 + # via fastapi exceptiongroup==1.2.2 # via anyio +fastapi[standard]==0.115.5 + # via karapace (/karapace/pyproject.toml) +fastapi-cli[standard]==0.0.6 + # via fastapi frozenlist==1.5.0 # via # aiohttp # aiosignal +h11==0.14.0 + # via + # httpcore + # uvicorn +httpcore==1.0.7 + # via httpx +httptools==0.6.4 + # via uvicorn +httpx==0.28.0 + # via fastapi idna==3.10 # via # anyio + # email-validator + # httpx # yarl +importlib-metadata==8.5.0 + # via opentelemetry-api isodate==0.7.2 # via karapace (/karapace/pyproject.toml) +jinja2==3.1.4 + # via fastapi jsonschema==4.23.0 # via karapace (/karapace/pyproject.toml) jsonschema-specifications==2024.10.1 @@ -55,6 +100,8 @@ lz4==4.3.3 # via karapace (/karapace/pyproject.toml) markdown-it-py==3.0.0 # via rich +markupsafe==3.0.2 + # via jinja2 mdurl==0.1.2 # via markdown-it-py multidict==6.1.0 @@ -65,43 +112,103 @@ mypy==1.13.0 # via karapace (/karapace/pyproject.toml) mypy-extensions==1.0.0 # via mypy -networkx==3.2.1 +networkx==3.4.2 # via karapace (/karapace/pyproject.toml) +opentelemetry-api==1.28.2 + # via + # karapace (/karapace/pyproject.toml) + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-instrumentation==0.49b2 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-asgi==0.49b2 + # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-fastapi==0.49b2 + # via karapace (/karapace/pyproject.toml) +opentelemetry-sdk==1.28.2 + # via karapace (/karapace/pyproject.toml) +opentelemetry-semantic-conventions==0.49b2 + # via + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-sdk +opentelemetry-util-http==0.49b2 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi packaging==24.2 - # via aiokafka + # via + # aiokafka + # opentelemetry-instrumentation prometheus-client==0.20.0 # via karapace (/karapace/pyproject.toml) protobuf==3.20.3 # via karapace (/karapace/pyproject.toml) +pydantic==2.10.2 + # via + # fastapi + # karapace (/karapace/pyproject.toml) + # pydantic-settings +pydantic-core==2.27.1 + # via pydantic +pydantic-settings==2.6.1 + # via karapace (/karapace/pyproject.toml) pygments==2.18.0 # via rich -pyjwt==2.9.0 +pyjwt==2.10.1 # via karapace (/karapace/pyproject.toml) python-dateutil==2.9.0.post0 # via karapace (/karapace/pyproject.toml) +python-dotenv==1.0.1 + # via + # pydantic-settings + # uvicorn +python-multipart==0.0.19 + # via fastapi python-snappy==0.7.3 # via karapace (/karapace/pyproject.toml) +pyyaml==6.0.2 + # via uvicorn referencing==0.35.1 # via # jsonschema # jsonschema-specifications # types-jsonschema rich==13.7.1 - # via karapace (/karapace/pyproject.toml) -rpds-py==0.21.0 + # via + # karapace (/karapace/pyproject.toml) + # rich-toolkit + # typer +rich-toolkit==0.11.3 + # via fastapi-cli +rpds-py==0.22.3 # via # jsonschema # referencing -sentry-sdk==2.18.0 +sentry-sdk==2.19.0 # via karapace (/karapace/pyproject.toml) +shellingham==1.5.4 + # via typer six==1.16.0 - # via python-dateutil + # via + # dependency-injector + # python-dateutil sniffio==1.3.1 # via anyio +starlette==0.41.3 + # via fastapi tenacity==9.0.0 # via karapace (/karapace/pyproject.toml) -tomli==2.1.0 +tomli==2.2.1 # via mypy +typer==0.15.1 + # via fastapi-cli types-cachetools==5.5.0.20240820 # via karapace (/karapace/pyproject.toml) types-jsonschema==4.23.0.20240813 @@ -111,20 +218,44 @@ types-protobuf==3.20.4.6 typing-extensions==4.12.2 # via # anyio + # asgiref + # fastapi # karapace (/karapace/pyproject.toml) # multidict # mypy + # opentelemetry-sdk + # pydantic + # pydantic-core + # rich-toolkit + # typer + # uvicorn ujson==5.10.0 # via karapace (/karapace/pyproject.toml) urllib3==2.2.3 # via sentry-sdk +uvicorn[standard]==0.32.1 + # via + # fastapi + # fastapi-cli +uvloop==0.21.0 + # via uvicorn watchfiles==0.24.0 - # via karapace (/karapace/pyproject.toml) + # via + # karapace (/karapace/pyproject.toml) + # uvicorn +websockets==14.1 + # via uvicorn +wrapt==1.17.0 + # via + # deprecated + # opentelemetry-instrumentation xxhash==3.5.0 # via karapace (/karapace/pyproject.toml) yarl==1.12.1 # via # aiohttp # karapace (/karapace/pyproject.toml) +zipp==3.21.0 + # via importlib-metadata zstandard==0.23.0 # via karapace (/karapace/pyproject.toml) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 5bb9cf22e..009cf167d 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,22 +1,29 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make pin-requirements # accept-types==0.4.1 # via karapace (/karapace/pyproject.toml) -aiohappyeyeballs==2.4.3 +aiohappyeyeballs==2.4.4 # via aiohttp -aiohttp==3.10.10 +aiohttp==3.10.11 # via karapace (/karapace/pyproject.toml) aiokafka==0.10.0 # via karapace (/karapace/pyproject.toml) aiosignal==1.3.1 # via aiohttp +annotated-types==0.7.0 + # via pydantic anyio==4.6.2.post1 - # via watchfiles -async-timeout==4.0.3 + # via + # httpx + # starlette + # watchfiles +asgiref==3.8.1 + # via opentelemetry-instrumentation-asgi +async-timeout==5.0.1 # via # aiohttp # aiokafka @@ -29,22 +36,61 @@ avro @ https://github.com/aiven/avro/archive/5a82d57f2a650fd87c819a30e433f1abb2c # via karapace (/karapace/pyproject.toml) cachetools==5.3.3 # via karapace (/karapace/pyproject.toml) +certifi==2024.8.30 + # via + # httpcore + # httpx +click==8.1.7 + # via + # rich-toolkit + # typer + # uvicorn confluent-kafka==2.4.0 # via karapace (/karapace/pyproject.toml) cramjam==2.9.0 # via python-snappy +dependency-injector==4.43.0 + # via karapace (/karapace/pyproject.toml) +deprecated==1.2.15 + # via + # opentelemetry-api + # opentelemetry-semantic-conventions +dnspython==2.7.0 + # via email-validator +email-validator==2.2.0 + # via fastapi exceptiongroup==1.2.2 # via anyio +fastapi[standard]==0.115.5 + # via karapace (/karapace/pyproject.toml) +fastapi-cli[standard]==0.0.6 + # via fastapi frozenlist==1.5.0 # via # aiohttp # aiosignal +h11==0.14.0 + # via + # httpcore + # uvicorn +httpcore==1.0.7 + # via httpx +httptools==0.6.4 + # via uvicorn +httpx==0.28.0 + # via fastapi idna==3.10 # via # anyio + # email-validator + # httpx # yarl +importlib-metadata==8.5.0 + # via opentelemetry-api isodate==0.7.2 # via karapace (/karapace/pyproject.toml) +jinja2==3.1.4 + # via fastapi jsonschema==4.23.0 # via karapace (/karapace/pyproject.toml) jsonschema-specifications==2024.10.1 @@ -53,58 +99,144 @@ lz4==4.3.3 # via karapace (/karapace/pyproject.toml) markdown-it-py==3.0.0 # via rich +markupsafe==3.0.2 + # via jinja2 mdurl==0.1.2 # via markdown-it-py multidict==6.1.0 # via # aiohttp # yarl -networkx==3.2.1 +networkx==3.4.2 # via karapace (/karapace/pyproject.toml) +opentelemetry-api==1.28.2 + # via + # karapace (/karapace/pyproject.toml) + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-instrumentation==0.49b2 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-asgi==0.49b2 + # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-fastapi==0.49b2 + # via karapace (/karapace/pyproject.toml) +opentelemetry-sdk==1.28.2 + # via karapace (/karapace/pyproject.toml) +opentelemetry-semantic-conventions==0.49b2 + # via + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-sdk +opentelemetry-util-http==0.49b2 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi packaging==24.2 - # via aiokafka + # via + # aiokafka + # opentelemetry-instrumentation prometheus-client==0.20.0 # via karapace (/karapace/pyproject.toml) protobuf==3.20.3 # via karapace (/karapace/pyproject.toml) +pydantic==2.10.2 + # via + # fastapi + # karapace (/karapace/pyproject.toml) + # pydantic-settings +pydantic-core==2.27.1 + # via pydantic +pydantic-settings==2.6.1 + # via karapace (/karapace/pyproject.toml) pygments==2.18.0 # via rich -pyjwt==2.9.0 +pyjwt==2.10.1 # via karapace (/karapace/pyproject.toml) python-dateutil==2.9.0.post0 # via karapace (/karapace/pyproject.toml) +python-dotenv==1.0.1 + # via + # pydantic-settings + # uvicorn +python-multipart==0.0.19 + # via fastapi python-snappy==0.7.3 # via karapace (/karapace/pyproject.toml) +pyyaml==6.0.2 + # via uvicorn referencing==0.35.1 # via # jsonschema # jsonschema-specifications rich==13.7.1 - # via karapace (/karapace/pyproject.toml) -rpds-py==0.21.0 + # via + # karapace (/karapace/pyproject.toml) + # rich-toolkit + # typer +rich-toolkit==0.11.3 + # via fastapi-cli +rpds-py==0.22.3 # via # jsonschema # referencing +shellingham==1.5.4 + # via typer six==1.16.0 - # via python-dateutil + # via + # dependency-injector + # python-dateutil sniffio==1.3.1 # via anyio +starlette==0.41.3 + # via fastapi tenacity==9.0.0 # via karapace (/karapace/pyproject.toml) +typer==0.15.1 + # via fastapi-cli typing-extensions==4.12.2 # via # anyio + # asgiref + # fastapi # karapace (/karapace/pyproject.toml) # multidict + # opentelemetry-sdk + # pydantic + # pydantic-core + # rich-toolkit + # typer + # uvicorn ujson==5.10.0 # via karapace (/karapace/pyproject.toml) +uvicorn[standard]==0.32.1 + # via + # fastapi + # fastapi-cli +uvloop==0.21.0 + # via uvicorn watchfiles==0.24.0 - # via karapace (/karapace/pyproject.toml) + # via + # karapace (/karapace/pyproject.toml) + # uvicorn +websockets==14.1 + # via uvicorn +wrapt==1.17.0 + # via + # deprecated + # opentelemetry-instrumentation xxhash==3.5.0 # via karapace (/karapace/pyproject.toml) yarl==1.12.1 # via # aiohttp # karapace (/karapace/pyproject.toml) +zipp==3.21.0 + # via importlib-metadata zstandard==0.23.0 # via karapace (/karapace/pyproject.toml) diff --git a/src/karapace/auth.py b/src/karapace/auth.py index cfc566cdf..e801bfae9 100644 --- a/src/karapace/auth.py +++ b/src/karapace/auth.py @@ -11,7 +11,7 @@ from karapace.config import Config, InvalidConfiguration from karapace.statsd import StatsClient from karapace.utils import json_decode, json_encode -from typing import Protocol +from typing import Final, Protocol from typing_extensions import override, TypedDict from watchfiles import awatch, Change @@ -114,6 +114,8 @@ def check_authorization_any(self, user: User | None, operation: Operation, resou class AuthenticatorAndAuthorizer(AuthenticateProtocol, AuthorizeProtocol): + MUST_AUTHENTICATE: Final[bool] = True + async def close(self) -> None: ... @@ -122,6 +124,8 @@ async def start(self, stats: StatsClient) -> None: class NoAuthAndAuthz(AuthenticatorAndAuthorizer): + MUST_AUTHENTICATE: Final[bool] = False + @override def authenticate(self, *, username: str, password: str) -> User: return None @@ -205,9 +209,9 @@ def check_authorization_any(self, user: User | None, operation: Operation, resou class HTTPAuthorizer(ACLAuthorizer, AuthenticatorAndAuthorizer): - def __init__(self, config: Config) -> None: + def __init__(self, auth_file: str) -> None: super().__init__() - self._auth_filename: str = config.registry_authfile + self._auth_filename: str = auth_file self._auth_mtime: float = -1 self._refresh_auth_task: asyncio.Task | None = None self._refresh_auth_awatch_stop_event = asyncio.Event() diff --git a/src/karapace/base_config.yaml b/src/karapace/base_config.yaml deleted file mode 100644 index 5aa4d42d2..000000000 --- a/src/karapace/base_config.yaml +++ /dev/null @@ -1,3 +0,0 @@ -karapace: - env_file: ${KARAPACE_DOTENV} - env_file_encoding: utf-8 diff --git a/src/karapace/config.py b/src/karapace/config.py index d445b56a8..5a66ea84f 100644 --- a/src/karapace/config.py +++ b/src/karapace/config.py @@ -12,19 +12,14 @@ from karapace.typing import ElectionStrategy, NameStrategy from karapace.utils import json_encode from pathlib import Path -from pydantic import BaseModel, BaseSettings, PyObject -from typing import Final +from pydantic import BaseModel, ImportString +from pydantic_settings import BaseSettings, SettingsConfigDict import logging import os import socket import ssl -KARAPACE_ROOT: Final[Path] = Path(__file__).parent -KARAPACE_BASE_CONFIG_YAML_PATH: Final[Path] = KARAPACE_ROOT / "base_config.yaml" - -HOSTNAME = socket.gethostname() - HOSTNAME = socket.gethostname() @@ -33,12 +28,14 @@ class KarapaceTags(BaseModel): class Config(BaseSettings): + model_config = SettingsConfigDict(env_prefix="karapace_", env_ignore_empty=True) + access_logs_debug: bool = False - access_log_class: PyObject = "aiohttp.web_log.AccessLogger" + access_log_class: ImportString = "karapace.utils.DebugAccessLogger" advertised_hostname: str | None = None advertised_port: int | None = None advertised_protocol: str = "http" - bootstrap_uri: str = "127.0.0.1:9092" + bootstrap_uri: str = "kafka:29092" sasl_bootstrap_uri: str | None = None client_id: str = "sr-1" compatibility: str = "BACKWARD" @@ -55,7 +52,7 @@ class Config(BaseSettings): port: int = 8081 server_tls_certfile: str | None = None server_tls_keyfile: str | None = None - registry_host: str = "127.0.0.1" + registry_host: str = "karapace-schema-registry" registry_port: int = 8081 registry_user: str | None = None registry_password: str | None = None @@ -95,7 +92,7 @@ class Config(BaseSettings): name_strategy_validation: bool = True master_election_strategy: str = "lowest" protobuf_runtime_directory: str = "runtime" - statsd_host: str = "127.0.0.1" + statsd_host: str = "statsd-exporter" statsd_port: int = 8125 kafka_schema_reader_strict_mode: bool = False kafka_retriable_errors_silenced: bool = True @@ -137,7 +134,7 @@ def set_config_defaults(self, new_config: Mapping[str, str]) -> Config: # Fallback to default port if `advertised_port` is not set if config.advertised_port is None: - config.advertised_port = new_config["port"] + config.advertised_port = config.port # Fallback to `advertised_*` constructed URI if not set if config.rest_base_uri is None: diff --git a/src/karapace/container.py b/src/karapace/container.py index 7c71e99ca..951956bf2 100644 --- a/src/karapace/container.py +++ b/src/karapace/container.py @@ -13,18 +13,13 @@ class KarapaceContainer(containers.DeclarativeContainer): - base_config = providers.Configuration() - config = providers.Singleton( - Config, - _env_file=base_config.karapace.env_file, - _env_file_encoding=base_config.karapace.env_file_encoding, - ) + config = providers.Singleton(Config) statsd = providers.Singleton(StatsClient, config=config) no_auth_authorizer = providers.Singleton(NoAuthAndAuthz) - http_authorizer = providers.Singleton(HTTPAuthorizer, config=config) + http_authorizer = providers.Singleton(HTTPAuthorizer, auth_file=config().registry_authfile) schema_registry = providers.Singleton(KarapaceSchemaRegistry, config=config) diff --git a/src/karapace/dependencies/config_dependency.py b/src/karapace/dependencies/config_dependency.py deleted file mode 100644 index 9c299b725..000000000 --- a/src/karapace/dependencies/config_dependency.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - -from fastapi import Depends -from karapace.config import Config -from typing import Annotated - -import os - -env_file = os.environ.get("KARAPACE_DOTENV", None) - - -class ConfigDependencyManager: - CONFIG = Config(_env_file=env_file, _env_file_encoding="utf-8") - - @classmethod - def get_config(cls) -> Config: - return ConfigDependencyManager.CONFIG - - -ConfigDep = Annotated[Config, Depends(ConfigDependencyManager.get_config)] diff --git a/src/karapace/dependencies/controller_dependency.py b/src/karapace/dependencies/controller_dependency.py deleted file mode 100644 index e056b52c2..000000000 --- a/src/karapace/dependencies/controller_dependency.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - - -from fastapi import Depends -from karapace.dependencies.config_dependency import ConfigDep -from karapace.dependencies.schema_registry_dependency import SchemaRegistryDep -from karapace.dependencies.stats_dependeny import StatsDep -from karapace.schema_registry_apis import KarapaceSchemaRegistryController -from typing import Annotated - - -async def get_controller( - config: ConfigDep, - stats: StatsDep, - schema_registry: SchemaRegistryDep, -) -> KarapaceSchemaRegistryController: - return KarapaceSchemaRegistryController(config=config, schema_registry=schema_registry, stats=stats) - - -KarapaceSchemaRegistryControllerDep = Annotated[KarapaceSchemaRegistryController, Depends(get_controller)] diff --git a/src/karapace/dependencies/forward_client_dependency.py b/src/karapace/dependencies/forward_client_dependency.py deleted file mode 100644 index 57459c371..000000000 --- a/src/karapace/dependencies/forward_client_dependency.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - -from fastapi import Depends -from karapace.forward_client import ForwardClient -from typing import Annotated - -FORWARD_CLIENT: ForwardClient | None = None - - -def get_forward_client() -> ForwardClient: - global FORWARD_CLIENT - if not FORWARD_CLIENT: - FORWARD_CLIENT = ForwardClient() - return FORWARD_CLIENT - - -ForwardClientDep = Annotated[ForwardClient, Depends(get_forward_client)] diff --git a/src/karapace/dependencies/schema_registry_dependency.py b/src/karapace/dependencies/schema_registry_dependency.py deleted file mode 100644 index 68d9b0700..000000000 --- a/src/karapace/dependencies/schema_registry_dependency.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - -from fastapi import Depends -from karapace.dependencies.config_dependency import ConfigDependencyManager -from karapace.schema_registry import KarapaceSchemaRegistry -from typing import Annotated - - -class SchemaRegistryDependencyManager: - SCHEMA_REGISTRY: KarapaceSchemaRegistry | None = None - - @classmethod - async def get_schema_registry(cls) -> KarapaceSchemaRegistry: - if not SchemaRegistryDependencyManager.SCHEMA_REGISTRY: - SchemaRegistryDependencyManager.SCHEMA_REGISTRY = KarapaceSchemaRegistry( - config=ConfigDependencyManager.get_config() - ) - return SchemaRegistryDependencyManager.SCHEMA_REGISTRY - - -SchemaRegistryDep = Annotated[KarapaceSchemaRegistry, Depends(SchemaRegistryDependencyManager.get_schema_registry)] diff --git a/src/karapace/dependencies/stats_dependeny.py b/src/karapace/dependencies/stats_dependeny.py deleted file mode 100644 index 98c116dac..000000000 --- a/src/karapace/dependencies/stats_dependeny.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - - -from fastapi import Depends -from karapace.dependencies.config_dependency import ConfigDependencyManager -from karapace.statsd import StatsClient -from typing import Annotated - - -class StatsDependencyManager: - STATS_CLIENT: StatsClient | None = None - - @classmethod - def get_stats(cls) -> StatsClient: - if not StatsDependencyManager.STATS_CLIENT: - StatsDependencyManager.STATS_CLIENT = StatsClient(config=ConfigDependencyManager.get_config()) - return StatsDependencyManager.STATS_CLIENT - - -StatsDep = Annotated[StatsClient, Depends(StatsDependencyManager.get_stats)] diff --git a/src/karapace/instrumentation/prometheus.py b/src/karapace/instrumentation/prometheus.py index 1336b4ab0..90d260057 100644 --- a/src/karapace/instrumentation/prometheus.py +++ b/src/karapace/instrumentation/prometheus.py @@ -22,6 +22,7 @@ class PrometheusInstrumentation: METRICS_ENDPOINT_PATH: Final[str] = "/metrics" + CONTENT_TYPE_LATEST: Final[str] = "text/plain; version=0.0.4; charset=utf-8" START_TIME_REQUEST_KEY: Final[str] = "start_time" registry: Final[CollectorRegistry] = CollectorRegistry() diff --git a/src/karapace/kafka_rest_apis/consumer_manager.py b/src/karapace/kafka_rest_apis/consumer_manager.py index af94b5a82..277a0aca1 100644 --- a/src/karapace/kafka_rest_apis/consumer_manager.py +++ b/src/karapace/kafka_rest_apis/consumer_manager.py @@ -477,7 +477,7 @@ async def fetch(self, internal_name: tuple[str, str], content_type: str, formats timeout = ( int(query_params["timeout"]) if "timeout" in query_params - else consumer_config.consumer.request.timeout.ms + else consumer_config["consumer.request.timeout.ms"] ) # we get to be more in line with the confluent proxy by doing a bunch of fetches each time and # respecting the max fetch request size diff --git a/src/karapace/karapace_all.py b/src/karapace/karapace_all.py index 80e36cd43..8090216e9 100644 --- a/src/karapace/karapace_all.py +++ b/src/karapace/karapace_all.py @@ -6,7 +6,7 @@ from dependency_injector.wiring import inject, Provide from karapace import version as karapace_version -from karapace.config import Config, KARAPACE_BASE_CONFIG_YAML_PATH +from karapace.config import Config from karapace.container import KarapaceContainer from karapace.instrumentation.prometheus import PrometheusInstrumentation from karapace.kafka_rest_apis import KafkaRest @@ -42,6 +42,5 @@ def main( if __name__ == "__main__": container = KarapaceContainer() - container.base_config.from_yaml(KARAPACE_BASE_CONFIG_YAML_PATH, envs_required=True, required=True) container.wire(modules=[__name__]) sys.exit(main()) diff --git a/src/karapace/protobuf/io.py b/src/karapace/protobuf/io.py index 36c76e491..89cdd26f1 100644 --- a/src/karapace/protobuf/io.py +++ b/src/karapace/protobuf/io.py @@ -97,7 +97,7 @@ def get_protobuf_class_instance( class_name: str, cfg: Config, ) -> _ProtobufModel: - directory = Path(cfg["protobuf_runtime_directory"]) + directory = Path(cfg.protobuf_runtime_directory) deps_list = crawl_dependencies(schema) root_class_name = "" for value in deps_list.values(): diff --git a/src/karapace/routers/compatibility_router.py b/src/karapace/routers/compatibility_router.py deleted file mode 100644 index 0db406d2a..000000000 --- a/src/karapace/routers/compatibility_router.py +++ /dev/null @@ -1,33 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - -from fastapi import APIRouter -from karapace.auth.auth import Operation -from karapace.auth.dependencies import AuthenticatorAndAuthorizerDep, CurrentUserDep -from karapace.dependencies.controller_dependency import KarapaceSchemaRegistryControllerDep -from karapace.routers.errors import unauthorized -from karapace.routers.requests import CompatibilityCheckResponse, SchemaRequest -from karapace.typing import Subject - -compatibility_router = APIRouter( - prefix="/compatibility", - tags=["compatibility"], - responses={404: {"description": "Not found"}}, -) - - -@compatibility_router.post("/subjects/{subject}/versions/{version}", response_model_exclude_none=True) -async def compatibility_post( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, - version: str, # TODO support actual Version object - schema_request: SchemaRequest, -) -> CompatibilityCheckResponse: - if authorizer and not authorizer.check_authorization(user, Operation.Read, f"Subject:{subject}"): - raise unauthorized() - - return await controller.compatibility_check(subject=subject, schema_request=schema_request, version=version) diff --git a/src/karapace/routers/config_router.py b/src/karapace/routers/config_router.py deleted file mode 100644 index a83f24f60..000000000 --- a/src/karapace/routers/config_router.py +++ /dev/null @@ -1,113 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - -from fastapi import APIRouter, Request -from karapace.auth.auth import Operation -from karapace.auth.dependencies import AuthenticatorAndAuthorizerDep, CurrentUserDep -from karapace.dependencies.controller_dependency import KarapaceSchemaRegistryControllerDep -from karapace.dependencies.forward_client_dependency import ForwardClientDep -from karapace.dependencies.schema_registry_dependency import SchemaRegistryDep -from karapace.routers.errors import no_primary_url_error, unauthorized -from karapace.routers.requests import CompatibilityLevelResponse, CompatibilityRequest, CompatibilityResponse -from karapace.typing import Subject - -config_router = APIRouter( - prefix="/config", - tags=["config"], - responses={404: {"description": "Not found"}}, -) - - -@config_router.get("") -async def config_get( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, -) -> CompatibilityLevelResponse: - if authorizer and not authorizer.check_authorization(user, Operation.Read, "Config:"): - raise unauthorized() - - return await controller.config_get() - - -@config_router.put("") -async def config_put( - request: Request, - controller: KarapaceSchemaRegistryControllerDep, - schema_registry: SchemaRegistryDep, - forward_client: ForwardClientDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - compatibility_level_request: CompatibilityRequest, -) -> CompatibilityResponse: - if authorizer and not authorizer.check_authorization(user, Operation.Write, "Config:"): - raise unauthorized() - - i_am_primary, primary_url = await schema_registry.get_master() - if i_am_primary: - return await controller.config_set(compatibility_level_request=compatibility_level_request) - elif not primary_url: - raise no_primary_url_error() - else: - return await forward_client.forward_request_remote(request=request, primary_url=primary_url) - - -@config_router.get("/{subject}") -async def config_get_subject( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, - defaultToGlobal: bool = False, -) -> CompatibilityLevelResponse: - if authorizer and not authorizer.check_authorization(user, Operation.Read, f"Subject:{subject}"): - raise unauthorized() - - return await controller.config_subject_get(subject=subject, default_to_global=defaultToGlobal) - - -@config_router.put("/{subject}") -async def config_set_subject( - request: Request, - controller: KarapaceSchemaRegistryControllerDep, - schema_registry: SchemaRegistryDep, - forward_client: ForwardClientDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, - compatibility_level_request: CompatibilityRequest, -) -> CompatibilityResponse: - if authorizer and not authorizer.check_authorization(user, Operation.Write, f"Subject:{subject}"): - raise unauthorized() - - i_am_primary, primary_url = await schema_registry.get_master() - if i_am_primary: - return await controller.config_subject_set(subject=subject, compatibility_level_request=compatibility_level_request) - elif not primary_url: - raise no_primary_url_error() - else: - return await forward_client.forward_request_remote(request=request, primary_url=primary_url) - - -@config_router.delete("/{subject}") -async def config_delete_subject( - request: Request, - controller: KarapaceSchemaRegistryControllerDep, - schema_registry: SchemaRegistryDep, - forward_client: ForwardClientDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, -) -> CompatibilityResponse: - if authorizer and not authorizer.check_authorization(user, Operation.Write, f"Subject:{subject}"): - raise unauthorized() - - i_am_primary, primary_url = await schema_registry.get_master() - if i_am_primary: - return await controller.config_subject_delete(subject=subject) - elif not primary_url: - raise no_primary_url_error() - else: - return await forward_client.forward_request_remote(request=request, primary_url=primary_url) diff --git a/src/karapace/routers/errors.py b/src/karapace/routers/errors.py deleted file mode 100644 index a16c9797a..000000000 --- a/src/karapace/routers/errors.py +++ /dev/null @@ -1,56 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - -from enum import Enum, unique -from fastapi import HTTPException, status -from fastapi.exceptions import RequestValidationError - - -@unique -class SchemaErrorCodes(Enum): - HTTP_BAD_REQUEST = status.HTTP_400_BAD_REQUEST - HTTP_NOT_FOUND = status.HTTP_404_NOT_FOUND - HTTP_CONFLICT = status.HTTP_409_CONFLICT - HTTP_UNPROCESSABLE_ENTITY = status.HTTP_422_UNPROCESSABLE_ENTITY - HTTP_INTERNAL_SERVER_ERROR = status.HTTP_500_INTERNAL_SERVER_ERROR - SUBJECT_NOT_FOUND = 40401 - VERSION_NOT_FOUND = 40402 - SCHEMA_NOT_FOUND = 40403 - SUBJECT_SOFT_DELETED = 40404 - SUBJECT_NOT_SOFT_DELETED = 40405 - SCHEMAVERSION_SOFT_DELETED = 40406 - SCHEMAVERSION_NOT_SOFT_DELETED = 40407 - SUBJECT_LEVEL_COMPATIBILITY_NOT_CONFIGURED_ERROR_CODE = 40408 - INVALID_VERSION_ID = 42202 - INVALID_COMPATIBILITY_LEVEL = 42203 - INVALID_SCHEMA = 42201 - INVALID_SUBJECT = 42208 - SCHEMA_TOO_LARGE_ERROR_CODE = 42209 - REFERENCES_SUPPORT_NOT_IMPLEMENTED = 44302 - REFERENCE_EXISTS = 42206 - NO_MASTER_ERROR = 50003 - - -class KarapaceValidationError(RequestValidationError): - def __init__(self, error_code: int, error: str): - super().__init__(errors=[], body=error) - self.error_code = error_code - - -def no_primary_url_error() -> HTTPException: - return HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail={ - "error_code": SchemaErrorCodes.NO_MASTER_ERROR, - "message": "Error while forwarding the request to the master.", - }, - ) - - -def unauthorized() -> HTTPException: - return HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail={"message": "Forbidden"}, - ) diff --git a/src/karapace/routers/health_router.py b/src/karapace/routers/health_router.py deleted file mode 100644 index 950e08cfc..000000000 --- a/src/karapace/routers/health_router.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - -from fastapi import APIRouter, HTTPException, status -from karapace.dependencies.schema_registry_dependency import SchemaRegistryDep -from pydantic import BaseModel - - -class HealthStatus(BaseModel): - schema_registry_ready: bool - schema_registry_startup_time_sec: float - schema_registry_reader_current_offset: int - schema_registry_reader_highest_offset: int - schema_registry_is_primary: bool | None - schema_registry_is_primary_eligible: bool - schema_registry_primary_url: str | None - schema_registry_coordinator_running: bool - schema_registry_coordinator_generation_id: int - - -class HealthCheck(BaseModel): - status: HealthStatus - healthy: bool - - -health_router = APIRouter( - prefix="/_health", - tags=["health"], - responses={404: {"description": "Not found"}}, -) - - -@health_router.get("") -async def health( - schema_registry: SchemaRegistryDep, -) -> HealthCheck: - starttime = 0.0 - if schema_registry.schema_reader.ready(): - starttime = schema_registry.schema_reader.last_check - schema_registry.schema_reader.start_time - - cs = schema_registry.mc.get_coordinator_status() - - health_status = HealthStatus( - schema_registry_ready=schema_registry.schema_reader.ready(), - schema_registry_startup_time_sec=starttime, - schema_registry_reader_current_offset=schema_registry.schema_reader.offset, - schema_registry_reader_highest_offset=schema_registry.schema_reader.highest_offset(), - schema_registry_is_primary=cs.is_primary, - schema_registry_is_primary_eligible=cs.is_primary_eligible, - schema_registry_primary_url=cs.primary_url, - schema_registry_coordinator_running=cs.is_running, - schema_registry_coordinator_generation_id=cs.group_generation_id, - ) - # if self._auth is not None: - # resp["schema_registry_authfile_timestamp"] = self._auth.authfile_last_modified - - if not await schema_registry.schema_reader.is_healthy(): - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - ) - - return HealthCheck(status=health_status, healthy=True) diff --git a/src/karapace/routers/mode_router.py b/src/karapace/routers/mode_router.py deleted file mode 100644 index d8c98363a..000000000 --- a/src/karapace/routers/mode_router.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - -from fastapi import APIRouter -from karapace.auth.auth import Operation -from karapace.auth.dependencies import AuthenticatorAndAuthorizerDep, CurrentUserDep -from karapace.dependencies.controller_dependency import KarapaceSchemaRegistryControllerDep -from karapace.routers.errors import unauthorized -from karapace.typing import Subject - -mode_router = APIRouter( - prefix="/mode", - tags=["mode"], - responses={404: {"description": "Not found"}}, -) - - -@mode_router.get("") -async def mode_get( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, -): - if authorizer and not authorizer.check_authorization(user, Operation.Read, "Config:"): - raise unauthorized() - - return await controller.get_global_mode() - - -@mode_router.get("/{subject}") -async def mode_get_subject( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, -): - if authorizer and not authorizer.check_authorization(user, Operation.Read, f"Subject:{subject}"): - raise unauthorized() - - return await controller.get_subject_mode(subject=subject) diff --git a/src/karapace/routers/requests.py b/src/karapace/routers/requests.py deleted file mode 100644 index 8400f629d..000000000 --- a/src/karapace/routers/requests.py +++ /dev/null @@ -1,101 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - -from karapace.routers.errors import KarapaceValidationError -from karapace.schema_type import SchemaType -from karapace.typing import Subject -from pydantic import BaseModel, Field, validator -from typing import Any - - -class SchemaReference(BaseModel): - name: str - subject: Subject - version: int - - -class SchemaRequest(BaseModel): - schema_str: str = Field(alias="schema") - schema_type: SchemaType = Field(alias="schemaType", default=SchemaType.AVRO) - references: list[SchemaReference] | None = None - metadata: Any | None - ruleSet: Any | None - - class Config: - extra = "forbid" - - @validator("schema_str") - def validate_schema(cls, schema_str: str) -> str: - if not schema_str and not schema_str.strip(): - raise KarapaceValidationError( - error_code=42201, - error="Empty schema", - ) - return schema_str - - -class SchemaResponse(BaseModel): - subject: Subject - version: int - schema_id: int = Field(alias="id") - schema_str: str = Field(alias="schema") - schema_type: SchemaType | None = Field(alias="schemaType", default=None) - - -class SchemasResponse(BaseModel): - schema_str: str = Field(alias="schema") - subjects: list[Subject] | None = None - schema_type: SchemaType | None = Field(alias="schemaType", default=None) - references: list[Any] | None = None # TODO: typing - maxId: int | None = None - - -class SchemaListingItem(BaseModel): - subject: Subject - schema_str: str = Field(alias="schema") - version: int - schema_id: int = Field(alias="id") - schema_type: SchemaType | None = Field(alias="schemaType", default=None) - references: list[Any] | None - - -class SchemaIdResponse(BaseModel): - schema_id: int = Field(alias="id") - - -class CompatibilityRequest(BaseModel): - compatibility: str - - -class CompatibilityResponse(BaseModel): - compatibility: str - - -class CompatibilityLevelResponse(BaseModel): - compatibility_level: str = Field(alias="compatibilityLevel") - - -class CompatibilityCheckResponse(BaseModel): - is_compatible: bool - messages: list[str] | None = None - - -class ModeResponse(BaseModel): - mode: str - - -class SubjectVersion(BaseModel): - subject: Subject - version: int - - -class SubjectSchemaVersionResponse(BaseModel): - subject: Subject - version: int - schema_id: int = Field(alias="id") - schema_str: str = Field(alias="schema") - references: list[Any] | None = None - schema_type: SchemaType | None = Field(alias="schemaType", default=None) - compatibility: str | None = None diff --git a/src/karapace/routers/root_router.py b/src/karapace/routers/root_router.py deleted file mode 100644 index 6bec6cb9c..000000000 --- a/src/karapace/routers/root_router.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - -from fastapi import APIRouter - -root_router = APIRouter( - tags=["root"], - responses={404: {"description": "Not found"}}, -) - - -@root_router.get("/") -async def root() -> dict: - return {} diff --git a/src/karapace/routers/schemas_router.py b/src/karapace/routers/schemas_router.py deleted file mode 100644 index c06cd4a48..000000000 --- a/src/karapace/routers/schemas_router.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - -from fastapi import APIRouter -from karapace.auth.dependencies import AuthenticatorAndAuthorizerDep, CurrentUserDep -from karapace.dependencies.controller_dependency import KarapaceSchemaRegistryControllerDep -from karapace.routers.requests import SchemaListingItem, SchemasResponse, SubjectVersion - -schemas_router = APIRouter( - prefix="/schemas", - tags=["schemas"], - responses={404: {"description": "Not found"}}, -) - - -# TODO is this needed? Is this actually the ids/schema/id/schema?? -@schemas_router.get("") -async def schemas_get_list( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - deleted: bool = False, - latestOnly: bool = False, -) -> list[SchemaListingItem]: - return await controller.schemas_list( - deleted=deleted, - latest_only=latestOnly, - user=user, - authorizer=authorizer, - ) - - -@schemas_router.get("/ids/{schema_id}", response_model_exclude_none=True) -async def schemas_get( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - schema_id: str, # TODO: type to actual type - includeSubjects: bool = False, # TODO: include subjects? - fetchMaxId: bool = False, # TODO: fetch max id? - format: str = "", -) -> SchemasResponse: - return await controller.schemas_get( - schema_id=schema_id, - include_subjects=includeSubjects, - fetch_max_id=fetchMaxId, - format_serialized=format, - user=user, - authorizer=authorizer, - ) - - -# @schemas_router.get("/ids/{schema_id}/schema") -# async def schemas_get_only_id( -# controller: KarapaceSchemaRegistryControllerDep, -# ) -> SchemasResponse: -# # TODO retrieve by id only schema -# return await controller.schemas_get() - - -@schemas_router.get("/ids/{schema_id}/versions") -async def schemas_get_versions( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - schema_id: str, - deleted: bool = False, -) -> list[SubjectVersion]: - return await controller.schemas_get_versions( - schema_id=schema_id, - deleted=deleted, - user=user, - authorizer=authorizer, - ) - - -@schemas_router.get("/types") -async def schemas_get_subjects_list( - controller: KarapaceSchemaRegistryControllerDep, -) -> list[str]: - return await controller.schemas_types() diff --git a/src/karapace/routers/subjects_router.py b/src/karapace/routers/subjects_router.py deleted file mode 100644 index 9bde67743..000000000 --- a/src/karapace/routers/subjects_router.py +++ /dev/null @@ -1,189 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - -from fastapi import APIRouter, Request -from karapace.auth.auth import Operation -from karapace.auth.dependencies import AuthenticatorAndAuthorizerDep, CurrentUserDep -from karapace.dependencies.controller_dependency import KarapaceSchemaRegistryControllerDep -from karapace.dependencies.forward_client_dependency import ForwardClientDep -from karapace.dependencies.schema_registry_dependency import SchemaRegistryDep -from karapace.routers.errors import no_primary_url_error, unauthorized -from karapace.routers.requests import SchemaIdResponse, SchemaRequest, SchemaResponse, SubjectSchemaVersionResponse -from karapace.typing import Subject - -import logging - -LOG = logging.getLogger(__name__) - - -subjects_router = APIRouter( - prefix="/subjects", - tags=["subjects"], - responses={404: {"description": "Not found"}}, -) - - -@subjects_router.get("") -async def subjects_get( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - deleted: bool = False, -) -> list[str]: - return await controller.subjects_list( - deleted=deleted, - user=user, - authorizer=authorizer, - ) - - -@subjects_router.post("/{subject}", response_model_exclude_none=True) -async def subjects_subject_post( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, - schema_request: SchemaRequest, - deleted: bool = False, - normalize: bool = False, -) -> SchemaResponse: - if authorizer and not authorizer.check_authorization(user, Operation.Read, f"Subject:{subject}"): - raise unauthorized() - - return await controller.subjects_schema_post( - subject=subject, - schema_request=schema_request, - deleted=deleted, - normalize=normalize, - ) - - -@subjects_router.delete("/{subject}") -async def subjects_subject_delete( - request: Request, - controller: KarapaceSchemaRegistryControllerDep, - schema_registry: SchemaRegistryDep, - forward_client: ForwardClientDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, - permanent: bool = False, -) -> list[int]: - if authorizer and not authorizer.check_authorization(user, Operation.Write, f"Subject:{subject}"): - raise unauthorized() - - i_am_primary, primary_url = await schema_registry.get_master() - if i_am_primary: - return await controller.subject_delete(subject=subject, permanent=permanent) - elif not primary_url: - raise no_primary_url_error() - else: - return await forward_client.forward_request_remote(request=request, primary_url=primary_url) - - -@subjects_router.post("/{subject}/versions") -async def subjects_subject_versions_post( - request: Request, - controller: KarapaceSchemaRegistryControllerDep, - forward_client: ForwardClientDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, - schema_request: SchemaRequest, - normalize: bool = False, -) -> SchemaIdResponse: - if authorizer and not authorizer.check_authorization(user, Operation.Write, f"Subject:{subject}"): - raise unauthorized() - - # TODO: split the functionality so primary error and forwarding can be handled here - # and local/primary write is in controller. - return await controller.subject_post( - subject=subject, - schema_request=schema_request, - normalize=normalize, - forward_client=forward_client, - request=request, - ) - - -@subjects_router.get("/{subject}/versions") -async def subjects_subject_versions_list( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, - deleted: bool = False, -) -> list[int]: - if authorizer and not authorizer.check_authorization(user, Operation.Read, f"Subject:{subject}"): - raise unauthorized() - - return await controller.subject_versions_list(subject=subject, deleted=deleted) - - -@subjects_router.get("/{subject}/versions/{version}", response_model_exclude_none=True) -async def subjects_subject_version_get( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, - version: str, - deleted: bool = False, -) -> SubjectSchemaVersionResponse: - if authorizer and not authorizer.check_authorization(user, Operation.Read, f"Subject:{subject}"): - raise unauthorized() - - return await controller.subject_version_get(subject=subject, version=version, deleted=deleted) - - -@subjects_router.delete("/{subject}/versions/{version}") -async def subjects_subject_version_delete( - request: Request, - controller: KarapaceSchemaRegistryControllerDep, - schema_registry: SchemaRegistryDep, - forward_client: ForwardClientDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, - version: str, - permanent: bool = False, -) -> int: - if authorizer and not authorizer.check_authorization(user, Operation.Write, f"Subject:{subject}"): - raise unauthorized() - - i_am_primary, primary_url = await schema_registry.get_master() - if i_am_primary: - return await controller.subject_version_delete(subject=subject, version=version, permanent=permanent) - elif not primary_url: - raise no_primary_url_error() - else: - return await forward_client.forward_request_remote(request=request, primary_url=primary_url) - - -@subjects_router.get("/{subject}/versions/{version}/schema") -async def subjects_subject_version_schema_get( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, - version: str, -) -> dict: - if authorizer and not authorizer.check_authorization(user, Operation.Read, f"Subject:{subject}"): - raise unauthorized() - - return await controller.subject_version_schema_get(subject=subject, version=version) - - -@subjects_router.get("/{subject}/versions/{version}/referencedby") -async def subjects_subject_version_referenced_by( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, - version: str, -) -> list[int]: - if authorizer and not authorizer.check_authorization(user, Operation.Read, f"Subject:{subject}"): - raise unauthorized() - - return await controller.subject_version_referencedby_get(subject=subject, version=version) diff --git a/src/karapace/schema_registry_apis.py b/src/karapace/schema_registry_apis.py deleted file mode 100644 index 44d8bd128..000000000 --- a/src/karapace/schema_registry_apis.py +++ /dev/null @@ -1,982 +0,0 @@ -""" -Copyright (c) 2023 Aiven Ltd -See LICENSE for details -""" -from __future__ import annotations - -from avro.errors import SchemaParseException -from enum import Enum, unique -from fastapi import HTTPException, Request, Response, status -from karapace.auth.auth import Operation, User -from karapace.auth.dependencies import AuthenticatorAndAuthorizerDep -from karapace.compatibility import CompatibilityModes -from karapace.compatibility.jsonschema.checks import is_incompatible -from karapace.compatibility.schema_compatibility import SchemaCompatibility -from karapace.config import Config -from karapace.errors import ( - IncompatibleSchema, - InvalidReferences, - InvalidSchema, - InvalidSchemaType, - InvalidVersion, - ReferenceExistsException, - SchemasNotFoundException, - SchemaTooLargeException, - SchemaVersionNotSoftDeletedException, - SchemaVersionSoftDeletedException, - SubjectNotFoundException, - SubjectNotSoftDeletedException, - SubjectSoftDeletedException, - VersionNotFoundException, -) -from karapace.forward_client import ForwardClient -from karapace.protobuf.exception import ProtobufUnresolvedDependencyException -from karapace.routers.errors import no_primary_url_error -from karapace.routers.requests import ( - CompatibilityCheckResponse, - CompatibilityLevelResponse, - CompatibilityRequest, - CompatibilityResponse, - ModeResponse, - SchemaIdResponse, - SchemaListingItem, - SchemaRequest, - SchemaResponse, - SchemasResponse, - SubjectSchemaVersionResponse, - SubjectVersion, -) -from karapace.schema_models import ParsedTypedSchema, SchemaType, SchemaVersion, TypedSchema, ValidatedTypedSchema, Versioner -from karapace.schema_references import LatestVersionReference, Reference -from karapace.schema_registry import KarapaceSchemaRegistry -from karapace.statsd import StatsClient -from karapace.typing import JsonData, JsonObject, SchemaId, Subject, Version -from karapace.utils import JSONDecodeError -from typing import Any, cast - -import json -import logging -import time - -LOG = logging.getLogger(__name__) - - -# TODO Remove, already in router/errors -@unique -class SchemaErrorCodes(Enum): - HTTP_BAD_REQUEST = status.HTTP_400_BAD_REQUEST - HTTP_NOT_FOUND = status.HTTP_404_NOT_FOUND - HTTP_CONFLICT = status.HTTP_409_CONFLICT - HTTP_UNPROCESSABLE_ENTITY = status.HTTP_422_UNPROCESSABLE_ENTITY - HTTP_INTERNAL_SERVER_ERROR = status.HTTP_500_INTERNAL_SERVER_ERROR - SUBJECT_NOT_FOUND = 40401 - VERSION_NOT_FOUND = 40402 - SCHEMA_NOT_FOUND = 40403 - SUBJECT_SOFT_DELETED = 40404 - SUBJECT_NOT_SOFT_DELETED = 40405 - SCHEMAVERSION_SOFT_DELETED = 40406 - SCHEMAVERSION_NOT_SOFT_DELETED = 40407 - SUBJECT_LEVEL_COMPATIBILITY_NOT_CONFIGURED_ERROR_CODE = 40408 - INVALID_VERSION_ID = 42202 - INVALID_COMPATIBILITY_LEVEL = 42203 - INVALID_SCHEMA = 42201 - INVALID_SUBJECT = 42208 - SCHEMA_TOO_LARGE_ERROR_CODE = 42209 - REFERENCES_SUPPORT_NOT_IMPLEMENTED = 44302 - REFERENCE_EXISTS = 42206 - NO_MASTER_ERROR = 50003 - - -@unique -class SchemaErrorMessages(Enum): - SUBJECT_NOT_FOUND_FMT = "Subject '{subject}' not found." - INVALID_COMPATIBILITY_LEVEL = ( - "Invalid compatibility level. Valid values are none, backward, " - "forward, full, backward_transitive, forward_transitive, and " - "full_transitive" - ) - SUBJECT_LEVEL_COMPATIBILITY_NOT_CONFIGURED_FMT = ( - "Subject '{subject}' does not have subject-level compatibility configured" - ) - REFERENCES_SUPPORT_NOT_IMPLEMENTED = "Schema references are not supported for '{schema_type}' schema type" - - -class KarapaceSchemaRegistryController: - def __init__(self, config: Config, schema_registry: KarapaceSchemaRegistry, stats: StatsClient) -> None: - # super().__init__(config=config, not_ready_handler=self._forward_if_not_ready_to_serve) - - self.config = config - self._process_start_time = time.monotonic() - self.stats = stats - self.schema_registry = schema_registry - - def _add_schema_registry_routes(self) -> None: - pass - - def _subject_get(self, subject: Subject, include_deleted: bool = False) -> dict[Version, SchemaVersion]: - try: - schema_versions = self.schema_registry.subject_get(subject, include_deleted) - except SubjectNotFoundException: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.SUBJECT_NOT_FOUND.value, - "message": SchemaErrorMessages.SUBJECT_NOT_FOUND_FMT.value.format(subject=subject), - }, - ) - except SchemasNotFoundException: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.SUBJECT_NOT_FOUND.value, - "message": SchemaErrorMessages.SUBJECT_NOT_FOUND_FMT.value.format(subject=subject), - }, - ) - return schema_versions - - def _invalid_version(self, version: str | int) -> HTTPException: - """Shall be called when InvalidVersion is raised""" - return HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail={ - "error_code": SchemaErrorCodes.INVALID_VERSION_ID.value, - "message": ( - f"The specified version '{version}' is not a valid version id. " - 'Allowed values are between [1, 2^31-1] and the string "latest"' - ), - }, - ) - - async def compatibility_check( - self, - *, - subject: Subject, - schema_request: SchemaRequest, - version: str, - ) -> CompatibilityCheckResponse: - """Check for schema compatibility""" - try: - compatibility_mode = self.schema_registry.get_compatibility_mode(subject=subject) - except ValueError as ex: - # Using INTERNAL_SERVER_ERROR because the subject and configuration - # should have been validated before. - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail={ - "error_code": SchemaErrorCodes.HTTP_INTERNAL_SERVER_ERROR.value, - "message": str(ex), - }, - ) - - new_schema = self.get_new_schema(schema_request=schema_request) - old_schema = self.get_old_schema(subject, Versioner.V(version)) # , content_type) - if compatibility_mode.is_transitive(): - # Ignore the schema version provided in the rest api call (`version`) - # Instead check against all previous versions (including `version` if existing) - result = self.schema_registry.check_schema_compatibility(new_schema, subject) - else: - # Check against the schema version provided in the rest api call (`version`) - result = SchemaCompatibility.check_compatibility(old_schema, new_schema, compatibility_mode) - - if is_incompatible(result): - return CompatibilityCheckResponse(is_compatible=False, messages=list(result.messages)) - return CompatibilityCheckResponse(is_compatible=True) - - async def schemas_list( - self, - *, - deleted: bool, - latest_only: bool, - user: User | None, - authorizer: AuthenticatorAndAuthorizerDep | None, - ) -> list[SchemaListingItem]: - schemas = await self.schema_registry.schemas_list(include_deleted=deleted, latest_only=latest_only) - response_schemas: list[SchemaListingItem] = [] - for subject, schema_versions in schemas.items(): - if authorizer and not authorizer.check_authorization(user, Operation.Read, f"Subject:{subject}"): - continue - for schema_version in schema_versions: - references: list[Any] | None = None - if schema_version.references: - references = [r.to_dict() for r in schema_version.references] - response_schemas.append( - SchemaListingItem( - subject=schema_version.subject, - schema=schema_version.schema.schema_str, - version=schema_version.version.value, - id=schema_version.schema_id, - schemaType=schema_version.schema.schema_type, - references=references, - ) - ) - - return response_schemas - - async def schemas_get( - self, - *, - schema_id: str, - fetch_max_id: bool, - include_subjects: bool, - format_serialized: str, - user: User | None, - authorizer: AuthenticatorAndAuthorizerDep, - ) -> SchemasResponse: - try: - parsed_schema_id = SchemaId(int(schema_id)) - except ValueError: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.HTTP_NOT_FOUND.value, - "message": "HTTP 404 Not Found", - }, - ) - - def _has_subject_with_id() -> bool: - # Fast path - if authorizer is None or authorizer.check_authorization(user, Operation.Read, "Subject:*"): - return True - - subjects = self.schema_registry.database.subjects_for_schema(schema_id=parsed_schema_id) - resources = [f"Subject:{subject}" for subject in subjects] - return authorizer.check_authorization_any(user=user, operation=Operation.Read, resources=resources) - - if authorizer: - has_subject = _has_subject_with_id() - if not has_subject: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.SCHEMA_NOT_FOUND.value, - "message": "Schema not found", - }, - ) - - schema = self.schema_registry.schemas_get(parsed_schema_id, fetch_max_id=fetch_max_id) - if not schema: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.SCHEMA_NOT_FOUND.value, - "message": "Schema not found", - }, - ) - - schema_str = schema.schema_str - if format_serialized and schema.schema_type == SchemaType.PROTOBUF: - parsed_schema = ParsedTypedSchema.parse(schema_type=schema.schema_type, schema_str=schema_str) - schema_str = parsed_schema.serialize() - - subjects: list[Subject] | None = None - schema_type: SchemaType | None = None - references: list[Any] | None = None # TODO: typing - maxId: int | None = None - - if include_subjects: - subjects = self.schema_registry.database.subjects_for_schema(parsed_schema_id) - if schema.schema_type is not SchemaType.AVRO: - schema_type = schema.schema_type - if schema.references: - references = [r.to_dict() for r in schema.references] - if fetch_max_id: - maxId = schema.max_id - - return SchemasResponse( - schema=schema_str, - subjects=subjects, - schemaType=schema_type, - references=references, - maxId=maxId, - ) - - async def schemas_get_versions( - self, - *, - schema_id: str, - deleted: bool, - user: User | None, - authorizer: AuthenticatorAndAuthorizerDep, - ) -> list[SubjectVersion]: - try: - schema_id_int = SchemaId(int(schema_id)) - except ValueError: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.HTTP_NOT_FOUND.value, - "message": "HTTP 404 Not Found", - }, - ) - - subject_versions = [] - for subject_version in self.schema_registry.get_subject_versions_for_schema(schema_id_int, include_deleted=deleted): - subject = subject_version["subject"] - if authorizer and not authorizer.check_authorization(user, Operation.Read, f"Subject:{subject}"): - continue - subject_versions.append( - # TODO correct typing - SubjectVersion( - subject=subject_version["subject"], - version=subject_version["version"].value, - ), - ) - return subject_versions - - async def schemas_types(self) -> list[str]: - return ["JSON", "AVRO", "PROTOBUF"] - - async def config_get(self) -> CompatibilityLevelResponse: - # Note: The format sent by the user differs from the return value, this - # is for compatibility reasons. - return CompatibilityLevelResponse(compatibilityLevel=self.schema_registry.schema_reader.config.compatibility) - - async def config_set( - self, - *, - compatibility_level_request: CompatibilityRequest, - ) -> CompatibilityResponse: - try: - compatibility_level = CompatibilityModes(compatibility_level_request.compatibility) - except (ValueError, KeyError): - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail={ - "error_code": SchemaErrorCodes.INVALID_COMPATIBILITY_LEVEL.value, - "message": SchemaErrorMessages.INVALID_COMPATIBILITY_LEVEL.value, - }, - ) - - self.schema_registry.send_config_message(compatibility_level=compatibility_level, subject=None) - return CompatibilityResponse(compatibility=self.schema_registry.schema_reader.config.compatibility) - - async def config_subject_get( - self, - *, - subject: str, - default_to_global: bool, - ) -> CompatibilityLevelResponse: - # Config for a subject can exist without schemas so no need to check for their existence - assert self.schema_registry.schema_reader, "KarapaceSchemaRegistry not initialized. Missing call to _init" - if self.schema_registry.database.find_subject(subject=Subject(subject)) is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.SUBJECT_NOT_FOUND.value, - "message": SchemaErrorMessages.SUBJECT_NOT_FOUND_FMT.value.format(subject=subject), - }, - ) - - compatibility = self.schema_registry.database.get_subject_compatibility(subject=Subject(subject)) - if not compatibility and default_to_global: - compatibility = self.schema_registry.compatibility - if compatibility: - # Note: The format sent by the user differs from the return - # value, this is for compatibility reasons. - return CompatibilityLevelResponse(compatibilityLevel=compatibility) - - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.SUBJECT_LEVEL_COMPATIBILITY_NOT_CONFIGURED_ERROR_CODE.value, - "message": SchemaErrorMessages.SUBJECT_LEVEL_COMPATIBILITY_NOT_CONFIGURED_FMT.value.format(subject=subject), - }, - ) - - async def config_subject_set( - self, - *, - subject: str, - compatibility_level_request: CompatibilityRequest, - ) -> CompatibilityResponse: - try: - compatibility_level = CompatibilityModes(compatibility_level_request.compatibility) - except (ValueError, KeyError): - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail={ - "error_code": SchemaErrorCodes.INVALID_COMPATIBILITY_LEVEL.value, - "message": "Invalid compatibility level", - }, - ) - - self.schema_registry.send_config_message(compatibility_level=compatibility_level, subject=Subject(subject)) - return CompatibilityResponse(compatibility=compatibility_level.value) - - async def config_subject_delete( - self, - *, - subject: str, - ) -> CompatibilityResponse: - self.schema_registry.send_config_subject_delete_message(subject=Subject(subject)) - return CompatibilityResponse(compatibility=self.schema_registry.schema_reader.config.compatibility) - - async def subjects_list( - self, - deleted: bool, - user: User | None, - authorizer: AuthenticatorAndAuthorizerDep | None, - ) -> list[str]: - subjects = [str(subject) for subject in self.schema_registry.database.find_subjects(include_deleted=deleted)] - if authorizer: - subjects = list( - filter( - lambda subject: authorizer.check_authorization(user, Operation.Read, f"Subject:{subject}"), - subjects, - ) - ) - return subjects - - async def subject_delete( - self, - *, - subject: str, - permanent: bool, - ) -> list[int]: - try: - version_list = await self.schema_registry.subject_delete_local(subject=Subject(subject), permanent=permanent) - return [version.value for version in version_list] - except (SubjectNotFoundException, SchemasNotFoundException): - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.SUBJECT_NOT_FOUND.value, - "message": SchemaErrorMessages.SUBJECT_NOT_FOUND_FMT.value.format(subject=subject), - }, - ) - except SubjectNotSoftDeletedException: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.SUBJECT_NOT_SOFT_DELETED.value, - "message": f"Subject '{subject}' was not deleted first before being permanently deleted", - }, - ) - except SubjectSoftDeletedException: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.SUBJECT_SOFT_DELETED.value, - "message": f"Subject '{subject}' was soft deleted.Set permanent=true to delete permanently", - }, - ) - - except ReferenceExistsException as arg: - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail={ - "error_code": SchemaErrorCodes.REFERENCE_EXISTS.value, - "message": ( - f"One or more references exist to the schema " - f"{{magic=1,keytype=SCHEMA,subject={subject},version={arg.version}}}." - ), - }, - ) - - async def subject_version_get( - self, - subject: str, - version: str, - deleted: bool, - ) -> SubjectSchemaVersionResponse: - try: - subject_data = self.schema_registry.subject_version_get( - Subject(subject), Versioner.V(version), include_deleted=deleted - ) - return SubjectSchemaVersionResponse( - subject=subject_data["subject"], - version=subject_data["version"], - id=subject_data["id"], - schema=subject_data["schema"], - references=subject_data.get("references", None), - schemaType=subject_data.get("schemaType", None), - compatibility=None, # Do not return compatibility from this endpoint. - ) - except (SubjectNotFoundException, SchemasNotFoundException): - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.SUBJECT_NOT_FOUND.value, - "message": SchemaErrorMessages.SUBJECT_NOT_FOUND_FMT.value.format(subject=subject), - }, - ) - except VersionNotFoundException: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.VERSION_NOT_FOUND.value, - "message": f"Version {version} not found.", - }, - ) - except InvalidVersion: - raise self._invalid_version(version) - - async def subject_version_delete( - self, - *, - subject: str, - version: str, - permanent: bool, - ) -> int: - try: - resolved_version = await self.schema_registry.subject_version_delete_local( - Subject(subject), Versioner.V(version), permanent - ) - return resolved_version.value - except (SubjectNotFoundException, SchemasNotFoundException): - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.SUBJECT_NOT_FOUND.value, - "message": SchemaErrorMessages.SUBJECT_NOT_FOUND_FMT.value.format(subject=subject), - }, - ) - except VersionNotFoundException: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.VERSION_NOT_FOUND.value, - "message": f"Version {version} not found.", - }, - ) - except SchemaVersionSoftDeletedException: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.SCHEMAVERSION_SOFT_DELETED.value, - "message": ( - f"Subject '{subject}' Version {version} was soft deleted. " - "Set permanent=true to delete permanently" - ), - }, - ) - except SchemaVersionNotSoftDeletedException: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.SCHEMAVERSION_NOT_SOFT_DELETED.value, - "message": ( - f"Subject '{subject}' Version {version} was not deleted " "first before being permanently deleted" - ), - }, - ) - except ReferenceExistsException as arg: - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail={ - "error_code": SchemaErrorCodes.REFERENCE_EXISTS.value, - "message": ( - f"One or more references exist to the schema " - f"{{magic=1,keytype=SCHEMA,subject={subject},version={arg.version}}}." - ), - }, - ) - except InvalidVersion: - self._invalid_version(version) - - async def subject_version_schema_get( - self, - *, - subject: str, - version: str, - ) -> dict: - try: - subject_data = self.schema_registry.subject_version_get(Subject(subject), Versioner.V(version)) - return json.loads(cast(str, subject_data["schema"])) # TODO typing - except InvalidVersion: - raise self._invalid_version(version) - except VersionNotFoundException: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.VERSION_NOT_FOUND.value, - "message": f"Version {version} not found.", - }, - ) - except (SchemasNotFoundException, SubjectNotFoundException): - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.SUBJECT_NOT_FOUND.value, - "message": SchemaErrorMessages.SUBJECT_NOT_FOUND_FMT.value.format(subject=subject), - }, - ) - - async def subject_version_referencedby_get( - self, - *, - subject: str, - version, - ) -> list[int]: - referenced_by: list[int] = [] - try: - referenced_by = await self.schema_registry.subject_version_referencedby_get( - Subject(subject), Versioner.V(version) - ) - except (SubjectNotFoundException, SchemasNotFoundException): - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.SUBJECT_NOT_FOUND.value, - "message": SchemaErrorMessages.SUBJECT_NOT_FOUND_FMT.value.format(subject=subject), - }, - ) - except VersionNotFoundException: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.VERSION_NOT_FOUND.value, - "message": f"Version {version} not found.", - }, - ) - except InvalidVersion: - raise self._invalid_version(version) - - return referenced_by - - async def subject_versions_list( - self, - *, - subject: str, - deleted: bool, - ) -> list[int]: - try: - schema_versions = self.schema_registry.subject_get(Subject(subject), include_deleted=deleted) - version_list = [version.value for version in schema_versions] - return version_list - except (SubjectNotFoundException, SchemasNotFoundException): - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.SUBJECT_NOT_FOUND.value, - "message": SchemaErrorMessages.SUBJECT_NOT_FOUND_FMT.value.format(subject=subject), - }, - ) - - def _validate_schema_type(self, data: JsonData) -> SchemaType: - # TODO: simplify the calling code, this functionality should not be required - # for old schemas. - if not isinstance(data, dict): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail={ - "error_code": SchemaErrorCodes.HTTP_BAD_REQUEST.value, - "message": "Malformed request", - }, - ) - schema_type_unparsed = data.get("schemaType", SchemaType.AVRO.value) - try: - schema_type = SchemaType(schema_type_unparsed) - except ValueError: - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail={ - "error_code": SchemaErrorCodes.HTTP_UNPROCESSABLE_ENTITY.value, - "message": f"Invalid schemaType {schema_type_unparsed}", - }, - ) - return schema_type - - def _validate_references( - self, - schema_request: SchemaRequest, - ) -> list[Reference | LatestVersionReference] | None: - references = schema_request.references - # Allow passing `null` as value for compatibility - if references is None: - return None - if references and schema_request.schema_type != SchemaType.PROTOBUF: - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail={ - "error_code": SchemaErrorCodes.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value, - "message": SchemaErrorMessages.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value.format( - schema_type=schema_request.schema_type.value - ), - }, - ) - - validated_references = [] - for reference in references: - version = Versioner.V(reference.version) - if version.is_latest: - validated_references.append( - LatestVersionReference( - name=reference.name, - subject=Subject(reference.subject), - ) - ) - else: - validated_references.append( - Reference( - name=reference.name, - subject=Subject(reference.subject), - version=version, - ) - ) - if validated_references: - return validated_references - return None - - async def subjects_schema_post( - self, - *, - subject: Subject, - schema_request: SchemaRequest, - deleted: bool, - normalize: bool, - ) -> SchemaResponse: - try: - subject_data = self._subject_get(subject, include_deleted=deleted) - except (SchemasNotFoundException, SubjectNotFoundException): - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.SUBJECT_NOT_FOUND.value, - "message": SchemaErrorMessages.SUBJECT_NOT_FOUND_FMT.value.format(subject=subject), - }, - ) - references = None - new_schema_dependencies = None - references = self._validate_references(schema_request) - references, new_schema_dependencies = self.schema_registry.resolve_references(references) - - new_schema: ParsedTypedSchema | None = None - try: - # When checking if schema is already registered, allow unvalidated schema in as - # there might be stored schemas that are non-compliant from the past. - new_schema = ParsedTypedSchema.parse( - schema_type=schema_request.schema_type, - schema_str=schema_request.schema_str, - references=references, - dependencies=new_schema_dependencies, - normalize=normalize, - use_protobuf_formatter=self.config.use_protobuf_formatter, - ) - except InvalidSchema: - LOG.warning("Invalid schema: %r", schema_request.schema_str) - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail={ - "error_code": SchemaErrorCodes.INVALID_SCHEMA.value, - "message": f"Error while looking up schema under subject {subject}", - }, - ) - except InvalidReferences: - human_error = "Provided references is not valid" - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail={ - "error_code": SchemaErrorCodes.INVALID_SCHEMA.value, - "message": f"Invalid {schema_request.schema_type} references. Error: {human_error}", - }, - ) - - # Match schemas based on version from latest to oldest - for schema_version in sorted(subject_data.values(), key=lambda item: item.version, reverse=True): - other_references, other_dependencies = self.schema_registry.resolve_references(schema_version.references) - try: - parsed_typed_schema = ParsedTypedSchema.parse( - schema_version.schema.schema_type, - schema_version.schema.schema_str, - references=other_references, - dependencies=other_dependencies, - normalize=normalize, - ) - except InvalidSchema as e: - failed_schema_id = schema_version.schema_id - LOG.exception("Existing schema failed to parse. Id: %s", failed_schema_id) - self.stats.unexpected_exception( - ex=e, where="Matching existing schemas to posted. Failed schema id: {failed_schema_id}" - ) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail={ - "error_code": SchemaErrorCodes.HTTP_INTERNAL_SERVER_ERROR.value, - "message": f"Error while looking up schema under subject {subject}", - }, - ) - - if schema_request.schema_type is SchemaType.JSONSCHEMA: - schema_valid = parsed_typed_schema.to_dict() == new_schema.to_dict() - else: - schema_valid = new_schema.match(parsed_typed_schema) - if parsed_typed_schema.schema_type == new_schema.schema_type and schema_valid: - schema_type: SchemaType | None = None - if schema_request.schema_type is not SchemaType.AVRO: - schema_type = schema_request.schema_type - return SchemaResponse( - subject=subject, - version=schema_version.version.value, - id=schema_version.schema_id, - schema=parsed_typed_schema.schema_str, - schemaType=schema_type, - ) - else: - LOG.debug("Schema %r did not match %r", schema_version, parsed_typed_schema) - - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.SCHEMA_NOT_FOUND.value, - "message": "Schema not found", - }, - ) - - async def subject_post( - self, - *, - subject: str, - schema_request: SchemaRequest, - normalize: bool, - forward_client: ForwardClient, - request: Request, - ) -> SchemaIdResponse | Response: - LOG.debug("POST with subject: %r, request: %r", subject, schema_request) - - references = self._validate_references(schema_request=schema_request) - - try: - references, resolved_dependencies = self.schema_registry.resolve_references(references) - new_schema = ValidatedTypedSchema.parse( - schema_type=schema_request.schema_type, - schema_str=schema_request.schema_str, - references=references, - dependencies=resolved_dependencies, - normalize=normalize, - use_protobuf_formatter=self.config.use_protobuf_formatter, - ) - except (InvalidReferences, InvalidSchema, InvalidSchemaType) as e: - LOG.warning("Invalid schema: %r", schema_request.schema_str, exc_info=True) - if isinstance(e.__cause__, (SchemaParseException, JSONDecodeError, ProtobufUnresolvedDependencyException)): - human_error = f"{e.__cause__.args[0]}" # pylint: disable=no-member - else: - from_body_schema_str = schema_request.schema_str - human_error = ( - f"Invalid schema {from_body_schema_str} with refs {references} of type {schema_request.schema_type}" - ) - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail={ - "error_code": SchemaErrorCodes.INVALID_SCHEMA.value, - "message": f"Invalid {schema_request.schema_type.value} schema. Error: {human_error}", - }, - ) - - schema_id = self.get_schema_id_if_exists(subject=Subject(subject), schema=new_schema, include_deleted=False) - if schema_id is not None: - return SchemaIdResponse(id=schema_id) - - i_am_primary, primary_url = await self.schema_registry.get_master() - if i_am_primary: - try: - schema_id = await self.schema_registry.write_new_schema_local(Subject(subject), new_schema, references) - return SchemaIdResponse(id=schema_id) - except InvalidSchema as ex: - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail={ - "error_code": SchemaErrorCodes.INVALID_SCHEMA.value, - "message": f"Invalid {schema_request.schema_type.value} schema. Error: {str(ex)}", - }, - ) - except IncompatibleSchema as ex: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail={ - "error_code": SchemaErrorCodes.HTTP_CONFLICT.value, - "message": str(ex), - }, - ) - except SchemaTooLargeException: - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail={ - "error_code": SchemaErrorCodes.SCHEMA_TOO_LARGE_ERROR_CODE.value, - "message": "Schema is too large", - }, - ) - except Exception as xx: - raise xx - - elif not primary_url: - raise no_primary_url_error() - else: - return await forward_client.forward_request_remote(request=request, primary_url=primary_url) - - async def get_global_mode(self) -> ModeResponse: - return ModeResponse(mode=str(self.schema_registry.get_global_mode())) - - async def get_subject_mode( - self, - *, - subject: str, - ) -> ModeResponse: - if self.schema_registry.database.find_subject(subject=Subject(subject)) is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.SUBJECT_NOT_FOUND.value, - "message": SchemaErrorMessages.SUBJECT_NOT_FOUND_FMT.value.format(subject=subject), - }, - ) - return ModeResponse(mode=str(self.schema_registry.get_global_mode())) - - def get_schema_id_if_exists(self, *, subject: Subject, schema: TypedSchema, include_deleted: bool) -> SchemaId | None: - schema_id = self.schema_registry.database.get_schema_id_if_exists( - subject=subject, schema=schema, include_deleted=include_deleted - ) - return schema_id - - def get_new_schema(self, schema_request: SchemaRequest) -> ValidatedTypedSchema: - references = self._validate_references(schema_request=schema_request) - try: - references, new_schema_dependencies = self.schema_registry.resolve_references(references) - new_schema_dependencies = {} - return ValidatedTypedSchema.parse( - schema_type=schema_request.schema_type, - schema_str=schema_request.schema_str, - references=references, - dependencies=new_schema_dependencies, - use_protobuf_formatter=self.config.use_protobuf_formatter, - ) - except InvalidSchema: - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail={ - "error_code": SchemaErrorCodes.INVALID_SCHEMA.value, - "message": f"Invalid {schema_request.schema_type} schema", - }, - ) - - def get_old_schema(self, subject: Subject, version: Version) -> ParsedTypedSchema: - old: JsonObject | None = None - try: - old = self.schema_registry.subject_version_get(subject=subject, version=version) - except InvalidVersion: - self._invalid_version(version.value) - except (VersionNotFoundException, SchemasNotFoundException, SubjectNotFoundException): - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={ - "error_code": SchemaErrorCodes.VERSION_NOT_FOUND.value, - "message": f"Version {version} not found.", - }, - ) - assert old is not None - old_schema_type = self._validate_schema_type(data=old) - try: - old_references = old.get("references", None) - old_dependencies = None - if old_references: - old_references, old_dependencies = self.schema_registry.resolve_references(old_references) - old_schema = ParsedTypedSchema.parse(old_schema_type, old["schema"], old_references, old_dependencies) - return old_schema - except InvalidSchema: - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail={ - "error_code": SchemaErrorCodes.INVALID_SCHEMA.value, - "message": f"Found an invalid {old_schema_type} schema registered", - }, - ) diff --git a/src/karapace/typing.py b/src/karapace/typing.py index 753b234aa..a205ae9de 100644 --- a/src/karapace/typing.py +++ b/src/karapace/typing.py @@ -8,6 +8,7 @@ from collections.abc import Mapping, Sequence from enum import Enum, unique from karapace.errors import InvalidVersion +from pydantic import ValidationInfo from typing import Any, ClassVar, NewType, Union from typing_extensions import TypeAlias @@ -35,11 +36,13 @@ class Subject(str): @classmethod + # TODO[pydantic]: We couldn't refactor `__get_validators__`, please create the `__get_pydantic_core_schema__` manually. + # Check https://docs.pydantic.dev/latest/migration/#defining-custom-types for more information. def __get_validators__(cls): yield cls.validate @classmethod - def validate(cls, subject_str: str) -> str: + def validate(cls, subject_str: str, _: ValidationInfo) -> str: """Subject may not contain control characters.""" if bool([c for c in subject_str if (ord(c) <= 31 or (ord(c) >= 127 and ord(c) <= 159))]): raise ValueError(f"The specified subject '{subject_str}' is not a valid.") diff --git a/src/schema_registry/__main__.py b/src/schema_registry/__main__.py index 0663bf774..7ff513584 100644 --- a/src/schema_registry/__main__.py +++ b/src/schema_registry/__main__.py @@ -2,7 +2,6 @@ Copyright (c) 2024 Aiven Ltd See LICENSE for details """ -from karapace.config import KARAPACE_BASE_CONFIG_YAML_PATH from karapace.container import KarapaceContainer from schema_registry.container import SchemaRegistryContainer from schema_registry.factory import create_karapace_application, karapace_schema_registry_lifespan @@ -11,6 +10,7 @@ import schema_registry.routers.compatibility import schema_registry.routers.config import schema_registry.routers.health +import schema_registry.routers.master_availability import schema_registry.routers.metrics import schema_registry.routers.mode import schema_registry.routers.schemas @@ -21,7 +21,6 @@ if __name__ == "__main__": container = KarapaceContainer() - container.base_config.from_yaml(KARAPACE_BASE_CONFIG_YAML_PATH, envs_required=True, required=True) container.wire( modules=[ __name__, @@ -42,6 +41,7 @@ schema_registry.routers.config, schema_registry.routers.compatibility, schema_registry.routers.mode, + schema_registry.routers.master_availability, ] ) diff --git a/src/schema_registry/routers/health.py b/src/schema_registry/routers/health.py index df3a8822f..b02d2f760 100644 --- a/src/schema_registry/routers/health.py +++ b/src/schema_registry/routers/health.py @@ -15,9 +15,9 @@ class HealthStatus(BaseModel): schema_registry_startup_time_sec: float schema_registry_reader_current_offset: int schema_registry_reader_highest_offset: int - schema_registry_is_primary: bool | None + schema_registry_is_primary: bool | None = None schema_registry_is_primary_eligible: bool - schema_registry_primary_url: str | None + schema_registry_primary_url: str | None = None schema_registry_coordinator_running: bool schema_registry_coordinator_generation_id: int @@ -40,13 +40,13 @@ async def health( schema_registry: KarapaceSchemaRegistry = Depends(Provide[SchemaRegistryContainer.karapace_container.schema_registry]), ) -> HealthCheck: starttime = 0.0 - if schema_registry.schema_reader.ready: + if schema_registry.schema_reader.ready(): starttime = schema_registry.schema_reader.last_check - schema_registry.schema_reader.start_time cs = schema_registry.mc.get_coordinator_status() health_status = HealthStatus( - schema_registry_ready=schema_registry.schema_reader.ready, + schema_registry_ready=schema_registry.schema_reader.ready(), schema_registry_startup_time_sec=starttime, schema_registry_reader_current_offset=schema_registry.schema_reader.offset, schema_registry_reader_highest_offset=schema_registry.schema_reader.highest_offset(), diff --git a/src/karapace/routers/master_available_router.py b/src/schema_registry/routers/master_availability.py similarity index 71% rename from src/karapace/routers/master_available_router.py rename to src/schema_registry/routers/master_availability.py index e55389f42..55e792275 100644 --- a/src/karapace/routers/master_available_router.py +++ b/src/schema_registry/routers/master_availability.py @@ -3,13 +3,14 @@ See LICENSE for details """ -from fastapi import APIRouter, HTTPException, Request, Response, status +from dependency_injector.wiring import inject, Provide +from fastapi import APIRouter, Depends, HTTPException, Request, Response, status from fastapi.responses import JSONResponse -from karapace.config import LOG -from karapace.dependencies.config_dependency import ConfigDep -from karapace.dependencies.forward_client_dependency import ForwardClientDep -from karapace.dependencies.schema_registry_dependency import SchemaRegistryDep +from karapace.config import Config +from karapace.forward_client import ForwardClient +from karapace.schema_registry import KarapaceSchemaRegistry from pydantic import BaseModel +from schema_registry.container import SchemaRegistryContainer from typing import Final import json @@ -33,12 +34,13 @@ class MasterAvailabilityResponse(BaseModel): @master_availability_router.get("") -async def master_available( - config: ConfigDep, - schema_registry: SchemaRegistryDep, - forward_client: ForwardClientDep, +@inject +async def master_availability( request: Request, response: Response, + config: Config = Depends(Provide[SchemaRegistryContainer.karapace_container.config]), + forward_client: ForwardClient = Depends(Provide[SchemaRegistryContainer.karapace_container.forward_client]), + schema_registry: KarapaceSchemaRegistry = Depends(Provide[SchemaRegistryContainer.karapace_container.schema_registry]), ) -> MasterAvailabilityResponse: are_we_master, master_url = await schema_registry.get_master() LOG.info("are master %s, master url %s", are_we_master, master_url) diff --git a/src/schema_registry/routers/requests.py b/src/schema_registry/routers/requests.py index fb4b51511..3d7f108b3 100644 --- a/src/schema_registry/routers/requests.py +++ b/src/schema_registry/routers/requests.py @@ -5,7 +5,7 @@ from karapace.schema_type import SchemaType from karapace.typing import Subject -from pydantic import BaseModel, Field, validator +from pydantic import BaseModel, ConfigDict, Field, field_validator from schema_registry.routers.errors import KarapaceValidationError from typing import Any @@ -20,13 +20,12 @@ class SchemaRequest(BaseModel): schema_str: str = Field(alias="schema") schema_type: SchemaType = Field(alias="schemaType", default=SchemaType.AVRO) references: list[SchemaReference] | None = None - metadata: Any | None - ruleSet: Any | None + metadata: Any | None = None + ruleSet: Any | None = None + model_config = ConfigDict(extra="forbid") - class Config: - extra = "forbid" - - @validator("schema_str") + @field_validator("schema_str") + @classmethod def validate_schema(cls, schema_str: str) -> str: if not schema_str and not schema_str.strip(): raise KarapaceValidationError( @@ -58,7 +57,7 @@ class SchemaListingItem(BaseModel): version: int schema_id: int = Field(alias="id") schema_type: SchemaType | None = Field(alias="schemaType", default=None) - references: list[Any] | None + references: list[Any] | None = None class SchemaIdResponse(BaseModel): diff --git a/src/schema_registry/routers/setup.py b/src/schema_registry/routers/setup.py index fe0b6be9b..663639583 100644 --- a/src/schema_registry/routers/setup.py +++ b/src/schema_registry/routers/setup.py @@ -7,6 +7,7 @@ from schema_registry.routers.compatibility import compatibility_router from schema_registry.routers.config import config_router from schema_registry.routers.health import health_router +from schema_registry.routers.master_availability import master_availability_router from schema_registry.routers.metrics import metrics_router from schema_registry.routers.mode import mode_router from schema_registry.routers.root import root_router @@ -23,3 +24,4 @@ def setup_routers(app: FastAPI) -> None: app.include_router(schemas_router) app.include_router(subjects_router) app.include_router(metrics_router) + app.include_router(master_availability_router) diff --git a/src/schema_registry/schema_registry_apis.py b/src/schema_registry/schema_registry_apis.py index cc9a01bb2..13f6bb8f2 100644 --- a/src/schema_registry/schema_registry_apis.py +++ b/src/schema_registry/schema_registry_apis.py @@ -65,9 +65,6 @@ class KarapaceSchemaRegistryController: def __init__(self, config: Config, schema_registry: KarapaceSchemaRegistry, stats: StatsClient) -> None: # super().__init__(config=config, not_ready_handler=self._forward_if_not_ready_to_serve) - print("+++++++++========") - print(schema_registry) - self.config = config self._process_start_time = time.monotonic() self.stats = stats @@ -219,8 +216,6 @@ def _has_subject_with_id() -> bool: ) schema = self.schema_registry.schemas_get(parsed_schema_id, fetch_max_id=fetch_max_id) - print("+++++++++========") - print(schema) if not schema: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, diff --git a/src/schema_registry/user.py b/src/schema_registry/user.py index 16cd55705..b3d6919a2 100644 --- a/src/schema_registry/user.py +++ b/src/schema_registry/user.py @@ -13,24 +13,17 @@ @inject async def get_current_user( - credentials: Annotated[HTTPBasicCredentials, Depends(HTTPBasic())], + credentials: Annotated[HTTPBasicCredentials, Depends(HTTPBasic(auto_error=False))], authorizer: AuthenticatorAndAuthorizer = Depends(Provide[SchemaRegistryContainer.karapace_container.authorizer]), ) -> User: - import logging - - logging.info("get_current_user ++++++++++++=============") - logging.info(f"credentials: {credentials}") - logging.info(f"authorizer: {authorizer}") - if authorizer and not credentials: + if authorizer.MUST_AUTHENTICATE and not credentials: raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail={"message": "Unauthorized"}, headers={"WWW-Authenticate": 'Basic realm="Karapace Schema Registry"'}, ) - assert authorizer is not None - assert credentials is not None - username: str = credentials.username - password: str = credentials.password + username: str = credentials.username if credentials else "" + password: str = credentials.password if credentials else "" try: return authorizer.authenticate(username=username, password=password) except AuthenticationError as exc: diff --git a/tests/conftest.py b/tests/conftest.py index 91fb0b02d..f6776d924 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,7 +3,6 @@ See LICENSE for details """ from avro.compatibility import SchemaCompatibilityResult -from karapace.config import KARAPACE_BASE_CONFIG_YAML_PATH from karapace.container import KarapaceContainer from pathlib import Path from schema_registry.container import SchemaRegistryContainer @@ -186,9 +185,7 @@ def fixture_tmp_file(): @pytest.fixture(name="karapace_container", scope="session") def fixture_karapace_container() -> KarapaceContainer: - container = KarapaceContainer() - container.base_config.from_yaml(KARAPACE_BASE_CONFIG_YAML_PATH, envs_required=True, required=True) - return container + return KarapaceContainer() @pytest.fixture diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 000000000..f53be7121 --- /dev/null +++ b/tests/e2e/__init__.py @@ -0,0 +1,4 @@ +""" +Copyright (c) 2024 Aiven Ltd +See LICENSE for details +""" diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py new file mode 100644 index 000000000..8e1d23d64 --- /dev/null +++ b/tests/e2e/conftest.py @@ -0,0 +1,128 @@ +""" +karapace - conftest + +Copyright (c) 2023 Aiven Ltd +See LICENSE for details +""" +from __future__ import annotations + +from _pytest.fixtures import SubRequest +from aiohttp import BasicAuth +from collections.abc import AsyncGenerator, Iterator +from confluent_kafka.admin import NewTopic +from karapace.client import Client +from karapace.container import KarapaceContainer +from karapace.kafka.admin import KafkaAdminClient +from karapace.kafka.consumer import AsyncKafkaConsumer, KafkaConsumer +from karapace.kafka.producer import AsyncKafkaProducer, KafkaProducer +from tests.integration.utils.cluster import RegistryDescription, RegistryEndpoint +from tests.integration.utils.kafka_server import KafkaServers + +import asyncio +import pytest +import secrets + + +@pytest.fixture(scope="session", name="basic_auth") +def fixture_basic_auth() -> BasicAuth: + return BasicAuth("test", "test") + + +@pytest.fixture(name="karapace_container", scope="session") +def fixture_karapace_container() -> KarapaceContainer: + return KarapaceContainer() + + +@pytest.fixture(scope="session", name="kafka_servers") +def fixture_kafka_server(karapace_container: KarapaceContainer) -> Iterator[KafkaServers]: + yield KafkaServers([karapace_container.config().bootstrap_uri]) + + +@pytest.fixture(scope="function", name="producer") +def fixture_producer(kafka_servers: KafkaServers) -> Iterator[KafkaProducer]: + yield KafkaProducer(bootstrap_servers=kafka_servers.bootstrap_servers) + + +@pytest.fixture(scope="function", name="admin_client") +def fixture_admin(kafka_servers: KafkaServers) -> Iterator[KafkaAdminClient]: + yield KafkaAdminClient(bootstrap_servers=kafka_servers.bootstrap_servers) + + +@pytest.fixture(scope="function", name="consumer") +def fixture_consumer( + kafka_servers: KafkaServers, +) -> Iterator[KafkaConsumer]: + consumer = KafkaConsumer( + bootstrap_servers=kafka_servers.bootstrap_servers, + auto_offset_reset="earliest", + enable_auto_commit=False, + topic_metadata_refresh_interval_ms=200, # Speed things up for consumer tests to discover topics, etc. + ) + try: + yield consumer + finally: + consumer.close() + + +@pytest.fixture(scope="function", name="asyncproducer") +async def fixture_asyncproducer( + kafka_servers: KafkaServers, + loop: asyncio.AbstractEventLoop, +) -> AsyncGenerator[AsyncKafkaProducer, None]: + asyncproducer = AsyncKafkaProducer(bootstrap_servers=kafka_servers.bootstrap_servers, loop=loop) + await asyncproducer.start() + yield asyncproducer + await asyncproducer.stop() + + +@pytest.fixture(scope="function", name="asyncconsumer") +async def fixture_asyncconsumer( + kafka_servers: KafkaServers, + loop: asyncio.AbstractEventLoop, +) -> AsyncGenerator[AsyncKafkaConsumer, None]: + asyncconsumer = AsyncKafkaConsumer( + bootstrap_servers=kafka_servers.bootstrap_servers, + loop=loop, + auto_offset_reset="earliest", + enable_auto_commit=False, + topic_metadata_refresh_interval_ms=200, # Speed things up for consumer tests to discover topics, etc. + ) + await asyncconsumer.start() + yield asyncconsumer + await asyncconsumer.stop() + + +@pytest.fixture(scope="function", name="registry_cluster") +async def fixture_registry_cluster( + karapace_container: KarapaceContainer, + loop: asyncio.AbstractEventLoop, # pylint: disable=unused-argument +) -> RegistryDescription: + protocol = "http" + endpoint = RegistryEndpoint( + protocol, karapace_container.config().registry_host, karapace_container.config().registry_port + ) + return RegistryDescription(endpoint, karapace_container.config().topic_name) + + +@pytest.fixture(scope="function", name="registry_async_client") +async def fixture_registry_async_client( + request: SubRequest, + basic_auth: BasicAuth, + registry_cluster: RegistryDescription, + loop: asyncio.AbstractEventLoop, # pylint: disable=unused-argument +) -> AsyncGenerator[Client, None]: + client = Client( + server_uri=registry_cluster.endpoint.to_url(), + server_ca=request.config.getoption("server_ca"), + session_auth=basic_auth, + ) + try: + yield client + finally: + await client.close() + + +@pytest.fixture(scope="function", name="new_topic") +def fixture_new_topic(admin_client: KafkaAdminClient) -> NewTopic: + topic_name = secrets.token_hex(4) + return admin_client.new_topic(topic_name, num_partitions=1, replication_factor=1) diff --git a/src/karapace/routers/__init__.py b/tests/e2e/instrumentation/__init__.py similarity index 100% rename from src/karapace/routers/__init__.py rename to tests/e2e/instrumentation/__init__.py diff --git a/tests/integration/instrumentation/test_prometheus.py b/tests/e2e/instrumentation/test_prometheus.py similarity index 100% rename from tests/integration/instrumentation/test_prometheus.py rename to tests/e2e/instrumentation/test_prometheus.py diff --git a/tests/integration/instrumentation/__init__.py b/tests/e2e/kafka/__init__.py similarity index 100% rename from tests/integration/instrumentation/__init__.py rename to tests/e2e/kafka/__init__.py diff --git a/tests/integration/kafka/test_admin.py b/tests/e2e/kafka/test_admin.py similarity index 100% rename from tests/integration/kafka/test_admin.py rename to tests/e2e/kafka/test_admin.py diff --git a/tests/integration/kafka/test_consumer.py b/tests/e2e/kafka/test_consumer.py similarity index 100% rename from tests/integration/kafka/test_consumer.py rename to tests/e2e/kafka/test_consumer.py diff --git a/tests/integration/kafka/test_producer.py b/tests/e2e/kafka/test_producer.py similarity index 100% rename from tests/integration/kafka/test_producer.py rename to tests/e2e/kafka/test_producer.py diff --git a/tests/integration/kafka/__init__.py b/tests/e2e/schema_registry/__init__.py similarity index 100% rename from tests/integration/kafka/__init__.py rename to tests/e2e/schema_registry/__init__.py diff --git a/tests/integration/schema_registry/test_jsonschema.py b/tests/e2e/schema_registry/test_jsonschema.py similarity index 100% rename from tests/integration/schema_registry/test_jsonschema.py rename to tests/e2e/schema_registry/test_jsonschema.py diff --git a/tests/e2e/test_karapace.py b/tests/e2e/test_karapace.py new file mode 100644 index 000000000..ccf4bde1e --- /dev/null +++ b/tests/e2e/test_karapace.py @@ -0,0 +1,64 @@ +""" +Copyright (c) 2023 Aiven Ltd +See LICENSE for details +""" +from collections.abc import Iterator +from contextlib import closing, contextmanager, ExitStack +from pathlib import Path +from tests.integration.utils.kafka_server import KafkaServers +from tests.integration.utils.process import stop_process +from tests.utils import popen_karapace_all + +import socket + + +@contextmanager +def allocate_port_no_reuse() -> Iterator[int]: + """Allocate random free port and do not allow reuse.""" + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: + sock.bind(("127.0.0.1", 0)) + yield sock.getsockname()[1] + + +def test_regression_server_must_exit_on_exception( + tmp_path: Path, + kafka_servers: Iterator[KafkaServers], +) -> None: + """Regression test for Karapace properly exiting. + + Karapace was not closing all its background threads, so when an exception + was raised an reached the top-level, the webserver created by asyncio would + be stopped but the threads would keep the server running. + Karapace exit on exception is done by setting a reserved port as server port. + """ + with ExitStack() as stack: + karapace_rest_proxy_port = stack.enter_context(allocate_port_no_reuse()) + karapace_schema_registry_port = stack.enter_context(allocate_port_no_reuse()) + logfile = stack.enter_context((tmp_path / "karapace.log").open("w")) + errfile = stack.enter_context((tmp_path / "karapace.err").open("w")) + + karapace_rest_proxy_env = { + "KARAPACE_BOOTSTRAP_URI": kafka_servers.bootstrap_servers[0], + "KARAPACE_PORT": str(karapace_rest_proxy_port), + "KARAPACE_REGISTRY_HOST": "127.0.0.1", + "KARAPACE_REGISTRY_PORT": str(karapace_schema_registry_port), + "KARAPACE_KARAPACE_REST": "true", + } + karapace_rest_proxy = popen_karapace_all( + module="karapace.karapace_all", env=karapace_rest_proxy_env, stdout=logfile, stderr=errfile + ) + stack.callback(stop_process, karapace_rest_proxy) # make sure to stop the process if the test fails + assert karapace_rest_proxy.wait(timeout=10) != 0, "Process should have exited with an error, port is already is use" + + karapace_schema_registry_env = { + "KARAPACE_BOOTSTRAP_URI": kafka_servers.bootstrap_servers[0], + "KARAPACE_PORT": str(karapace_schema_registry_port), + "KARAPACE_KARAPACE_REGISTRY": "true", + } + karapace_schema_registry = popen_karapace_all( + module="schema_registry", env=karapace_schema_registry_env, stdout=logfile, stderr=errfile + ) + stack.callback(stop_process, karapace_schema_registry) # make sure to stop the process if the test fails + assert ( + karapace_schema_registry.wait(timeout=10) != 0 + ), "Process should have exited with an error, port is already is use" diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 84be44595..5a435f235 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -15,7 +15,7 @@ from dataclasses import asdict from filelock import FileLock from karapace.client import Client -from karapace.config import Config, write_config +from karapace.config import Config from karapace.kafka.admin import KafkaAdminClient from karapace.kafka.consumer import AsyncKafkaConsumer, KafkaConsumer from karapace.kafka.producer import AsyncKafkaProducer, KafkaProducer @@ -149,12 +149,12 @@ def create_kafka_server( stack.callback(stop_process, zk_proc) # Make sure zookeeper is running before trying to start Kafka - wait_for_port_subprocess(zk_config.client_port, zk_proc, wait_time=20) + wait_for_port_subprocess(zk_config.client_port, zk_proc, wait_time=KAFKA_WAIT_TIMEOUT) data_dir = session_datadir / "kafka" log_dir = session_logdir / "kafka" - data_dir.mkdir(parents=True) - log_dir.mkdir(parents=True) + data_dir.mkdir(parents=True, exist_ok=True) + log_dir.mkdir(parents=True, exist_ok=True) kafka_config = KafkaConfig( datadir=str(data_dir), logdir=str(log_dir), @@ -275,15 +275,12 @@ async def fixture_rest_async( yield None return - config_path = tmp_path / "karapace_config.json" - config = Config() config.admin_metadata_max_age = 2 config.bootstrap_uri = kafka_servers.bootstrap_servers[0] # Use non-default max request size for REST producer. config.producer_max_request_size = REST_PRODUCER_MAX_REQUEST_BYTES config.waiting_time_before_acting_as_master_ms = 300 - write_config(config_path, config) rest = KafkaRest(config=config) assert rest.serializer.registry_client @@ -346,8 +343,6 @@ async def fixture_rest_async_novalidation( yield None return - config_path = tmp_path / "karapace_config.json" - config = Config() config.admin_metadata_max_age = 2 config.bootstrap_uri = kafka_servers.bootstrap_servers[0] @@ -355,7 +350,6 @@ async def fixture_rest_async_novalidation( config.producer_max_request_size = REST_PRODUCER_MAX_REQUEST_BYTES config.name_strategy_validation = False # This should be only difference from rest_async config.waiting_time_before_acting_as_master_ms = 300 - write_config(config_path, config) rest = KafkaRest(config=config) assert rest.serializer.registry_client diff --git a/tests/integration/schema_registry/__init__.py b/tests/integration/schema_registry/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/integration/test_dependencies_compatibility_protobuf.py b/tests/integration/test_dependencies_compatibility_protobuf.py index c52a49141..9c6e997fc 100644 --- a/tests/integration/test_dependencies_compatibility_protobuf.py +++ b/tests/integration/test_dependencies_compatibility_protobuf.py @@ -507,7 +507,7 @@ async def test_protobuf_schema_references_rejected_values(registry_async_client: ) assert res.status_code == 422 assert res.json()["message"] == [ - {"loc": ["body", "references"], "msg": "value is not a valid list", "type": "type_error.list"} + {'type': 'list_type', 'loc': ['body', 'references'], 'msg': 'Input should be a valid list', 'input': 1} ] res = await registry_async_client.post( @@ -515,7 +515,7 @@ async def test_protobuf_schema_references_rejected_values(registry_async_client: ) assert res.status_code == 422 assert res.json()["message"] == [ - {"loc": ["body", "references"], "msg": "value is not a valid list", "type": "type_error.list"} + {'type': 'list_type', 'loc': ['body', 'references'], 'msg': 'Input should be a valid list', 'input': 'foo'} ] res = await registry_async_client.post( @@ -523,7 +523,7 @@ async def test_protobuf_schema_references_rejected_values(registry_async_client: ) assert res.status_code == 422 assert res.json()["message"] == [ - {"loc": ["body", "references"], "msg": "value is not a valid list", "type": "type_error.list"} + {'type': 'list_type', 'loc': ['body', 'references'], 'msg': 'Input should be a valid list', 'input': False} ] res = await registry_async_client.post( @@ -532,7 +532,7 @@ async def test_protobuf_schema_references_rejected_values(registry_async_client: ) assert res.status_code == 422 assert res.json()["message"] == [ - {"loc": ["body", "references"], "msg": "value is not a valid list", "type": "type_error.list"} + {'type': 'list_type', 'loc': ['body', 'references'], 'msg': 'Input should be a valid list', 'input': {"this_is_object": True}} ] diff --git a/tests/integration/test_karapace.py b/tests/integration/test_karapace.py deleted file mode 100644 index 043e3e21d..000000000 --- a/tests/integration/test_karapace.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -Copyright (c) 2023 Aiven Ltd -See LICENSE for details -""" -from collections.abc import Iterator -from contextlib import closing, contextmanager, ExitStack -from karapace.config import Config, write_env_file -from pathlib import Path -from tests.integration.utils.kafka_server import KafkaServers -from tests.integration.utils.process import stop_process -from tests.utils import popen_karapace_all - -import socket - - -@contextmanager -def allocate_port_no_reuse() -> Iterator[int]: - """Allocate random free port and do not allow reuse.""" - with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: - sock.bind(("127.0.0.1", 0)) - yield sock.getsockname()[1] - - -def test_regression_server_must_exit_on_exception( - tmp_path: Path, - kafka_servers: Iterator[KafkaServers], -) -> None: - """Regression test for Karapace properly exiting. - - Karapace was not closing all its background threads, so when an exception - was raised an reached the top-level, the webserver created by asyncio would - be stopped but the threads would keep the server running. - Karapace exit on exception is done by setting a reserved port as server port. - """ - with ExitStack() as stack: - port = stack.enter_context(allocate_port_no_reuse()) - - config = Config() - config.bootstrap_uri = kafka_servers.bootstrap_servers[0] - config.port = port - config.karapace_registry = True - - env_path = tmp_path / "karapace.env" - - print(f"{tmp_path}/karapace.log") - print(f"{tmp_path}/karapace.err") - - logfile = stack.enter_context((tmp_path / "karapace.log").open("w")) - errfile = stack.enter_context((tmp_path / "karapace.err").open("w")) - - write_env_file(dot_env_path=env_path, config=config) - process = popen_karapace_all(env_path=env_path, stdout=logfile, stderr=errfile) - stack.callback(stop_process, process) # make sure to stop the process if the test fails - assert process.wait(timeout=10) != 0, "Process should have exited with an error, port is already is use" diff --git a/tests/integration/test_schema.py b/tests/integration/test_schema.py index 9a9eb31e7..3d12930b6 100644 --- a/tests/integration/test_schema.py +++ b/tests/integration/test_schema.py @@ -9,7 +9,7 @@ from karapace.client import Client from karapace.kafka.producer import KafkaProducer from karapace.rapu import is_success -from karapace.schema_registry_apis import SchemaErrorMessages +from schema_registry.schema_registry_apis import SchemaErrorMessages from karapace.schema_type import SchemaType from karapace.utils import json_encode from tests.base_testcase import BaseTestCase @@ -1428,7 +1428,7 @@ async def test_schema_missing_body(registry_async_client: Client) -> None: ) assert res.status_code == 422 assert res.json()["error_code"] == 422 - assert res.json()["message"] == [{"loc": ["body", "schema"], "msg": "field required", "type": "value_error.missing"}] + assert res.json()["message"] == [{'type': 'missing', 'loc': ['body', 'schema'], 'msg': 'Field required', 'input': {}}] async def test_schema_missing_schema_body_ok(registry_async_client: Client) -> None: @@ -1539,7 +1539,7 @@ async def test_schema_subject_post_invalid(registry_async_client: Client) -> Non res = await registry_async_client.post(f"subjects/{subject_1}", json={}) assert res.status_code == 422 assert res.json()["error_code"] == 422 - assert res.json()["message"] == [{"loc": ["body", "schema"], "msg": "field required", "type": "value_error.missing"}] + assert res.json()["message"] == [{'type': 'missing', 'loc': ['body', 'schema'], 'msg': 'Field required', 'input': {}}] # Schema not included in the request body for subject that does not exist subject_3 = subject_name_factory() @@ -1549,7 +1549,7 @@ async def test_schema_subject_post_invalid(registry_async_client: Client) -> Non ) assert res.status_code == 422 assert res.json()["error_code"] == 422 - assert res.json()["message"] == [{"loc": ["body", "schema"], "msg": "field required", "type": "value_error.missing"}] + assert res.json()["message"] == [{'type': 'missing', 'loc': ['body', 'schema'], 'msg': 'Field required', 'input': {}}] async def test_schema_lifecycle(registry_async_client: Client) -> None: @@ -2306,8 +2306,8 @@ async def test_schema_body_validation(registry_async_client: Client) -> None: assert res.status_code == 422 assert res.json()["error_code"] == 422 assert res.json()["message"] == [ - {"loc": ["body", "schema"], "msg": "field required", "type": "value_error.missing"}, - {"loc": ["body", "invalid_field"], "msg": "extra fields not permitted", "type": "value_error.extra"}, + {'type': 'missing', 'loc': ['body', 'schema'], 'msg': 'Field required', 'input': {'invalid_field': 'invalid_value'}}, + {'type': 'extra_forbidden', 'loc': ['body', 'invalid_field'], 'msg': 'Extra inputs are not permitted', 'input': 'invalid_value'}, ] # Additional field res = await registry_async_client.post( @@ -2316,13 +2316,13 @@ async def test_schema_body_validation(registry_async_client: Client) -> None: assert res.status_code == 422 assert res.json()["error_code"] == 422 assert res.json()["message"] == [ - {"loc": ["body", "invalid_field"], "msg": "extra fields not permitted", "type": "value_error.extra"} + {'type': 'extra_forbidden', 'loc': ['body', 'invalid_field'], 'msg': 'Extra inputs are not permitted', 'input': 'invalid_value'}, ] # Invalid body type res = await registry_async_client.post(endpoint, json="invalid") assert res.status_code == 422 assert res.json()["error_code"] == 422 - assert res.json()["message"] == [{"loc": ["body"], "msg": "value is not a valid dict", "type": "type_error.dict"}] + assert res.json()["message"] == [{'type': 'model_attributes_type', 'loc': ['body'], 'msg': 'Input should be a valid dictionary or object to extract fields from', 'input': 'invalid'}] async def test_version_number_validation(registry_async_client: Client) -> None: diff --git a/tests/integration/utils/cluster.py b/tests/integration/utils/cluster.py index 66df9335a..5557ee02c 100644 --- a/tests/integration/utils/cluster.py +++ b/tests/integration/utils/cluster.py @@ -52,21 +52,10 @@ async def start_schema_registry_cluster( all_registries = [] with ExitStack() as stack: for pos, config in enumerate(config_templates): - # For testing we don't want to expose the hostname, usually the loopback interface is - # used (127.0.0.1), and the name resolution would instead return the machine's network - # address, (e.g. 192.168.0.1), which would cause connect failures - host = config.host - config.advertised_hostname = host - config.topic_name = schemas_topic - config.karapace_registry = True - config.log_level = "DEBUG" - config.log_format = "%(asctime)s [%(threadName)s] %(filename)s:%(funcName)s:%(lineno)d %(message)s" - actual_group_id = config.group_id = group_id - - port = config.port = stack.enter_context(allocate_port()) + port = stack.enter_context(allocate_port()) assert isinstance(port, int), "Port must be an integer" - group_dir = data_dir / str(actual_group_id) + group_dir = data_dir / str(group_id) group_dir.mkdir(parents=True, exist_ok=True) env_path = group_dir / f"{pos}.env" log_path = group_dir / f"{pos}.log" @@ -77,16 +66,31 @@ async def start_schema_registry_cluster( logfile = stack.enter_context(open(log_path, "w")) errfile = stack.enter_context(open(error_path, "w")) - process = popen_karapace_all(env_path=env_path, stdout=logfile, stderr=errfile) + + env = { + "KARAPACE_HOST": config.host, + "KARAPACE_PORT": str(port), + "KARAPACE_GROUP_ID": group_id, + "KARAPACE_ADVERTISED_HOSTNAME": config.host, + "KARAPACE_BOOTSTRAP_URI": config.bootstrap_uri, + "KARAPACE_TOPIC_NAME": schemas_topic, + "KARAPACE_LOG_LEVEL": "DEBUG", + "KARAPACE_LOG_FORMAT": "%(asctime)s [%(threadName)s] %(filename)s:%(funcName)s:%(lineno)d %(message)s", + "KARAPACE_KARAPACE_REGISTRY": "true", + "KARAPACE_REGISTRY_AUTHFILE": config.registry_authfile if config.registry_authfile else "", + "KARAPACE_SERVER_TLS_CERTFILE": config.server_tls_certfile if config.server_tls_certfile else "", + "KARAPACE_SERVER_TLS_KEYFILE": config.server_tls_keyfile if config.server_tls_keyfile else "", + } + process = popen_karapace_all(module="schema_registry", env=env, stdout=logfile, stderr=errfile) stack.callback(stop_process, process) all_processes.append(process) protocol = "http" if config.server_tls_keyfile is None else "https" - endpoint = RegistryEndpoint(protocol, host, port) + endpoint = RegistryEndpoint(protocol, config.host, port) description = RegistryDescription(endpoint, schemas_topic) all_registries.append(description) for process in all_processes: - wait_for_port_subprocess(port, process, hostname=host) + wait_for_port_subprocess(port, process, hostname=config.host, wait_time=120) yield all_registries diff --git a/tests/integration/utils/zookeeper.py b/tests/integration/utils/zookeeper.py index 5dffcfeca..1ffb798db 100644 --- a/tests/integration/utils/zookeeper.py +++ b/tests/integration/utils/zookeeper.py @@ -25,7 +25,7 @@ def configure_and_start_zk(config: ZKConfig, kafka_description: KafkaDescription zk_dir = Path(config.path) cfg_path = zk_dir / "zoo.cfg" logs_dir = zk_dir / "logs" - logs_dir.mkdir(parents=True) + logs_dir.mkdir(parents=True, exist_ok=True) zoo_cfg = { # Number of milliseconds of each tick diff --git a/tests/utils.py b/tests/utils.py index e36093dc0..ecddea84e 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -10,7 +10,7 @@ from karapace.utils import Expiration from pathlib import Path from subprocess import Popen -from typing import Any, Callable, IO, Union +from typing import Any, Callable, IO from urllib.parse import quote import asyncio @@ -344,12 +344,12 @@ def python_exe() -> str: return python -def popen_karapace_all(*, env_path: Union[Path, str], stdout: IO, stderr: IO, **kwargs) -> Popen: +def popen_karapace_all(*, module: str, env: dict[str], stdout: IO, stderr: IO, **kwargs) -> Popen: kwargs["stdout"] = stdout kwargs["stderr"] = stderr return Popen( - [python_exe(), "-m", "karapace.karapace_all"], - env={"KARAPACE_DOTENV": str(env_path)}, + [python_exe(), "-m", module], + env=env, **kwargs, )