chore: use source layout

.coveragerc

@@ -1,4 +1,4 @@
 [run]
-branch = True
-relative_files = True
-source = karapace
+branch = true
+relative_files = true
+source = src/karapace

.dockerignore

@@ -3,11 +3,13 @@
 *
 
 # Include source directories and files required for building.
-!karapace
 !go
+!src
 !requirements/*.txt
 !README.rst
+!LICENSE
 !pyproject.toml
+!setup.py
 !container/start.sh
 !container/healthcheck.py
 

.github/workflows/container-smoke-test.yml

@@ -18,10 +18,11 @@ jobs:
           fetch-depth: 0
 
       - name: Install requirements
-        run: pip install .
+        run: make install
 
       - name: Resolve Karapace version
         run: |
+          source ./venv/bin/activate
           KARAPACE_VERSION=$(python -c "from karapace import version; print(version.__version__)")
           echo KARAPACE_VERSION=$KARAPACE_VERSION >> $GITHUB_ENV
 

.github/workflows/lint.yml

@@ -44,5 +44,10 @@ jobs:
     - name: Install libsnappy-dev
       run: sudo apt install libsnappy-dev
     - name: Install requirements and typing requirements
-      run: pip install .[typing]
-    - run: mypy
+      run: |
+        python -m venv venv
+        source ./venv/bin/activate
+        pip --require-virtualenv install .[typing]
+    - run: |
+        source ./venv/bin/activate
+        mypy src

.github/workflows/schema.yml

@@ -22,12 +22,16 @@ jobs:
       - name: Install libsnappy-dev
         run: sudo apt install libsnappy-dev
       - name: Install requirements
-      - run: pip install .
+        run: make install-dev
       # Compare with latest release when running on main.
-      - run: make schema against=$(git describe --abbrev=0 --tags)
+      - run: |
+          source ./venv/bin/activate
+          make schema against=$(git describe --abbrev=0 --tags)
         if: github.ref == 'refs/heads/main'
       # Compare with main when running on branches/PRs.
-      - run: make schema
+      - run: |
+          source ./venv/bin/activate
+          make schema
         if: github.ref != 'refs/heads/main'
       - run: |
           diff=$(git --no-pager diff)

.github/workflows/tests.yml

@@ -40,7 +40,7 @@ jobs:
       with:
         go-version: '1.21.0'
 
-    - run: make install
+    - run: make install-dev
     - run: make unit-tests
       env:
         COVERAGE_FILE: ".coverage.${{ matrix.python-version }}"
@@ -73,7 +73,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - run: make install
+    - run: make install-dev
 
     - name: Download coverage
       id: download_coverage

GNUmakefile

@@ -42,29 +42,42 @@ venv/.make:
 
 .PHONY: install
 install: venv/.deps
-venv/.deps: requirements/requirements-dev.txt requirements/requirements.txt | venv/.make
+venv/.deps: venv/.make
 	set +x
 	source ./bin/get-java
 	source ./bin/get-protoc
 	source ./bin/get-snappy
 	set -x
-	$(PIP) install --use-pep517 . .[dev]
+	$(PIP) install --use-pep517 .
 	$(PIP) check
 	touch '$(@)'
 
+.PHONY: install-dev
+install-dev: venv/.deps-dev
+venv/.deps-dev: venv/.make
+	set +x
+	source ./bin/get-java
+	source ./bin/get-protoc
+	source ./bin/get-snappy
+	set -x
+	$(PIP) install -e .[dev]
+	$(PIP) check
+	touch '$(@)'
+
+
 .PHONY: test
 tests: unit-tests integration-tests
 
 .PHONY: unit-tests
 unit-tests: export PYTEST_ARGS ?=
-unit-tests: venv/.deps
+unit-tests: venv/.deps-dev
 	rm -fr runtime/*
 	$(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/unit/
 	rm -fr runtime/*
 
 .PHONY: integration-tests
 unit-tests: export PYTEST_ARGS ?=
-integration-tests: venv/.deps
+integration-tests: venv/.deps-dev
 	rm -fr runtime/*
 	$(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/integration/
 	rm -fr runtime/*
@@ -82,7 +95,7 @@ cleanest: cleaner
 	rm -fr '$(VENV_DIR)'
 
 .PHONY: requirements
-requirements: export CUSTOM_COMPILE_COMMAND='make requirements'
+requirements:
 requirements:
 	$(PIP) install --upgrade pip setuptools pip-tools
 	$(PIP) install .[dev,typing]

MANIFEST.in

@@ -1,4 +1,4 @@
-include karapace/*.py
+graft src
 include karapace.json
 include karapace.unit
 include tests/*.py
@@ -7,6 +7,5 @@ include README.rst
 include pyproject.toml
 include LICENSE
 include MANIFEST.in
-include *.so
 
 recursive-exclude examples *~ *.pyc \.*

container/Dockerfile

@@ -29,7 +29,7 @@ COPY . /build/karapace-repo
 WORKDIR /build/karapace-repo
 RUN --mount=type=cache,target=/root/.cache/pip \
     if [ -z "${KARAPACE_VERSION}" ]; then \
-        PRETEND_VERSION="$(python -c 'from karapace import version; print(version.__version__)')"; \
+        PRETEND_VERSION="$(python -c 'from src.karapace import version; print(version.__version__)')"; \
     else \
         PRETEND_VERSION=$KARAPACE_VERSION; \
     fi; \

pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools > 64", "setuptools-scm", "setuptools-golang"]
+requires = ["setuptools > 64", "setuptools-scm > 8.0.0", "setuptools-golang"]
 build-backend = "setuptools.build_meta"
 
 [project]
@@ -103,11 +103,8 @@ typing = [
 [tool.setuptools]
 include-package-data = true
 
-[tool.setuptools.packages.find]
-include = ["karapace"]
-
 [tool.setuptools_scm]
-version_file = "karapace/version.py"
+version_file = "src/karapace/version.py"
 
 [tool.black]
 target-version = ["py38"]

karapace/backup/backends/v3/schema_tool.py → src/karapace/backup/backends/v3/schema_tool.py

@@ -58,6 +58,17 @@ def relative_path(path: pathlib.Path) -> pathlib.Path:
     return pathlib.Path(str_path[len(cwd) + 1 :]) if str_path.startswith(cwd) else path
 
 
+def target_has_source_layout(git_target: str) -> bool:
+    with subprocess.Popen(
+        ["git", "show", f"{git_target}:src"],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    ) as cp:
+        if cp.returncode == 128:
+            return False
+        return True
+
+
 def check_compatibility(git_target: str) -> None:
     errored = False
     found_any = False
@@ -70,8 +81,13 @@ def check_compatibility(git_target: str) -> None:
 
     subprocess.run(["git", "fetch", remote, branch], check=True, capture_output=True)
 
+    # Does the target version have source layout
+    source_layout = target_has_source_layout(git_target)
+
     for file in schema_directory.glob(f"*{extension}"):
         relative = relative_path(file)
+        if source_layout:
+            relative = pathlib.Path(*relative.parts[1:])
         with subprocess.Popen(
             ["git", "show", f"{git_target}:{relative}"],
             stdout=subprocess.PIPE,