Add strict and offline index support (#36)

Creates two separate indexes, `packages/local` and `packages/all`
instead of just `packages/all`. `packages/all` inherits from `root/pypi`
and `packages/local`. `packages/local` does not allow packages to be
downloaded from PyPI.

All of the "build" dependencies must be published to `packages/local`
for it to be used to install source distributions. This pull request
vendors all of the needed dependencies.

Additionally, there is an `--offline` flag for `packse index up` which
does not allow any PyPI access. This is kind of nice when you are not
planning on using fallback to PyPI as it prevents the devpi server from
doing any additional indexing of the real PyPI.

This was tested by installing a scenario published to the strict and
offline indexes.

.github/workflows/publish.yaml

@@ -79,8 +79,8 @@ jobs:
       - name: Publish scenarios [local]
         if: github.ref != 'refs/heads/main'
         run: |
-          # Start the local index server
-          index_url=$(poetry run packse index up --bg)
+          # Start the local index server, do not allow packages from PyPI
+          index_url=$(poetry run packse index up --bg --offline)
 
           # Publish the packages
           poetry run packse publish --anonymous --index-url "$index_url" dist/*

README.md

@@ -103,6 +103,14 @@ packse index up
 
 The `--bg` flag can be passed to run the index in the background.
 
+Two package indexes are created:
+
+- `packages/local`: which only allows locally published packages to be installed
+- `packages/all`: which includes local packages but allows missing packages to be pulled from PyPI
+
+When publishing scenario packages, you should always use the `packages/local` package index; or they will not be
+available when installing from `packages/all`.
+
 When running an index in the background, state will be stored in the `~/.packse` directory. The `PACKSE_STORAGE_PATH`
 environment variable can be used to override the storage directory. Additionally, `--storage-path` can be passed
 to the `index` command to change the storage path of server state.

src/packse/cli.py

@@ -76,6 +76,8 @@ def _call_index_up(args):
         reset=args.reset,
         storage_path=args.storage_path,
         background=args.bg,
+        offline=args.offline,
+        all=args.all,
     )
 
 
@@ -238,6 +240,16 @@ def _add_index_parser(subparsers):
         action="store_true",
         help="Run the index server in the background, exiting after it is ready.",
     )
+    up.add_argument(
+        "--offline",
+        action="store_true",
+        help="Run the all index servers without acccess to the real PyPI.",
+    )
+    up.add_argument(
+        "--all",
+        action="store_true",
+        help="Output the index server URL that allows fallback to the real PyPI.",
+    )
     up.set_defaults(call=_call_index_up)
 
     down = subparsers.add_parser("down", help="Stop a running package index server.")

src/packse/index.py

@@ -10,6 +10,7 @@
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
+from packse import __development_base_path__
 from packse.error import ServeAddressInUse, ServeAlreadyRunning
 
 logger = logging.getLogger(__name__)
@@ -21,6 +22,8 @@ def index_up(
     background: bool = False,
     storage_path: Path | None = None,
     reset: bool = False,
+    offline: bool = False,
+    all: bool = False,
 ):
     server_url = f"http://{host}:{port}"
 
@@ -60,7 +63,12 @@ def index_up(
 
         logger.info("Starting server at %s...", server_url)
         with start_index_server(
-            storage_path, server_storage, host, port, server_log_path
+            storage_path,
+            server_storage,
+            host,
+            port,
+            server_log_path,
+            offline=offline,
         ) as server_process:
             init_client(client_storage, server_url)
 
@@ -77,16 +85,43 @@ def index_up(
             logger.debug("Logging in...")
             login_user(username, password, client_storage)
 
-            index = "packages/all"
-            create_index(index, client_storage, exists_ok=background)
+            all_index = "packages/all"
+            local_index = "packages/local"
+            pypi_index = "root/pypi"
+
+            # First, create the "local" index which does not allow fallback to PyPI
+            create_index(local_index, client_storage, exists_ok=background, bases=[])
+
+            # Then, create the "all" index which pulls from the "local" index or PyPI
+            create_index(
+                all_index,
+                client_storage,
+                exists_ok=background,
+                bases=[local_index, pypi_index],
+            )
+
+            logger.info("Ensuring local index has build dependencies...")
+            add_build_requirements(local_index, client_storage)
+
+            all_index_url = f"{server_url}/{all_index}"
+            local_index_url = f"{server_url}/{local_index}"
+            logger.info(
+                "Indexes available at %s and %s", all_index_url, local_index_url
+            )
 
-            index_url = f"{server_url}/{index}"
-            logger.info("Index available at %s", index_url)
+            logger.debug(
+                "To use `devpi` commands, include `--clientdir %s`", client_storage
+            )
 
             if background:
                 logger.info("Running in background with pid %s", server_process.pid)
                 logger.info("Stop index server with `packse index down`.")
-                print(index_url)
+
+                if all:
+                    print(all_index_url)
+                else:
+                    print(local_index_url)
+
             else:
                 logger.info("Ready! [Stop with Ctrl-C]")
 
@@ -147,21 +182,26 @@ def start_index_server(
     host: str,
     port: int,
     server_log_path: Path,
+    offline: bool,
 ) -> subprocess.Popen:
     server_url = f"http://{host}:{port}"
+    command = [
+        "devpi-server",
+        "--serverdir",
+        str(server_storage),
+        # Future default, let's just opt-in now for better output
+        "--absolute-urls",
+        "--host",
+        host,
+        "--port",
+        str(port),
+    ]
+
+    if offline:
+        command.append("--offline")
 
     server_process = subprocess.Popen(
-        [
-            "devpi-server",
-            "--serverdir",
-            str(server_storage),
-            # Future default, let's just opt-in now for better output
-            "--absolute-urls",
-            "--host",
-            host,
-            "--port",
-            str(port),
-        ],
+        command,
         stderr=subprocess.STDOUT,
         stdout=server_log_path.open("wb") if server_log_path else subprocess.PIPE,
     )
@@ -187,6 +227,10 @@ def start_index_server(
     except BaseException as exc:
         server_process.kill()
 
+        stdout, _ = server_process.communicate(timeout=2)
+        if logger.getEffectiveLevel() <= logging.DEBUG:
+            print(stdout.decode())
+
         # Do not reset the pidfile when a server was already running!
         if not isinstance(exc, ServeAddressInUse):
             reset_pidfile(storage_path)
@@ -222,21 +266,51 @@ def init_client(client_storage: Path, server_url: str):
     )
 
 
-def create_index(name: str, client_storage: Path, exists_ok: bool = False):
+def add_build_requirements(
+    to_index: str,
+    client_storage: Path,
+):
+    """
+    Pushes package build requirements to an index that does not allow fallback to PyPI.
+    """
+    build_directory = __development_base_path__ / "vendor" / "build"
+    logger.debug("Uploading build packages to index %s", to_index)
+    subprocess.check_output(
+        [
+            "devpi",
+            "upload",
+            "--clientdir",
+            str(client_storage),
+            "--from-dir",
+            str(build_directory.resolve()),
+            "--index",
+            to_index,
+        ]
+    )
+
+
+def create_index(
+    name: str,
+    client_storage: Path,
+    bases: list[str],
+    exists_ok: bool = False,
+):
     logger.info("Creating package index %r...", name)
+    command = [
+        "devpi",
+        "index",
+        "--clientdir",
+        str(client_storage),
+        "-c",
+        name,
+        "volatile=False",
+        "acl_upload=:ANONYMOUS:",  # Do not require auth to upload packages
+    ]
+    if bases:
+        command.append("bases={}".format(",".join(bases)))
     try:
         subprocess.check_output(
-            [
-                "devpi",
-                "index",
-                "--clientdir",
-                str(client_storage),
-                "-c",
-                name,
-                "bases=root/pypi",  # TODO(zanieb): Allow users to disable pull from real PyPI
-                "volatile=False",
-                "acl_upload=:ANONYMOUS:",  # Do not require auth to upload packages
-            ],
+            command,
             stderr=subprocess.STDOUT,
         )
     except subprocess.CalledProcessError as exc:

tests/__snapshots__/test_index.ambr

@@ -48,14 +48,16 @@
       Initializing server...
       Starting server at http://localhost:3141...
       Configuring client...
+      Creating package index 'packages/local'...
       Creating package index 'packages/all'...
-      Index available at http://localhost:3141/packages/all
+      Ensuring local index has build dependencies...
+      Indexes available at http://localhost:3141/packages/all and http://localhost:3141/packages/local
       Running in background with pid [PID]
       Stop index server with `packse index down`.
 
     ''',
     'stdout': '''
-      http://localhost:3141/packages/all
+      http://localhost:3141/packages/local
 
     ''',
   })
@@ -67,8 +69,10 @@
       Initializing server...
       Starting server at http://localhost:3141...
       Configuring client...
+      Creating package index 'packages/local'...
       Creating package index 'packages/all'...
-      Index available at http://localhost:3141/packages/all
+      Ensuring local index has build dependencies...
+      Indexes available at http://localhost:3141/packages/all and http://localhost:3141/packages/local
       Ready! [Stop with Ctrl-C]
       Interrupted!
 
@@ -83,14 +87,16 @@
       Initializing server...
       Starting server at http://localhost:3141...
       Configuring client...
+      Creating package index 'packages/local'...
       Creating package index 'packages/all'...
-      Index available at http://localhost:3141/packages/all
+      Ensuring local index has build dependencies...
+      Indexes available at http://localhost:3141/packages/all and http://localhost:3141/packages/local
       Running in background with pid [PID]
       Stop index server with `packse index down`.
 
     ''',
     'stdout': '''
-      http://localhost:3141/packages/all
+      http://localhost:3141/packages/local
 
     ''',
   })

tests/common.py

@@ -34,8 +34,8 @@ def snapshot_command(
         (re.escape(str(Path(sys.executable).parent)), "[PYTHON_BINDIR]"),
         (re.escape(str((working_directory or Path.cwd()).absolute())), "[PWD]"),
         (
-            r"(\d+\.)?\d+(ms|s)",
-            "[TIME]",
+            r"in (\d+\.)?\d+(ms|s)",
+            "in [TIME]",
         ),
         (re.escape(str(__development_base_path__.absolute())), "[PROJECT_ROOT]"),
     ]

tests/test_index.py

@@ -1,3 +1,4 @@
+import os
 import subprocess
 
 import psutil
@@ -44,7 +45,7 @@ def test_index_up_background(snapshot, tmpcwd, tmpenviron):
             == snapshot
         )
     finally:
-        subprocess.call(["packse", "index", "down"])
+        subprocess.call(["packse", "index", "down"], env=tmpenviron)
 
 
 def test_index_up_foreground(snapshot, tmpcwd, tmpenviron):
@@ -54,7 +55,8 @@ def test_index_up_foreground(snapshot, tmpcwd, tmpenviron):
         snapshot_command(
             ["index", "up"],
             extra_filters=FILTERS,
-            interrupt_after=5,
+            # Send a keyboard interrupt after a bit — use a longer delay for slow CI machines
+            interrupt_after=10 if os.environ.get("CI") else 5,
         )
         == snapshot
     )

vendor/build/README.md

@@ -0,0 +1,4 @@
+# Vendored build dependencies
+
+This directoy contains vendored build dependencies for running a package index in offline mode while allowing
+package scenarios to be built and installed.