Merge pull request #1 from TJCSec/feat/zip-provide

feat!: add zip provide

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "rcds"
-version = "0.2.0"
+version = "0.3.0"
 description = "An automated CTF challenge deployment tool"
 readme = "README.rst"
 authors = ["redpwn <[email protected]>"]
@@ -15,13 +15,13 @@ packages = [
 rcds = "rcds.cli:cli"
 
 [tool.poetry.dependencies]
-python = "^3.6"
+python = ">=3.7,<4.0"
 pyyaml = "^5.3.1"
-pathspec = "^0.8.1"
+pathspec = "^0.9.0"
 docker = "^4.3.1"
 jsonschema = "^3.2.0"
 dataclasses = { version = ">=0.7,<0.9", python = "~3.6" }
-Jinja2 = "^2.11.2"
+Jinja2 = "^3.1.1"
 kubernetes = "^21.7.0"
 requests = "^2.24.0"
 requests-toolbelt = "^0.9.1"

rcds/backends/k8s/backend.py

@@ -101,7 +101,9 @@ def render_and_append(env: Environment, template: str) -> None:
                 "config": container_config,
             }
             if expose_config is not None:
-                container_env.globals["container"]["expose"] = expose_config
+                container_env.globals["container"][
+                    "expose"
+                ] = expose_config  # type: ignore
 
             render_and_append(container_env, "deployment.yaml")
             render_and_append(container_env, "service.yaml")

rcds/challenge/challenge.py

@@ -1,6 +1,11 @@
+import base64
+import io
 import re
+import zipfile
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, Dict, List, cast
+from typing import TYPE_CHECKING, Any, Callable, Dict, cast
+
+import pathspec  # type: ignore
 
 from ..util import SUPPORTED_EXTENSIONS, deep_merge, find_files
 from .config import ConfigLoader
@@ -60,7 +65,9 @@ class Challenge:
     config: Dict[str, Any]
     context: Dict[str, Any]  # overrides to Jinja context
     _asset_manager_context: "AssetManagerContext"
-    _asset_sources: List[Callable[["AssetManagerTransaction"], None]]
+    _asset_sources: Dict[
+        str, Callable[["AssetManagerTransaction", Dict[str, Any]], None]
+    ]
 
     def __init__(self, project: "Project", root: Path, config: dict):
         self.project = project
@@ -70,37 +77,94 @@ def __init__(self, project: "Project", root: Path, config: dict):
         self._asset_manager_context = self.project.asset_manager.create_context(
             self.config["id"]
         )
-        self._asset_sources = []
-
-        self.register_asset_source(self._add_static_assets)
+        self._asset_sources = {}
+
+        self.register_asset_source("file", self._add_file_asset)
+        self.register_asset_source("zip", self._add_zip_asset)
+
+    # def _add_static_assets(self, transaction: "AssetManagerTransaction") -> None:
+    #     if "provide" not in self.config:
+    #         return
+    #     for provide in self.config["provide"]:
+    #         if isinstance(provide, str):
+    #             path = self.root / Path(provide)
+    #             name = path.name
+    #         else:
+    #             path = self.root / Path(provide["file"])
+    #             name = provide["as"]
+    #         transaction.add_file(name, path)
+
+    def _add_file_asset(
+        self, transaction: "AssetManagerTransaction", spec: Dict[str, Any]
+    ) -> None:
+        path = self.root / Path(spec["file"])
+        transaction.add_file(spec["as"], path)
 
-    def _add_static_assets(self, transaction: "AssetManagerTransaction") -> None:
-        if "provide" not in self.config:
-            return
-        for provide in self.config["provide"]:
-            if isinstance(provide, str):
-                path = self.root / Path(provide)
-                name = path.name
-            else:
-                path = self.root / Path(provide["file"])
-                name = provide["as"]
-            transaction.add_file(name, path)
+    def _add_zip_asset(
+        self, transaction: "AssetManagerTransaction", spec: Dict[str, Any]
+    ) -> None:
+        exclude: pathspec.PathSpec = None
+        base: Path = self.root
+        if "base" in spec:
+            base = base / spec["base"]
+        if "exclude" in spec:
+            exclude = pathspec.PathSpec.from_lines("gitwildmatch", spec["exclude"])
+        buf: io.BytesIO = io.BytesIO()
+        mtime: float = 0.0
+        with zipfile.ZipFile(buf, "w") as zf:
+
+            def add(path: Path):
+                nonlocal mtime
+                if exclude is not None and exclude.match_file(path.relative_to(base)):
+                    return
+                if path.is_file():
+                    mtime = max(mtime, path.stat().st_mtime)
+                    zf.write(path, path.relative_to(base), zipfile.ZIP_DEFLATED)
+                elif path.is_dir():
+                    zf.write(path, path.relative_to(base), zipfile.ZIP_STORED)
+                    for nm in path.iterdir():
+                        add(nm)
+
+            for glob in spec["files"]:
+                for path in base.glob(glob):
+                    add(path)
+
+            if "additional" in spec:
+                for additional in spec["additional"]:
+                    if "str" in additional:
+                        content = additional["str"]
+                    elif "base64" in additional:
+                        content = base64.b64decode(additional["base64"])
+                    else:
+                        raise ValueError(
+                            "Either `str` or `base64` is required in `additional`"
+                        )
+                    zf.writestr(additional["path"], content, zipfile.ZIP_DEFLATED)
+        transaction.add(spec["as"], mtime, buf.getvalue())
 
     def register_asset_source(
-        self, do_add: Callable[["AssetManagerTransaction"], None]
+        self,
+        kind: str,
+        do_add: Callable[["AssetManagerTransaction", Dict[str, Any]], None],
     ) -> None:
         """
         Register a function to add assets to the transaction for this challenge.
         """
-        self._asset_sources.append(do_add)
+        self._asset_sources[kind] = do_add
 
     def create_transaction(self) -> "AssetManagerTransaction":
         """
         Get a transaction to update this challenge's assets
         """
         transaction = self._asset_manager_context.transaction()
-        for do_add in self._asset_sources:
-            do_add(transaction)
+        if "provide" not in self.config:
+            return transaction
+        for provide in self.config["provide"]:
+            if isinstance(provide, str):
+                path = self.root / Path(provide)
+                transaction.add_file(path.name, path)
+            else:
+                self._asset_sources[provide["kind"]](transaction, provide["spec"])
         return transaction
 
     def get_asset_manager_context(self) -> "AssetManagerContext":

rcds/challenge/challenge.schema.yaml

@@ -91,17 +91,84 @@ properties:
           Path to the file to provide
       - type: object
         properties:
-          file:
-            type: string
-            description: >-
-              Path to the file to provide
-          as:
-            type: string
-            description: >-
-              Name of file as shown to competitors
+          kind:
+            const: file
+          spec:
+            type: object
+            properties:
+              file:
+                type: string
+                description: >-
+                  Path to the file to provide
+              as:
+                type: string
+                description: >-
+                  Name of file as shown to competitors
+            required:
+              - file
+              - as
+        required:
+          - kind
+          - spec
+      - type: object
+        properties:
+          kind:
+            const: zip
+          spec:
+            type: object
+            properties:
+              as:
+                type: string
+                description: >-
+                  Name of file as shown to competitors
+              base:
+                type: string
+                description: >-
+                  Base directory for archiving
+              files:
+                type: array
+                description: >-
+                  Files to provide in archive
+                items:
+                  type: string
+                  description: >-
+                    Glob of file(s) to provide
+              additional:
+                type: array
+                description: >-
+                  Additional files to include in archive
+                items:
+                  type: object
+                  properties:
+                    path:
+                      type: string
+                      description: >-
+                        Path in archive
+                    str:
+                      type: string
+                      description: >-
+                        Additional file contents, as raw string
+                    base64:
+                      type: string
+                      description: >-
+                        Additional file contents, as base64 string
+                  required:
+                    - path
+                  anyOf:
+                    - required: ["str"]
+                    - required: ["base64"]
+              exclude:
+                type: array
+                description: >-
+                  Pathspecs to exclude from archive
+                items:
+                  type: string
+            required:
+              - files
+              - as
         required:
-        - file
-        - as
+          - kind
+          - spec
 
   # Runtime (containers)
   deployed:

rcds/challenge/config.py

@@ -166,8 +166,10 @@ def parse_config(
                 for f in config["provide"]:
                     if isinstance(f, str):
                         f = Path(f)
+                    elif f["kind"] == "file":
+                        f = Path(f["spec"]["file"])
                     else:
-                        f = Path(f["file"])
+                        continue
                     if not (root / f).is_file():
                         yield TargetFileNotFoundError(
                             f'`provide` references file "{str(f)}" which does not '

tests/challenge/test_challenge/static-assets/challenge.yml

@@ -3,5 +3,7 @@ description: description
 
 provide:
 - ./file1.txt
-- file: ./file2.txt
-  as: file3.txt
+- kind: file
+  spec:
+    file: ./file2.txt
+    as: file3.txt

tests/challenge/test_config/valid/challenge.yml

@@ -10,8 +10,10 @@ description: |
 
 provide:
 - ./flag.txt
-- file: ./flag.txt
-  as: other-file.txt
+- kind: file
+  spec:
+    file: ./flag.txt
+    as: other-file.txt
 
 containers:
   main: