added parsing, fixed linting, formatting & tests

homcc/server/cache.py

@@ -11,10 +11,6 @@
 logger = logging.getLogger(__name__)
 
 
-def mib_to_bytes(mb: int) -> int:
-    return mb * 1024**2
-
-
 class Cache:
     """Represents the homcc server cache that is used to cache dependencies."""
 
@@ -25,9 +21,9 @@ class Cache:
     cache_folder: Path
     """Path to the cache on the file system."""
     max_size_bytes: int
-    """Maximum size in bytes of the cache."""
+    """Maximum size of the cache in bytes."""
     current_size: int
-    """Current size in bytes"""
+    """Current size of the cache in bytes."""
 
     def __init__(self, root_folder: Path, max_size_bytes: int):
         if max_size_bytes <= 0:
@@ -39,8 +35,8 @@ def __init__(self, root_folder: Path, max_size_bytes: int):
         self.max_size_bytes = max_size_bytes
         self.current_size = 0
 
-    def _get_cache_file_path(self, hash: str) -> Path:
-        return self.cache_folder / hash
+    def _get_cache_file_path(self, hash_value: str) -> Path:
+        return self.cache_folder / hash_value
 
     def __contains__(self, key: str):
         with self.cache_mutex:
@@ -84,13 +80,13 @@ def _create_cache_folder(root_temp_folder: Path) -> Path:
         logger.info("Created cache folder in '%s'.", cache_folder.absolute())
         return cache_folder
 
-    def get(self, hash: str) -> str:
+    def get(self, hash_value: str) -> str:
         """Gets an entry (path) from the cache given a hash."""
         with self.cache_mutex:
-            self.cache.move_to_end(hash)
-            return self.cache[hash]
+            self.cache.move_to_end(hash_value)
+            return self.cache[hash_value]
 
-    def put(self, hash: str, content: bytearray):
+    def put(self, hash_value: str, content: bytearray):
         """Stores a dependency in the cache."""
         if len(content) > self.max_size_bytes:
             logger.error(
@@ -102,11 +98,11 @@ def put(self, hash: str, content: bytearray):
             )
             raise RuntimeError("Cache size insufficient")
 
-        cached_file_path = self._get_cache_file_path(hash)
+        cached_file_path = self._get_cache_file_path(hash_value)
         with self.cache_mutex:
             while self.current_size + len(content) > self.max_size_bytes:
                 self._evict_oldest()
 
             Path.write_bytes(cached_file_path, content)
             self.current_size += len(content)
-            self.cache[hash] = str(cached_file_path)
+            self.cache[hash_value] = str(cached_file_path)

homcc/server/main.py

@@ -67,6 +67,9 @@ def main():
         level=LogLevel.INFO,
     )
 
+    # TODO(o.layer): The argument parsing code should below be moved to/abstracted in parsing.py,
+    # similar to how it is done for the client
+
     # LOG_LEVEL and VERBOSITY
     log_level: Optional[str] = homccd_args_dict["log_level"]
 
@@ -100,6 +103,10 @@ def main():
     if (address := homccd_args_dict["listen"]) is not None:
         homccd_config.address = address
 
+    # MAX_DEPENDENCY_CACHE_SIZE_BYTES
+    if (max_dependency_cache_size_bytes := homccd_args_dict["max_dependency_cache_size_bytes"]) is not None:
+        homccd_config.max_dependency_cache_size_bytes = max_dependency_cache_size_bytes
+
     # provide additional DEBUG information
     logger.debug(
         "%s - %s\n" "Caller:\t%s\n" "%s",  # homccd location and version; homccd caller; config info

homcc/server/parsing.py

@@ -22,6 +22,11 @@
 
 logger = logging.getLogger(__name__)
 
+
+def mib_to_bytes(mb: int) -> int:
+    return mb * 1024**2
+
+
 HOMCC_SERVER_CONFIG_SECTION: str = "homccd"
 
 DEFAULT_ADDRESS: str = "0.0.0.0"
@@ -31,6 +36,7 @@
     or os.cpu_count()  # total number of physical CPUs on the machine
     or -1  # fallback error value
 )
+DEFAULT_MAX_CACHE_SIZE_BYTES: int = mib_to_bytes(10000)
 
 
 class ShowVersion(Action):
@@ -74,6 +80,7 @@ class EnvironmentVariables:
         HOMCCD_ADDRESS_ENV_VAR: ClassVar[str] = "HOMCCD_ADDRESS"
         HOMCCD_LOG_LEVEL_ENV_VAR: ClassVar[str] = "HOMCCD_LOG_LEVEL"
         HOMCCD_VERBOSE_ENV_VAR: ClassVar[str] = "HOMCCD_VERBOSE"
+        HOMCCD_MAX_DEPENDENCY_CACHE_SIZE_BYTES: ClassVar[str] = "HOMCCD_MAX_DEPENDENCY_CACHE_SIZE_BYTES"
 
         @classmethod
         def __iter__(cls) -> Iterator[str]:
@@ -83,6 +90,7 @@ def __iter__(cls) -> Iterator[str]:
                 cls.HOMCCD_ADDRESS_ENV_VAR,
                 cls.HOMCCD_LOG_LEVEL_ENV_VAR,
                 cls.HOMCCD_VERBOSE_ENV_VAR,
+                cls.HOMCCD_MAX_DEPENDENCY_CACHE_SIZE_BYTES,
             )
 
         @classmethod
@@ -112,12 +120,20 @@ def get_verbose(cls) -> Optional[bool]:
                 return re.match(r"^(1)|(yes)|(true)|(on)$", verbose, re.IGNORECASE) is not None
             return None
 
+        @classmethod
+        def get_max_dependency_cache_size_bytes(cls) -> Optional[int]:
+            if max_dependency_cache_size_bytes := os.getenv(cls.HOMCCD_MAX_DEPENDENCY_CACHE_SIZE_BYTES):
+                return int(max_dependency_cache_size_bytes)
+
+            return None
+
     files: List[str]
     address: Optional[str]
     port: Optional[int]
     limit: Optional[int]
     log_level: Optional[LogLevel]
     verbose: bool
+    max_dependency_cache_size_bytes: Optional[int]
 
     def __init__(
         self,
@@ -128,6 +144,7 @@ def __init__(
         address: Optional[str] = None,
         log_level: Optional[str] = None,
         verbose: Optional[bool] = None,
+        max_dependency_cache_size_bytes: Optional[int] = None,
     ):
         self.files = files
 
@@ -140,6 +157,8 @@ def __init__(
         verbose = self.EnvironmentVariables.get_verbose() or verbose
         self.verbose = verbose is not None and verbose
 
+        self.max_dependency_cache_size_bytes = max_dependency_cache_size_bytes
+
     @classmethod
     def empty(cls):
         return cls(files=[])
@@ -151,8 +170,17 @@ def from_config_section(cls, files: List[str], homccd_config: SectionProxy) -> S
         address: Optional[str] = homccd_config.get("address")
         log_level: Optional[str] = homccd_config.get("log_level")
         verbose: Optional[bool] = homccd_config.getboolean("verbose")
-
-        return ServerConfig(files=files, limit=limit, port=port, address=address, log_level=log_level, verbose=verbose)
+        max_dependency_cache_size_bytes: Optional[int] = homccd_config.getint("max_dependency_cache_size_bytes")
+
+        return ServerConfig(
+            files=files,
+            limit=limit,
+            port=port,
+            address=address,
+            log_level=log_level,
+            verbose=verbose,
+            max_dependency_cache_size_bytes=max_dependency_cache_size_bytes,
+        )
 
     def __str__(self):
         return (
@@ -162,6 +190,7 @@ def __str__(self):
             f"\taddress:\t{self.address}\n"
             f"\tlog_level:\t{self.log_level}\n"
             f"\tverbose:\t{self.verbose}\n"
+            f"\tmax_dependency_cache_size_bytes:\t{self.max_dependency_cache_size_bytes}\n"
         )
 
 
@@ -181,6 +210,14 @@ def min_job_limit(value: Union[int, str], minimum: int = 0) -> int:
 
         raise ArgumentTypeError(f"LIMIT must be more than {minimum}")
 
+    def max_dependency_cache_size_bytes(value: Union[int, str]) -> int:
+        value = int(value)
+
+        if value <= 0:
+            raise ArgumentTypeError("Maximum dependency cache size must be larger than 0.")
+
+        return value
+
     general_options_group = parser.add_argument_group("Options")
     networking_group = parser.add_argument_group(" Networking")
     debug_group = parser.add_argument_group(" Debug")
@@ -206,6 +243,13 @@ def min_job_limit(value: Union[int, str], minimum: int = 0) -> int:
         action="store_true",
         help="enforce that only configurations provided via the CLI are used",
     )
+    general_options_group.add_argument(
+        "--max-dependency-cache-size-bytes",
+        required=False,
+        metavar="BYTES",
+        type=max_dependency_cache_size_bytes,
+        help=f"The maximum cache size for the dependency cache in bytes. Default: {DEFAULT_MAX_CACHE_SIZE_BYTES} bytes",
+    )
 
     # networking
     networking_group.add_argument(

homcc/server/server.py

@@ -41,6 +41,7 @@
 from homcc.server.parsing import (
     DEFAULT_ADDRESS,
     DEFAULT_LIMIT,
+    DEFAULT_MAX_CACHE_SIZE_BYTES,
     DEFAULT_PORT,
     ServerConfig,
 )
@@ -56,7 +57,9 @@
 class TCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
     """TCP Server instance, holding data relevant across compilations."""
 
-    def __init__(self, address: Optional[str], port: Optional[int], limit: Optional[int]):
+    def __init__(
+        self, address: Optional[str], port: Optional[int], limit: Optional[int], max_cache_size_bytes: Optional[int]
+    ):
         address = address or DEFAULT_ADDRESS
         port = port or DEFAULT_PORT
 
@@ -77,7 +80,8 @@ def __init__(self, address: Optional[str], port: Optional[int], limit: Optional[
         self.current_amount_connections: int = 0  # indicates the amount of clients that are currently connected
         self.current_amount_connections_mutex: Lock = Lock()
 
-        self.cache = Cache(root_folder=Path(self.root_temp_folder.name), max_entries=1000)  # TODO
+        max_cache_size_bytes = max_cache_size_bytes or DEFAULT_MAX_CACHE_SIZE_BYTES
+        self.cache = Cache(root_folder=Path(self.root_temp_folder.name), max_size_bytes=max_cache_size_bytes)
 
     @staticmethod
     def send_message(request, message: Message):
@@ -518,7 +522,7 @@ def handle(self):
 
 def start_server(config: ServerConfig) -> Tuple[TCPServer, threading.Thread]:
     try:
-        server: TCPServer = TCPServer(config.address, config.port, config.limit)
+        server: TCPServer = TCPServer(config.address, config.port, config.limit, config.max_dependency_cache_size_bytes)
     except OSError as err:
         logger.error("Could not start TCP server: %s", err)
         raise ServerInitializationError from err

tests/server/environment_test.py

@@ -135,9 +135,8 @@ def test_caching(self, mocker: MockerFixture):
         environment = create_mock_environment("", "")
         # pylint: disable=protected-access
         Cache._create_cache_folder = lambda *_: None  # type: ignore
-        cache = Cache(Path(""))
-        cache.cache = {"hash2": "some/path/to/be/linked"}
-
+        cache = Cache(Path(""), 1024)
+        cache.cache["hash2"] = "some/path/to/be/linked"
         needed_dependencies = environment.get_needed_dependencies(dependencies, cache)
 
         assert len(needed_dependencies) == 2