From 0f71589f3d9f8c896a36a09765a7d3efe775be43 Mon Sep 17 00:00:00 2001 From: Oliver Layer Date: Wed, 25 Oct 2023 09:58:10 +0200 Subject: [PATCH] added parsing, fixed linting, formatting & tests --- homcc/server/cache.py | 24 +++++++--------- homcc/server/main.py | 7 +++++ homcc/server/parsing.py | 48 ++++++++++++++++++++++++++++++-- homcc/server/server.py | 10 +++++-- tests/server/environment_test.py | 5 ++-- 5 files changed, 72 insertions(+), 22 deletions(-) diff --git a/homcc/server/cache.py b/homcc/server/cache.py index a0abf97..fca2a30 100644 --- a/homcc/server/cache.py +++ b/homcc/server/cache.py @@ -11,10 +11,6 @@ logger = logging.getLogger(__name__) -def mib_to_bytes(mb: int) -> int: - return mb * 1024**2 - - class Cache: """Represents the homcc server cache that is used to cache dependencies.""" @@ -25,9 +21,9 @@ class Cache: cache_folder: Path """Path to the cache on the file system.""" max_size_bytes: int - """Maximum size in bytes of the cache.""" + """Maximum size of the cache in bytes.""" current_size: int - """Current size in bytes""" + """Current size of the cache in bytes.""" def __init__(self, root_folder: Path, max_size_bytes: int): if max_size_bytes <= 0: @@ -39,8 +35,8 @@ def __init__(self, root_folder: Path, max_size_bytes: int): self.max_size_bytes = max_size_bytes self.current_size = 0 - def _get_cache_file_path(self, hash: str) -> Path: - return self.cache_folder / hash + def _get_cache_file_path(self, hash_value: str) -> Path: + return self.cache_folder / hash_value def __contains__(self, key: str): with self.cache_mutex: @@ -84,13 +80,13 @@ def _create_cache_folder(root_temp_folder: Path) -> Path: logger.info("Created cache folder in '%s'.", cache_folder.absolute()) return cache_folder - def get(self, hash: str) -> str: + def get(self, hash_value: str) -> str: """Gets an entry (path) from the cache given a hash.""" with self.cache_mutex: - self.cache.move_to_end(hash) - return self.cache[hash] + self.cache.move_to_end(hash_value) + return self.cache[hash_value] - def put(self, hash: str, content: bytearray): + def put(self, hash_value: str, content: bytearray): """Stores a dependency in the cache.""" if len(content) > self.max_size_bytes: logger.error( @@ -102,11 +98,11 @@ def put(self, hash: str, content: bytearray): ) raise RuntimeError("Cache size insufficient") - cached_file_path = self._get_cache_file_path(hash) + cached_file_path = self._get_cache_file_path(hash_value) with self.cache_mutex: while self.current_size + len(content) > self.max_size_bytes: self._evict_oldest() Path.write_bytes(cached_file_path, content) self.current_size += len(content) - self.cache[hash] = str(cached_file_path) + self.cache[hash_value] = str(cached_file_path) diff --git a/homcc/server/main.py b/homcc/server/main.py index 28fa1e0..b70cefe 100755 --- a/homcc/server/main.py +++ b/homcc/server/main.py @@ -67,6 +67,9 @@ def main(): level=LogLevel.INFO, ) + # TODO(o.layer): The argument parsing code should below be moved to/abstracted in parsing.py, + # similar to how it is done for the client + # LOG_LEVEL and VERBOSITY log_level: Optional[str] = homccd_args_dict["log_level"] @@ -100,6 +103,10 @@ def main(): if (address := homccd_args_dict["listen"]) is not None: homccd_config.address = address + # MAX_DEPENDENCY_CACHE_SIZE_BYTES + if (max_dependency_cache_size_bytes := homccd_args_dict["max_dependency_cache_size_bytes"]) is not None: + homccd_config.max_dependency_cache_size_bytes = max_dependency_cache_size_bytes + # provide additional DEBUG information logger.debug( "%s - %s\n" "Caller:\t%s\n" "%s", # homccd location and version; homccd caller; config info diff --git a/homcc/server/parsing.py b/homcc/server/parsing.py index d064e62..4798fb9 100644 --- a/homcc/server/parsing.py +++ b/homcc/server/parsing.py @@ -22,6 +22,11 @@ logger = logging.getLogger(__name__) + +def mib_to_bytes(mb: int) -> int: + return mb * 1024**2 + + HOMCC_SERVER_CONFIG_SECTION: str = "homccd" DEFAULT_ADDRESS: str = "0.0.0.0" @@ -31,6 +36,7 @@ or os.cpu_count() # total number of physical CPUs on the machine or -1 # fallback error value ) +DEFAULT_MAX_CACHE_SIZE_BYTES: int = mib_to_bytes(10000) class ShowVersion(Action): @@ -74,6 +80,7 @@ class EnvironmentVariables: HOMCCD_ADDRESS_ENV_VAR: ClassVar[str] = "HOMCCD_ADDRESS" HOMCCD_LOG_LEVEL_ENV_VAR: ClassVar[str] = "HOMCCD_LOG_LEVEL" HOMCCD_VERBOSE_ENV_VAR: ClassVar[str] = "HOMCCD_VERBOSE" + HOMCCD_MAX_DEPENDENCY_CACHE_SIZE_BYTES: ClassVar[str] = "HOMCCD_MAX_DEPENDENCY_CACHE_SIZE_BYTES" @classmethod def __iter__(cls) -> Iterator[str]: @@ -83,6 +90,7 @@ def __iter__(cls) -> Iterator[str]: cls.HOMCCD_ADDRESS_ENV_VAR, cls.HOMCCD_LOG_LEVEL_ENV_VAR, cls.HOMCCD_VERBOSE_ENV_VAR, + cls.HOMCCD_MAX_DEPENDENCY_CACHE_SIZE_BYTES, ) @classmethod @@ -112,12 +120,20 @@ def get_verbose(cls) -> Optional[bool]: return re.match(r"^(1)|(yes)|(true)|(on)$", verbose, re.IGNORECASE) is not None return None + @classmethod + def get_max_dependency_cache_size_bytes(cls) -> Optional[int]: + if max_dependency_cache_size_bytes := os.getenv(cls.HOMCCD_MAX_DEPENDENCY_CACHE_SIZE_BYTES): + return int(max_dependency_cache_size_bytes) + + return None + files: List[str] address: Optional[str] port: Optional[int] limit: Optional[int] log_level: Optional[LogLevel] verbose: bool + max_dependency_cache_size_bytes: Optional[int] def __init__( self, @@ -128,6 +144,7 @@ def __init__( address: Optional[str] = None, log_level: Optional[str] = None, verbose: Optional[bool] = None, + max_dependency_cache_size_bytes: Optional[int] = None, ): self.files = files @@ -140,6 +157,8 @@ def __init__( verbose = self.EnvironmentVariables.get_verbose() or verbose self.verbose = verbose is not None and verbose + self.max_dependency_cache_size_bytes = max_dependency_cache_size_bytes + @classmethod def empty(cls): return cls(files=[]) @@ -151,8 +170,17 @@ def from_config_section(cls, files: List[str], homccd_config: SectionProxy) -> S address: Optional[str] = homccd_config.get("address") log_level: Optional[str] = homccd_config.get("log_level") verbose: Optional[bool] = homccd_config.getboolean("verbose") - - return ServerConfig(files=files, limit=limit, port=port, address=address, log_level=log_level, verbose=verbose) + max_dependency_cache_size_bytes: Optional[int] = homccd_config.getint("max_dependency_cache_size_bytes") + + return ServerConfig( + files=files, + limit=limit, + port=port, + address=address, + log_level=log_level, + verbose=verbose, + max_dependency_cache_size_bytes=max_dependency_cache_size_bytes, + ) def __str__(self): return ( @@ -162,6 +190,7 @@ def __str__(self): f"\taddress:\t{self.address}\n" f"\tlog_level:\t{self.log_level}\n" f"\tverbose:\t{self.verbose}\n" + f"\tmax_dependency_cache_size_bytes:\t{self.max_dependency_cache_size_bytes}\n" ) @@ -181,6 +210,14 @@ def min_job_limit(value: Union[int, str], minimum: int = 0) -> int: raise ArgumentTypeError(f"LIMIT must be more than {minimum}") + def max_dependency_cache_size_bytes(value: Union[int, str]) -> int: + value = int(value) + + if value <= 0: + raise ArgumentTypeError("Maximum dependency cache size must be larger than 0.") + + return value + general_options_group = parser.add_argument_group("Options") networking_group = parser.add_argument_group(" Networking") debug_group = parser.add_argument_group(" Debug") @@ -206,6 +243,13 @@ def min_job_limit(value: Union[int, str], minimum: int = 0) -> int: action="store_true", help="enforce that only configurations provided via the CLI are used", ) + general_options_group.add_argument( + "--max-dependency-cache-size-bytes", + required=False, + metavar="BYTES", + type=max_dependency_cache_size_bytes, + help=f"The maximum cache size for the dependency cache in bytes. Default: {DEFAULT_MAX_CACHE_SIZE_BYTES} bytes", + ) # networking networking_group.add_argument( diff --git a/homcc/server/server.py b/homcc/server/server.py index 74c3bf6..0e7bd93 100644 --- a/homcc/server/server.py +++ b/homcc/server/server.py @@ -41,6 +41,7 @@ from homcc.server.parsing import ( DEFAULT_ADDRESS, DEFAULT_LIMIT, + DEFAULT_MAX_CACHE_SIZE_BYTES, DEFAULT_PORT, ServerConfig, ) @@ -56,7 +57,9 @@ class TCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): """TCP Server instance, holding data relevant across compilations.""" - def __init__(self, address: Optional[str], port: Optional[int], limit: Optional[int]): + def __init__( + self, address: Optional[str], port: Optional[int], limit: Optional[int], max_cache_size_bytes: Optional[int] + ): address = address or DEFAULT_ADDRESS port = port or DEFAULT_PORT @@ -77,7 +80,8 @@ def __init__(self, address: Optional[str], port: Optional[int], limit: Optional[ self.current_amount_connections: int = 0 # indicates the amount of clients that are currently connected self.current_amount_connections_mutex: Lock = Lock() - self.cache = Cache(root_folder=Path(self.root_temp_folder.name), max_entries=1000) # TODO + max_cache_size_bytes = max_cache_size_bytes or DEFAULT_MAX_CACHE_SIZE_BYTES + self.cache = Cache(root_folder=Path(self.root_temp_folder.name), max_size_bytes=max_cache_size_bytes) @staticmethod def send_message(request, message: Message): @@ -518,7 +522,7 @@ def handle(self): def start_server(config: ServerConfig) -> Tuple[TCPServer, threading.Thread]: try: - server: TCPServer = TCPServer(config.address, config.port, config.limit) + server: TCPServer = TCPServer(config.address, config.port, config.limit, config.max_dependency_cache_size_bytes) except OSError as err: logger.error("Could not start TCP server: %s", err) raise ServerInitializationError from err diff --git a/tests/server/environment_test.py b/tests/server/environment_test.py index a4f0b2c..6ac9c5d 100644 --- a/tests/server/environment_test.py +++ b/tests/server/environment_test.py @@ -135,9 +135,8 @@ def test_caching(self, mocker: MockerFixture): environment = create_mock_environment("", "") # pylint: disable=protected-access Cache._create_cache_folder = lambda *_: None # type: ignore - cache = Cache(Path("")) - cache.cache = {"hash2": "some/path/to/be/linked"} - + cache = Cache(Path(""), 1024) + cache.cache["hash2"] = "some/path/to/be/linked" needed_dependencies = environment.get_needed_dependencies(dependencies, cache) assert len(needed_dependencies) == 2