Move configuration functions to separate file

close #14
LivInTheLookingGlass · May 9, 2021 · c37ef8c · c37ef8c
1 parent d8e735e
commit c37ef8c
Show file tree

Hide file tree

Showing 3 changed files with 110 additions and 107 deletions.
diff --git a/example_main.py b/example_main.py
@@ -2,7 +2,8 @@
 from logging.handlers import RotatingFileHandler
 from typing import Type
 
-from src.framework import get_config, make_config_files, run_jobs, _sleeper
+from src.config import get_config, make_config_files
+from src.framework import run_jobs, _sleeper
 from src.zipped_logs import ZippedRotatingFileHandler
 
 

diff --git a/src/config.py b/src/config.py
@@ -0,0 +1,105 @@
+"""Module that deals with all configuration related functions."""
+from collections import OrderedDict
+from configparser import ConfigParser, ExtendedInterpolation
+from hashlib import sha3_512
+from json import JSONDecoder
+from logging import CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET
+from pathlib import Path
+from pickle import dumps
+from typing import TypeVar
+
+
+T = TypeVar("T")
+
+MACHINES_CONFIG = Path('machines.json')
+CSV_HEADER = Path('header.csv')
+RUNNER_CONFIG = Path('runner.config')
+
+CONFIGS = [
+    MACHINES_CONFIG,
+    CSV_HEADER,
+    RUNNER_CONFIG,
+]
+DEFAULT_CONFIG_PATH = Path(__file__).parent.joinpath('defaults', 'runner.config')
+RESULTS_CSV = Path('results.csv')
+
+CONFIG_TEMPLATES = [
+    '{\n\t"example_machine": [example_one_core_weight, example_cores_used]\n}',
+    '',
+    DEFAULT_CONFIG_PATH.read_text()
+]
+
+
+def get_entropy(*extras) -> bytes:
+    """Generate bytes for use as a seed.
+
+    Include additional objects only if they will be consistent from machine to machine.
+    """
+    hash_obj = sha3_512()
+    for name in CONFIGS:
+        hash_obj.update(name.read_bytes())
+    hash_obj.update(dumps(extras))
+    return hash_obj.digest()
+
+
+def make_config_files():
+    """Generate config files if not present, raising an error if any were not present."""
+    any_triggered = False
+    for name, default in zip(CONFIGS, CONFIG_TEMPLATES):
+        try:
+            with name.open('x') as f:
+                f.write(default)
+        except FileExistsError:
+            pass
+        else:
+            any_triggered = True
+    if any_triggered:
+        raise FileNotFoundError("Your configuration files have been created. Please fill them out.")
+
+
+def get_machines():  # -> OrderedDict[str, Tuple[int, int]]:
+    """Read the list of machine names and their associated thread weights and number of threads."""
+    decoder = JSONDecoder(object_pairs_hook=OrderedDict)
+    with MACHINES_CONFIG.open('r') as f:
+        return decoder.decode(f.read())
+
+
+def parse_file_size(value: str) -> int:
+    """Parse a string to return an integer file size."""
+    value = value.lower()
+    if value.endswith('t') or value.endswith('tib'):
+        return int(value.rstrip('t').rstrip('tib')) << 40
+    elif value.endswith('g') or value.endswith('gib'):
+        return int(value.rstrip('g').rstrip('gib')) << 30
+    elif value.endswith('m') or value.endswith('mib'):
+        return int(value.rstrip('m').rstrip('mib')) << 20
+    elif value.endswith('k') or value.endswith('kib'):
+        return int(value.rstrip('k').rstrip('kib')) << 10
+    elif value.endswith('tb'):
+        return int(value[:2]) * 10**12
+    elif value.endswith('gb'):
+        return int(value[:2]) * 10**9
+    elif value.endswith('mb'):
+        return int(value[:2]) * 10**6
+    elif value.endswith('kb'):
+        return int(value[:2]) * 10**3
+    return int(value)
+
+
+def get_config() -> ConfigParser:
+    """Read and parse the configuration file."""
+    config = ConfigParser(interpolation=ExtendedInterpolation(), converters={'filesize': parse_file_size})
+    with DEFAULT_CONFIG_PATH.open('r') as f:
+        config.read_file(f)
+    config.read(RUNNER_CONFIG)
+
+    config['logging']['level'] = {
+        'CRITICAL': str(CRITICAL),
+        'ERROR': str(ERROR),
+        'WARNING': str(WARNING),
+        'INFO': str(INFO),
+        'DEBUG': str(DEBUG),
+        'NOTSET': str(NOTSET),
+    }.get(config['logging']['level'], config['logging']['level'])
+
+    return config
diff --git a/src/framework.py b/src/framework.py
@@ -1,135 +1,32 @@
-from collections import OrderedDict
-from configparser import ConfigParser, ExtendedInterpolation
 from copy import copy
 from csv import writer
 from dis import distb
 from functools import partial
 from gzip import open as open_gzip
-from hashlib import sha3_512
 from io import StringIO
-from json import JSONDecoder
-from logging import getLogger, CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET
+from logging import getLogger
 from os import cpu_count
-from pathlib import Path
-from pickle import dumps
 from random import Random, random
 from subprocess import run
 from time import sleep, time
 from threading import Thread
-from typing import Any, Callable, Iterable, Optional, Sequence, TypeVar, cast
+from typing import Any, Callable, Iterable, Optional, Sequence, cast
 
 from multiprocessing_logging import install_mp_handler
 
+from .config import CSV_HEADER, RESULTS_CSV, T, get_config, get_entropy, get_machines, make_config_files
 from .progress import ProgressPool, Style
 
 try:
     from os import nice
 except ImportError:
     pass
 
-T = TypeVar("T")
-
-MACHINES_CONFIG = Path('machines.json')
-CSV_HEADER = Path('header.csv')
-RUNNER_CONFIG = Path('runner.config')
-LOG_TEMPLATE = Path('logging.template')
-
-CONFIGS = [
-    MACHINES_CONFIG,
-    CSV_HEADER,
-    RUNNER_CONFIG,
-    LOG_TEMPLATE,
-]
-DEFAULT_CONFIG_PATH = Path(__file__).parent.joinpath('defaults', 'runner.config')
-RESULTS_CSV = Path('results.csv')
-
-CONFIG_TEMPLATES = [
-    '{\n\t"example_machine": [example_one_core_weight, example_cores_used]\n}',
-    '',
-    DEFAULT_CONFIG_PATH.read_text(),
-    ''  # log_template
-]
-
 
 def _dummy(*args, **kwargs):
     pass
 
 
-def get_entropy(*extras) -> bytes:
-    """Generate bytes for use as a seed.
-
-    Include additional objects only if they will be consistent from machine to machine.
-    """
-    hash_obj = sha3_512()
-    for name in CONFIGS:
-        hash_obj.update(name.read_bytes())
-    hash_obj.update(dumps(extras))
-    return hash_obj.digest()
-
-
-def make_config_files():
-    """Generate config files if not present, raising an error if any were not present."""
-    any_triggered = False
-    for name, default in zip(CONFIGS, CONFIG_TEMPLATES):
-        try:
-            with name.open('x') as f:
-                f.write(default)
-        except FileExistsError:
-            pass
-        else:
-            any_triggered = True
-    if any_triggered:
-        raise FileNotFoundError("Your configuration files have been created. Please fill them out.")
-
-
-def get_machines():  # -> OrderedDict[str, Tuple[int, int]]:
-    """Read the list of machine names and their associated thread weights and number of threads."""
-    decoder = JSONDecoder(object_pairs_hook=OrderedDict)
-    with MACHINES_CONFIG.open('r') as f:
-        return decoder.decode(f.read())
-
-
-def parse_file_size(value: str) -> int:
-    """Parse a string to return an integer file size."""
-    value = value.lower()
-    if value.endswith('t') or value.endswith('tib'):
-        return int(value.rstrip('t').rstrip('tib')) << 40
-    elif value.endswith('g') or value.endswith('gib'):
-        return int(value.rstrip('g').rstrip('gib')) << 30
-    elif value.endswith('m') or value.endswith('mib'):
-        return int(value.rstrip('m').rstrip('mib')) << 20
-    elif value.endswith('k') or value.endswith('kib'):
-        return int(value.rstrip('k').rstrip('kib')) << 10
-    elif value.endswith('tb'):
-        return int(value[:2]) * 10**12
-    elif value.endswith('gb'):
-        return int(value[:2]) * 10**9
-    elif value.endswith('mb'):
-        return int(value[:2]) * 10**6
-    elif value.endswith('kb'):
-        return int(value[:2]) * 10**3
-    return int(value)
-
-
-def get_config() -> ConfigParser:
-    """Read and parse the configuration file."""
-    config = ConfigParser(interpolation=ExtendedInterpolation(), converters={'filesize': parse_file_size})
-    with DEFAULT_CONFIG_PATH.open('r') as f:
-        config.read_file(f)
-    config.read(RUNNER_CONFIG)
-
-    config['logging']['level'] = {
-        'CRITICAL': str(CRITICAL),
-        'ERROR': str(ERROR),
-        'WARNING': str(WARNING),
-        'INFO': str(INFO),
-        'DEBUG': str(DEBUG),
-        'NOTSET': str(NOTSET),
-    }.get(config['logging']['level'], config['logging']['level'])
-
-    return config
-
-
 def renicer_thread(pool):
     """Ensure that this process, and all its children, do not hog resources."""
     try: