From c37ef8c7d7221c26d17da730dd41273a2d15be39 Mon Sep 17 00:00:00 2001 From: Gabe Appleton Date: Sun, 9 May 2021 18:40:53 -0400 Subject: [PATCH] Move configuration functions to separate file close #14 --- example_main.py | 3 +- src/config.py | 105 +++++++++++++++++++++++++++++++++++++++++++++ src/framework.py | 109 ++--------------------------------------------- 3 files changed, 110 insertions(+), 107 deletions(-) create mode 100644 src/config.py diff --git a/example_main.py b/example_main.py index e996e8d..7a88c1a 100644 --- a/example_main.py +++ b/example_main.py @@ -2,7 +2,8 @@ from logging.handlers import RotatingFileHandler from typing import Type -from src.framework import get_config, make_config_files, run_jobs, _sleeper +from src.config import get_config, make_config_files +from src.framework import run_jobs, _sleeper from src.zipped_logs import ZippedRotatingFileHandler diff --git a/src/config.py b/src/config.py new file mode 100644 index 0000000..42f3268 --- /dev/null +++ b/src/config.py @@ -0,0 +1,105 @@ +"""Module that deals with all configuration related functions.""" +from collections import OrderedDict +from configparser import ConfigParser, ExtendedInterpolation +from hashlib import sha3_512 +from json import JSONDecoder +from logging import CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET +from pathlib import Path +from pickle import dumps +from typing import TypeVar + + +T = TypeVar("T") + +MACHINES_CONFIG = Path('machines.json') +CSV_HEADER = Path('header.csv') +RUNNER_CONFIG = Path('runner.config') + +CONFIGS = [ + MACHINES_CONFIG, + CSV_HEADER, + RUNNER_CONFIG, +] +DEFAULT_CONFIG_PATH = Path(__file__).parent.joinpath('defaults', 'runner.config') +RESULTS_CSV = Path('results.csv') + +CONFIG_TEMPLATES = [ + '{\n\t"example_machine": [example_one_core_weight, example_cores_used]\n}', + '', + DEFAULT_CONFIG_PATH.read_text() +] + + +def get_entropy(*extras) -> bytes: + """Generate bytes for use as a seed. + + Include additional objects only if they will be consistent from machine to machine. + """ + hash_obj = sha3_512() + for name in CONFIGS: + hash_obj.update(name.read_bytes()) + hash_obj.update(dumps(extras)) + return hash_obj.digest() + + +def make_config_files(): + """Generate config files if not present, raising an error if any were not present.""" + any_triggered = False + for name, default in zip(CONFIGS, CONFIG_TEMPLATES): + try: + with name.open('x') as f: + f.write(default) + except FileExistsError: + pass + else: + any_triggered = True + if any_triggered: + raise FileNotFoundError("Your configuration files have been created. Please fill them out.") + + +def get_machines(): # -> OrderedDict[str, Tuple[int, int]]: + """Read the list of machine names and their associated thread weights and number of threads.""" + decoder = JSONDecoder(object_pairs_hook=OrderedDict) + with MACHINES_CONFIG.open('r') as f: + return decoder.decode(f.read()) + + +def parse_file_size(value: str) -> int: + """Parse a string to return an integer file size.""" + value = value.lower() + if value.endswith('t') or value.endswith('tib'): + return int(value.rstrip('t').rstrip('tib')) << 40 + elif value.endswith('g') or value.endswith('gib'): + return int(value.rstrip('g').rstrip('gib')) << 30 + elif value.endswith('m') or value.endswith('mib'): + return int(value.rstrip('m').rstrip('mib')) << 20 + elif value.endswith('k') or value.endswith('kib'): + return int(value.rstrip('k').rstrip('kib')) << 10 + elif value.endswith('tb'): + return int(value[:2]) * 10**12 + elif value.endswith('gb'): + return int(value[:2]) * 10**9 + elif value.endswith('mb'): + return int(value[:2]) * 10**6 + elif value.endswith('kb'): + return int(value[:2]) * 10**3 + return int(value) + + +def get_config() -> ConfigParser: + """Read and parse the configuration file.""" + config = ConfigParser(interpolation=ExtendedInterpolation(), converters={'filesize': parse_file_size}) + with DEFAULT_CONFIG_PATH.open('r') as f: + config.read_file(f) + config.read(RUNNER_CONFIG) + + config['logging']['level'] = { + 'CRITICAL': str(CRITICAL), + 'ERROR': str(ERROR), + 'WARNING': str(WARNING), + 'INFO': str(INFO), + 'DEBUG': str(DEBUG), + 'NOTSET': str(NOTSET), + }.get(config['logging']['level'], config['logging']['level']) + + return config diff --git a/src/framework.py b/src/framework.py index 10f1aa0..96d5f0d 100644 --- a/src/framework.py +++ b/src/framework.py @@ -1,25 +1,20 @@ -from collections import OrderedDict -from configparser import ConfigParser, ExtendedInterpolation from copy import copy from csv import writer from dis import distb from functools import partial from gzip import open as open_gzip -from hashlib import sha3_512 from io import StringIO -from json import JSONDecoder -from logging import getLogger, CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET +from logging import getLogger from os import cpu_count -from pathlib import Path -from pickle import dumps from random import Random, random from subprocess import run from time import sleep, time from threading import Thread -from typing import Any, Callable, Iterable, Optional, Sequence, TypeVar, cast +from typing import Any, Callable, Iterable, Optional, Sequence, cast from multiprocessing_logging import install_mp_handler +from .config import CSV_HEADER, RESULTS_CSV, T, get_config, get_entropy, get_machines, make_config_files from .progress import ProgressPool, Style try: @@ -27,109 +22,11 @@ except ImportError: pass -T = TypeVar("T") - -MACHINES_CONFIG = Path('machines.json') -CSV_HEADER = Path('header.csv') -RUNNER_CONFIG = Path('runner.config') -LOG_TEMPLATE = Path('logging.template') - -CONFIGS = [ - MACHINES_CONFIG, - CSV_HEADER, - RUNNER_CONFIG, - LOG_TEMPLATE, -] -DEFAULT_CONFIG_PATH = Path(__file__).parent.joinpath('defaults', 'runner.config') -RESULTS_CSV = Path('results.csv') - -CONFIG_TEMPLATES = [ - '{\n\t"example_machine": [example_one_core_weight, example_cores_used]\n}', - '', - DEFAULT_CONFIG_PATH.read_text(), - '' # log_template -] - def _dummy(*args, **kwargs): pass -def get_entropy(*extras) -> bytes: - """Generate bytes for use as a seed. - - Include additional objects only if they will be consistent from machine to machine. - """ - hash_obj = sha3_512() - for name in CONFIGS: - hash_obj.update(name.read_bytes()) - hash_obj.update(dumps(extras)) - return hash_obj.digest() - - -def make_config_files(): - """Generate config files if not present, raising an error if any were not present.""" - any_triggered = False - for name, default in zip(CONFIGS, CONFIG_TEMPLATES): - try: - with name.open('x') as f: - f.write(default) - except FileExistsError: - pass - else: - any_triggered = True - if any_triggered: - raise FileNotFoundError("Your configuration files have been created. Please fill them out.") - - -def get_machines(): # -> OrderedDict[str, Tuple[int, int]]: - """Read the list of machine names and their associated thread weights and number of threads.""" - decoder = JSONDecoder(object_pairs_hook=OrderedDict) - with MACHINES_CONFIG.open('r') as f: - return decoder.decode(f.read()) - - -def parse_file_size(value: str) -> int: - """Parse a string to return an integer file size.""" - value = value.lower() - if value.endswith('t') or value.endswith('tib'): - return int(value.rstrip('t').rstrip('tib')) << 40 - elif value.endswith('g') or value.endswith('gib'): - return int(value.rstrip('g').rstrip('gib')) << 30 - elif value.endswith('m') or value.endswith('mib'): - return int(value.rstrip('m').rstrip('mib')) << 20 - elif value.endswith('k') or value.endswith('kib'): - return int(value.rstrip('k').rstrip('kib')) << 10 - elif value.endswith('tb'): - return int(value[:2]) * 10**12 - elif value.endswith('gb'): - return int(value[:2]) * 10**9 - elif value.endswith('mb'): - return int(value[:2]) * 10**6 - elif value.endswith('kb'): - return int(value[:2]) * 10**3 - return int(value) - - -def get_config() -> ConfigParser: - """Read and parse the configuration file.""" - config = ConfigParser(interpolation=ExtendedInterpolation(), converters={'filesize': parse_file_size}) - with DEFAULT_CONFIG_PATH.open('r') as f: - config.read_file(f) - config.read(RUNNER_CONFIG) - - config['logging']['level'] = { - 'CRITICAL': str(CRITICAL), - 'ERROR': str(ERROR), - 'WARNING': str(WARNING), - 'INFO': str(INFO), - 'DEBUG': str(DEBUG), - 'NOTSET': str(NOTSET), - }.get(config['logging']['level'], config['logging']['level']) - - return config - - def renicer_thread(pool): """Ensure that this process, and all its children, do not hog resources.""" try: