From ea45b86cd171e749fe8ee64ed47e7f8cd790f3ee Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 24 Jun 2024 13:35:46 +0100 Subject: [PATCH 01/55] global actions --- .github/workflows/python_actions.yml | 81 ++++------------------------ spalloc_client/py.typed | 13 +++++ 2 files changed, 24 insertions(+), 70 deletions(-) create mode 100644 spalloc_client/py.typed diff --git a/.github/workflows/python_actions.yml b/.github/workflows/python_actions.yml index 0ec7b4ea9..a8a0e0179 100644 --- a/.github/workflows/python_actions.yml +++ b/.github/workflows/python_actions.yml @@ -15,76 +15,17 @@ # This workflow will install Python dependencies, run tests, lint and rat with a variety of Python versions # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -name: Python Build +name: Python Actions on: [push] jobs: - build: - runs-on: ubuntu-latest - timeout-minutes: 10 - strategy: - matrix: - python-version: [3.8, 3.9, "3.10", "3.11", "3.12"] + call: + uses: SpiNNakerManchester/SupportScripts/.github/workflows/python_checks.yml@main + with: + dependencies: SpiNNUtils + test_directories: tests + coverage-package: spalloc tests + flake8-packages: spalloc_client tests + pylint-packages: spalloc_client + mypy-packages: spalloc_client tests + secrets: inherit - steps: - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Checkout - uses: actions/checkout@v4 - - - name: Checkout SupportScripts - uses: actions/checkout@v4 - with: - repository: SpiNNakerManchester/SupportScripts - path: support - - - name: Install pip, etc - uses: ./support/actions/python-tools - - - name: Install Spinnaker Dependencies - uses: ./support/actions/install-spinn-deps - with: - repositories: SpiNNUtils - install: true - - - name: Install - uses: ./support/actions/run-install - - - name: Docs requirements Install - run: pip install -r requirements-docs.txt - - - name: Test with pytest - uses: ./support/actions/pytest - with: - tests: tests - options: --durations=10 --timeout=120 - coverage: ${{ matrix.python-version == 3.12 }} - cover-packages: spalloc tests - coveralls-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Lint with flake8 - run: flake8 spalloc_client tests - - - name: Lint with pylint - uses: ./support/actions/pylint - with: - package: spalloc_client - exitcheck: 39 - - - name: Run rat copyright enforcement - if: matrix.python-version == 3.12 - uses: ./support/actions/check-copyrights - with: - config_file: rat_asl20.xml - - - name: Build documentation with sphinx - if: matrix.python-version == 3.12 - uses: ./support/actions/sphinx - with: - directory: docs/source - - - name: Validate CITATION.cff - if: matrix.python-version == 3.12 - uses: dieghernan/cff-validator@main \ No newline at end of file diff --git a/spalloc_client/py.typed b/spalloc_client/py.typed new file mode 100644 index 000000000..89860f15f --- /dev/null +++ b/spalloc_client/py.typed @@ -0,0 +1,13 @@ +# Copyright (c) 2024 The University of Manchester +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. From ef7229eca53595f94078d76059ac70bceaf01a0f Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Tue, 25 Jun 2024 06:55:43 +0100 Subject: [PATCH 02/55] docs --- spalloc_client/scripts/alloc.py | 37 ++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/spalloc_client/scripts/alloc.py b/spalloc_client/scripts/alloc.py index ebbd755c8..0fd421e0e 100644 --- a/spalloc_client/scripts/alloc.py +++ b/spalloc_client/scripts/alloc.py @@ -111,6 +111,7 @@ import subprocess import sys import tempfile +from typing import Dict, List, Optional, Tuple from shlex import quote from spalloc_client import ( config, Job, JobState, __version__, ProtocolError, ProtocolTimeoutError, @@ -305,7 +306,14 @@ def wait_for_job_ready(job): return 4, "Keyboard interrupt." -def parse_argv(argv): +def parse_argv(argv: List[str]) -> Tuple[ + argparse.ArgumentParser, argparse.Namespace]: + """ + Parse the arguments. + + :param list(str) argv: Arguments passed it + :rtype: (ArgumentParser, list(str) + """ cfg = config.read_config() parser = argparse.ArgumentParser( @@ -409,7 +417,15 @@ def parse_argv(argv): return parser, parser.parse_args(argv) -def run_job(job_args, job_kwargs, ip_file_filename): +def run_job(job_args: List[str], job_kwargs: Dict[str, str], ip_file_filename: str): + """ + Run a job + + :param list(str) job_args: + :param job_kwargs: + :param ip_file_filename: + :return: + """ # Reason for destroying the job reason = None @@ -448,11 +464,22 @@ def run_job(job_args, job_kwargs, ip_file_filename): job.destroy(reason) -def _minzero(value): +def _minzero(value: Optional[float]) -> Optional[float]: + """ + Makes sure a value is not negative. + + :type value: float, int or None + :rtpye: float or None + """ return value if value >= 0.0 else None -def main(argv=None): +def main(argv: List[str] = None): + """ + The main method run + + :param list(str) argv: + """ global arguments, t # pylint: disable=global-statement parser, arguments = parse_argv(argv) t = Terminal(stream=sys.stderr) @@ -509,7 +536,7 @@ def main(argv=None): try: return run_job(job_args, job_kwargs, ip_file_filename) except SpallocServerException as e: # pragma: no cover - info(t.red("Error from server: {}".format(e))) + info(t.red(f"Error from server: {e}")) return 6 finally: # Delete IP address list file From e30e66b9493a869e3a0937762aebd46c1f159460 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 26 Jun 2024 10:51:11 +0100 Subject: [PATCH 03/55] pylint fixes --- spalloc_client/config.py | 7 ++-- spalloc_client/job.py | 23 ++++++++----- spalloc_client/protocol_client.py | 4 ++- spalloc_client/scripts/alloc.py | 55 +++++++++++++++--------------- spalloc_client/scripts/job.py | 22 ++++++++---- spalloc_client/scripts/machine.py | 7 ++-- spalloc_client/scripts/support.py | 3 +- spalloc_client/scripts/where_is.py | 7 ++-- spalloc_client/states.py | 1 + spalloc_client/term.py | 22 ++++++------ tests/test_term.py | 2 +- 11 files changed, 86 insertions(+), 67 deletions(-) diff --git a/spalloc_client/config.py b/spalloc_client/config.py index e51a3d577..cff41673f 100644 --- a/spalloc_client/config.py +++ b/spalloc_client/config.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=wrong-spelling-in-docstring """ The spalloc command-line tool and Python library determine their default configuration options from a spalloc configuration file if present. @@ -62,7 +63,7 @@ The name of a specific machine on which to run all jobs or None to use any available machine. (Default: None) ``tags`` - The set of tags, comma seperated, to require a machine to have when + The set of tags, comma separated, to require a machine to have when allocating jobs. (Default: default) ``min_ratio`` Require that when allocating a number of boards the allocation is at least @@ -78,9 +79,9 @@ requires the allocation of a whole machine. If False, wrap-around links may or may-not be present in allocated machines. (Default: False) """ +import configparser import os.path import appdirs -import configparser # The application name to use in config file names _name = "spalloc" @@ -88,7 +89,7 @@ # Standard config file names/locations SYSTEM_CONFIG_FILE = appdirs.site_config_dir(_name) USER_CONFIG_FILE = appdirs.user_config_dir(_name) -CWD_CONFIG_FILE = os.path.join(os.curdir, ".{}".format(_name)) +CWD_CONFIG_FILE = os.path.join(os.curdir, f".{_name}") # Search path for config files (lowest to highest priority) SEARCH_PATH = [ diff --git a/spalloc_client/job.py b/spalloc_client/job.py index 730af2644..8c1b3ff46 100644 --- a/spalloc_client/job.py +++ b/spalloc_client/job.py @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from spalloc_client.scripts.support import ( - VERSION_RANGE_START, VERSION_RANGE_STOP) # A high-level Python interface for allocating SpiNNaker boards. @@ -21,6 +19,10 @@ import subprocess import time import sys + +from spalloc_client.scripts.support import ( + VERSION_RANGE_START, VERSION_RANGE_STOP) + from .protocol_client import ProtocolClient, ProtocolTimeoutError from .config import read_config, SEARCH_PATH from .states import JobState @@ -28,6 +30,7 @@ logger = logging.getLogger(__name__) +# pylint: disable=wrong-spelling-in-docstring # In Python 2, no default handler exists for software which doesn't configure # its own logging so we must add one ourselves as per # https://docs.python.org/3.1/library/logging.html#configuring-logging-for-a-library @@ -267,11 +270,14 @@ def __init__(self, *args, **kwargs): job_state = self._get_state() if (job_state.state == JobState.unknown or job_state.state == JobState.destroyed): - raise JobDestroyedError("Job {} does not exist: {}{}{}".format( - resume_job_id, - job_state.state.name, - ": " if job_state.reason is not None else "", - job_state.reason if job_state.reason is not None else "")) + if job_state.reason is not None: + reason = job_state.reason + else: + reason = "" + raise JobDestroyedError( + f"Job {resume_job_id} does not exist: " + f"{job_state.state.name}" + f"{': ' if job_state.reason is not None else ''}{reason}") # Snag the keepalive interval from the job self._keepalive = job_state.keepalive @@ -363,8 +369,7 @@ def _assert_compatible_version(self): if not (VERSION_RANGE_START <= v_ints < VERSION_RANGE_STOP): self._client.close() raise ValueError( - "Server version {} is not compatible with this client.".format( - v)) + f"Server version {v} is not compatible with this client.") def _reconnect(self): """ Reconnect to the server and check version. diff --git a/spalloc_client/protocol_client.py b/spalloc_client/protocol_client.py index 2da314e42..9d9e77f12 100644 --- a/spalloc_client/protocol_client.py +++ b/spalloc_client/protocol_client.py @@ -18,6 +18,7 @@ import errno import json import socket +from typing import Dict, Optional from threading import current_thread, RLock, local from spalloc_client._utils import time_left, timed_out, make_timeout @@ -423,7 +424,8 @@ def get_board_at_position(self, machine_name, x, y, z, frozenset("machine chip_x chip_y".split()), frozenset("job_id chip_x chip_y".split())]) - def where_is(self, timeout=None, **kwargs): + def where_is(self, timeout :Optional[int] = None, + **kwargs: Dict[str, object]): # Test for whether sane arguments are passed. keywords = frozenset(kwargs) if keywords not in ProtocolClient._acceptable_kwargs_for_where_is: diff --git a/spalloc_client/scripts/alloc.py b/spalloc_client/scripts/alloc.py index 0fd421e0e..559d76ef4 100644 --- a/spalloc_client/scripts/alloc.py +++ b/spalloc_client/scripts/alloc.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=wrong-spelling-in-docstring """ A command-line utility for creating jobs. .. note:: @@ -106,6 +107,7 @@ sent. Adding the ``--keepalive -1`` option when creating a job disables this. """ import argparse +import functools import logging import os import subprocess @@ -120,10 +122,11 @@ arguments = None t = None -_input = input # This is so we can monkeypatch input during testing +_input = input # This is so we can monkey patch input during testing -def write_ips_to_csv(connections, ip_file_filename): +def write_ips_to_csv(connections: Dict[Tuple[int, int], str], + ip_file_filename: str): """ Write the supplied IP addresses to a CSV file. The produced CSV has three columns: x, y and hostname where x and y give @@ -142,7 +145,8 @@ def write_ips_to_csv(connections, ip_file_filename): in sorted(connections.items()))) -def print_info(machine_name, connections, width, height, ip_file_filename): +def print_info(machine_name: str, connections: Dict[Tuple[int, int], str], + width: int, height: int, ip_file_filename: str): """ Print the current machine info in a human-readable form and wait for the user to press enter. @@ -178,8 +182,10 @@ def print_info(machine_name, connections, width, height, ip_file_filename): print("") -def run_command(command, job_id, machine_name, connections, width, height, - ip_file_filename): +def run_command( + command: List[str], job_id: int, machine_name: str, + connections: Dict[Tuple[int, int], str], width: int, height: int, + ip_file_filename: str): """ Run a user-specified command, substituting arguments for values taken from the allocated board. @@ -247,18 +253,25 @@ def run_command(command, job_id, machine_name, connections, width, height, p.terminate() -def info(msg): +def info(msg:str): + """ + Writes a message to the terminal + """ if not arguments.quiet: - t.stream.write("{}\n".format(msg)) + t.stream.write(f"{msg}\n") -def update(msg, colour, *args): +def update(msg:str, colour: functools.partial, *args: List[object]): + """ + Writes a message to the terminal in the schoosen colour. + """ info(t.update(colour(msg.format(*args)))) -def wait_for_job_ready(job): - # Wait for it to become ready, keeping the user informed along the - # way +def wait_for_job_ready(job: Job): + """ + Wait for it to become ready, keeping the user informed along the way + """ old_state = None cur_state = job.state try: @@ -310,9 +323,6 @@ def parse_argv(argv: List[str]) -> Tuple[ argparse.ArgumentParser, argparse.Namespace]: """ Parse the arguments. - - :param list(str) argv: Arguments passed it - :rtype: (ArgumentParser, list(str) """ cfg = config.read_config() @@ -350,7 +360,7 @@ def parse_argv(argv: List[str]) -> Tuple[ "--tags", "-t", nargs="*", metavar="TAG", default=cfg["tags"] or ["default"], help="only allocate boards which have (at least) the specified flags " - "(default: {})".format(" ".join(cfg["tags"] or []))) + f"(default: {' '.join(cfg['tags'] or [])})") allocation_args.add_argument( "--min-ratio", type=float, metavar="RATIO", default=cfg["min_ratio"], help="when allocating by number of boards, require that the " @@ -370,12 +380,11 @@ def parse_argv(argv: List[str]) -> Tuple[ "--require-torus", "-w", action="store_true", default=cfg["require_torus"], help="require that the allocation contain torus (a.k.a. wrap-around) " - "links {}".format("(default)" if cfg["require_torus"] else "")) + f"links {'(default)' if cfg['require_torus'] else ''}") allocation_args.add_argument( "--no-require-torus", "-W", action="store_false", dest="require_torus", help="do not require that the allocation contain torus (a.k.a. " - "wrap-around) links {}".format( - "" if cfg["require_torus"] else "(default)")) + f"wrap-around) links {'' if cfg['require_torus'] else '(default)'}") command_args = parser.add_argument_group("command wrapping arguments") command_args.add_argument( @@ -420,11 +429,6 @@ def parse_argv(argv: List[str]) -> Tuple[ def run_job(job_args: List[str], job_kwargs: Dict[str, str], ip_file_filename: str): """ Run a job - - :param list(str) job_args: - :param job_kwargs: - :param ip_file_filename: - :return: """ # Reason for destroying the job reason = None @@ -467,9 +471,6 @@ def run_job(job_args: List[str], job_kwargs: Dict[str, str], ip_file_filename: s def _minzero(value: Optional[float]) -> Optional[float]: """ Makes sure a value is not negative. - - :type value: float, int or None - :rtpye: float or None """ return value if value >= 0.0 else None @@ -477,8 +478,6 @@ def _minzero(value: Optional[float]) -> Optional[float]: def main(argv: List[str] = None): """ The main method run - - :param list(str) argv: """ global arguments, t # pylint: disable=global-statement parser, arguments = parse_argv(argv) diff --git a/spalloc_client/scripts/job.py b/spalloc_client/scripts/job.py index 1447bfbfc..1e927ea37 100644 --- a/spalloc_client/scripts/job.py +++ b/spalloc_client/scripts/job.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=wrong-spelling-in-docstring """ Command-line administrative job management interface. ``spalloc-job`` may be called with a job ID, or if no arguments supplied your @@ -75,13 +76,15 @@ """ import argparse import sys +from typing import List + from spalloc_client import __version__, JobState from spalloc_client.term import ( Terminal, render_definitions, render_boards, DEFAULT_BOARD_EDGES) +from spalloc_client import ProtocolClient from spalloc_client._utils import render_timestamp from .support import Terminate, Script - def _state_name(mapping): return JobState(mapping["state"]).name # pylint: disable=no-member @@ -136,8 +139,7 @@ def show_job_info(t, client, timeout, job_id): info["Request"] = "Job({}{}{})".format( ", ".join(map(str, args)), ",\n " if args and kwargs else "", - ",\n ".join("{}={!r}".format(k, v) for - k, v in sorted(kwargs.items())) + ",\n ".join(f"{k}={v!r}" for k, v in sorted(kwargs.items())) ) if job["boards"] is not None: @@ -270,7 +272,7 @@ def list_ips(client, timeout, job_id): raise Terminate(9, "Job {} is queued or does not exist", job_id) print("x,y,hostname") for ((x, y), hostname) in sorted(connections): - print("{},{},{}".format(x, y, hostname)) + print(f"{x},{y},{hostname}") def destroy_job(client, timeout, job_id, reason=None): @@ -296,12 +298,18 @@ def destroy_job(client, timeout, job_id, reason=None): class ManageJobScript(Script): + """ + A tool for running Job scripts. + """ def __init__(self): super().__init__() self.parser = None - def get_job_id(self, client, args): + def get_job_id(self, client: ProtocolClient, args: List[str]): + """ + get a job for the owner named in the args + """ if args.job_id is not None: return args.job_id # No Job ID specified, attempt to discover one @@ -354,7 +362,7 @@ def verify_arguments(self, args): if args.job_id is None and args.owner is None: self.parser.error("job ID (or --owner) not specified") - def body(self, client, args): + def body(self, client:ProtocolClient, args: argparse.Namespace): jid = self.get_job_id(client, args) # Do as the user asked @@ -369,7 +377,7 @@ def body(self, client, args): elif args.destroy is not None: # Set default destruction message if args.destroy == "" and args.owner: - args.destroy = "Destroyed by {}".format(args.owner) + args.destroy = f"Destroyed by {args.owner}" destroy_job(client, args.timeout, jid, args.destroy) else: show_job_info(Terminal(), client, args.timeout, jid) diff --git a/spalloc_client/scripts/machine.py b/spalloc_client/scripts/machine.py index 237dc9294..390e5d212 100644 --- a/spalloc_client/scripts/machine.py +++ b/spalloc_client/scripts/machine.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=wrong-spelling-in-docstring """ Command-line administrative machine management interface. When called with no arguments the ``spalloc-machine`` command lists all @@ -151,7 +152,7 @@ def show_machine(t, machines, jobs, machine_name, compact=False): info = dict() info["Name"] = machine["name"] info["Tags"] = ", ".join(machine["tags"]) - info["In-use"] = "{} of {}".format(num_in_use, num_boards) + info["In-use"] = f"{num_in_use} of {num_boards}" info["Jobs"] = len(displayed_jobs) print(render_definitions(info)) @@ -184,7 +185,7 @@ def show_machine(t, machines, jobs, machine_name, compact=False): key = job["key"] job_id = str(job["job_id"]) cells.append((len(key) + len(job_id) + 1, - "{}:{}".format(job["colour"](key), job_id))) + f"{job['colour'](key)}:{job_id}")) print("") print(render_cells(cells)) else: @@ -198,7 +199,7 @@ def show_machine(t, machines, jobs, machine_name, compact=False): for job in displayed_jobs: owner = job["owner"] if "keepalivehost" in job and job["keepalivehost"] is not None: - owner += " (%s)" % job["keepalivehost"] + owner += f" {job['keepalivehost']}" job_table.append([ (job["colour"], job["key"]), job["job_id"], diff --git a/spalloc_client/scripts/support.py b/spalloc_client/scripts/support.py index af6a3f488..c0af6d8ea 100644 --- a/spalloc_client/scripts/support.py +++ b/spalloc_client/scripts/support.py @@ -13,6 +13,7 @@ # limitations under the License. import sys +from typing import List from spalloc_client import ( config, ProtocolClient, ProtocolError, ProtocolTimeoutError, SpallocServerException) @@ -63,7 +64,7 @@ def verify_arguments(self, args): required. """ - def body(self, client, args): + def body(self, client: ProtocolClient, args: List[str]): """ How to do the processing of the script once a client has been\ obtained and verified to be compatible. """ diff --git a/spalloc_client/scripts/where_is.py b/spalloc_client/scripts/where_is.py index 0d67fedf6..26731b10b 100644 --- a/spalloc_client/scripts/where_is.py +++ b/spalloc_client/scripts/where_is.py @@ -142,7 +142,7 @@ def verify_arguments(self, args): } self.show_board_chip = True except ValueError as e: - self.parser.error("Error: {}".format(e)) + self.parser.error(f"Error: {e}") def body(self, client, args): # Ask the server @@ -152,8 +152,9 @@ def body(self, client, args): out = dict() out["Machine"] = location["machine"] - out["Physical location"] = "Cabinet {}, Frame {}, Board {}".format( - *location["physical"]) + cabinet, frame, board = location["physical"] + out["Physical location"] = ( + f"Cabinet {cabinet}, Frame {frame}, Board {board}") out["Board coordinate"] = tuple(location["logical"]) out["Machine chip coordinates"] = tuple(location["chip"]) if self.show_board_chip: diff --git a/spalloc_client/states.py b/spalloc_client/states.py index 5ec6295d5..d37241ba3 100644 --- a/spalloc_client/states.py +++ b/spalloc_client/states.py @@ -22,6 +22,7 @@ class JobState(IntEnum): """ All the possible states that a job may be in. """ + # pylint: disable=invalid-name unknown = 0 """ The job ID requested was not recognised. """ diff --git a/spalloc_client/term.py b/spalloc_client/term.py index 31ff37274..e6c85de2d 100644 --- a/spalloc_client/term.py +++ b/spalloc_client/term.py @@ -27,7 +27,7 @@ class ANSIDisplayAttributes(IntEnum): """ Code numbers of ANSI display attributes for use with `ESC[...m`\ sequences. """ - + # pylint: disable=invalid-name reset = 0 bright = 1 dim = 2 @@ -121,7 +121,7 @@ def clear_screen(self): """ return self("\033[2J\033[;H") - def update(self, string="", start_again=False): + def update(self, string: str = "", start_again: bool = False): """ Print before a line and it will replace the previous line prefixed\ with :py:meth:`.update`. @@ -254,7 +254,7 @@ def render_table(table, column_sep=" "): return "\n".join(column_sep.join(row).rstrip() for row in out) -def render_definitions(definitions, seperator=": "): +def render_definitions(definitions, separator=": "): """ Render a definition list. Such a list looks like this:: @@ -270,7 +270,7 @@ def render_definitions(definitions, seperator=": "): ---------- definitions : :py:class:`collections.OrderedDict` The key/value set to display. - seperator : str + separator : str The seperator inserted between keys and values. """ # Special case since max would fail @@ -279,8 +279,8 @@ def render_definitions(definitions, seperator=": "): col_width = max(map(len, definitions)) return "\n".join("{:>{}s}{}{}".format( - key, col_width, seperator, str(value).replace( - "\n", "\n{}".format(" "*(col_width + len(seperator))))) + key, col_width, separator, str(value).replace( + "\n", f"\n{' '*(col_width + len(separator))}")) for key, value in definitions.items()) @@ -393,10 +393,10 @@ def render_boards(board_groups, dead_links=frozenset(), dead_edge : ("___", "\\", "/") The strings to use to draw dead links. blank_label : " " - The 3-character string to use to label non-existant boards. (Blank by + The 3-character string to use to label non-existent boards. (Blank by default) blank_edge : ("___", "\\", "/") - The characters to use to render non-existant board edges. (Blank by + The characters to use to render non-existent board edges. (Blank by default) """ # pylint: disable=too-many-locals @@ -407,11 +407,11 @@ def render_boards(board_groups, dead_links=frozenset(), board_edges = {} # The set of all boards defined (used to filter displaying of dead links to - # non-existant boards + # non-existent boards all_boards = set() for boards, label, edge_inner, edge_outer in board_groups: - # Convert to cartesian coords + # Convert to Cartesian coords boards = set(_board_to_cartesian(x, y, z) for x, y, z in boards) all_boards.update(boards) @@ -458,7 +458,7 @@ def render_boards(board_groups, dead_links=frozenset(), # . /0 0\___/2 0\___/ 0 Even # . \___/ \___/ -1 Odd # -1 0 1 2 3 4 - # Odd Evn Odd Evn Odd Evn + # Odd Even Odd Even Odd Even out = [] for y in range(y_max, y_min - 1, -1): even_row = (y % 2) == 0 diff --git a/tests/test_term.py b/tests/test_term.py index 88b56b93e..b03628c57 100644 --- a/tests/test_term.py +++ b/tests/test_term.py @@ -199,7 +199,7 @@ def test_render_definitions(): # Alternative seperator assert render_definitions(dict([("foo", "bar")]), - seperator="=") == "foo=bar" + separator="=") == "foo=bar" # Linebreaks assert render_definitions({"Key": "Lines\nBroken\nUp."}) == ( "Key: Lines\n" From 1bd0a14582bbc32f86cd793580b5a36a2b3cc3ee Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 26 Jun 2024 10:58:14 +0100 Subject: [PATCH 04/55] flake8 --- spalloc_client/protocol_client.py | 2 +- spalloc_client/scripts/alloc.py | 9 +++++---- spalloc_client/scripts/job.py | 3 ++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/spalloc_client/protocol_client.py b/spalloc_client/protocol_client.py index 9d9e77f12..d765ff268 100644 --- a/spalloc_client/protocol_client.py +++ b/spalloc_client/protocol_client.py @@ -424,7 +424,7 @@ def get_board_at_position(self, machine_name, x, y, z, frozenset("machine chip_x chip_y".split()), frozenset("job_id chip_x chip_y".split())]) - def where_is(self, timeout :Optional[int] = None, + def where_is(self, timeout: Optional[int] = None, **kwargs: Dict[str, object]): # Test for whether sane arguments are passed. keywords = frozenset(kwargs) diff --git a/spalloc_client/scripts/alloc.py b/spalloc_client/scripts/alloc.py index 559d76ef4..19192660c 100644 --- a/spalloc_client/scripts/alloc.py +++ b/spalloc_client/scripts/alloc.py @@ -185,7 +185,7 @@ def print_info(machine_name: str, connections: Dict[Tuple[int, int], str], def run_command( command: List[str], job_id: int, machine_name: str, connections: Dict[Tuple[int, int], str], width: int, height: int, - ip_file_filename: str): + ip_file_filename: str): """ Run a user-specified command, substituting arguments for values taken from the allocated board. @@ -253,7 +253,7 @@ def run_command( p.terminate() -def info(msg:str): +def info(msg: str): """ Writes a message to the terminal """ @@ -261,7 +261,7 @@ def info(msg:str): t.stream.write(f"{msg}\n") -def update(msg:str, colour: functools.partial, *args: List[object]): +def update(msg: str, colour: functools.partial, *args: List[object]): """ Writes a message to the terminal in the schoosen colour. """ @@ -426,7 +426,8 @@ def parse_argv(argv: List[str]) -> Tuple[ return parser, parser.parse_args(argv) -def run_job(job_args: List[str], job_kwargs: Dict[str, str], ip_file_filename: str): +def run_job(job_args: List[str], job_kwargs: Dict[str, str], + ip_file_filename: str): """ Run a job """ diff --git a/spalloc_client/scripts/job.py b/spalloc_client/scripts/job.py index 1e927ea37..d6370c25b 100644 --- a/spalloc_client/scripts/job.py +++ b/spalloc_client/scripts/job.py @@ -85,6 +85,7 @@ from spalloc_client._utils import render_timestamp from .support import Terminate, Script + def _state_name(mapping): return JobState(mapping["state"]).name # pylint: disable=no-member @@ -362,7 +363,7 @@ def verify_arguments(self, args): if args.job_id is None and args.owner is None: self.parser.error("job ID (or --owner) not specified") - def body(self, client:ProtocolClient, args: argparse.Namespace): + def body(self, client: ProtocolClient, args: argparse.Namespace): jid = self.get_job_id(client, args) # Do as the user asked From d170db67f8ce52f024c23ef6cfb189be5c015cf8 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 26 Jun 2024 11:20:48 +0100 Subject: [PATCH 05/55] pylint fixes --- spalloc_client/protocol_client.py | 7 ++++--- spalloc_client/scripts/alloc.py | 1 + spalloc_client/scripts/machine.py | 2 +- spalloc_client/scripts/ps.py | 3 ++- spalloc_client/scripts/support.py | 4 ++-- spalloc_client/term.py | 13 ++++++++----- 6 files changed, 18 insertions(+), 12 deletions(-) diff --git a/spalloc_client/protocol_client.py b/spalloc_client/protocol_client.py index d765ff268..ceb3cdf20 100644 --- a/spalloc_client/protocol_client.py +++ b/spalloc_client/protocol_client.py @@ -56,8 +56,9 @@ class ProtocolClient(object): This minimal implementation is intended to serve both simple applications and as an example implementation of the protocol for other applications. - This implementation simply implements the protocol, presenting an RPC-like - interface to the server. For a higher-level interface built on top of this + This implementation simply implements the protocol, + presenting a Remote procedure call-like interface to the server. + For a higher-level interface built on top of this client, see :py:class:`spalloc.Job`. Usage examples:: @@ -430,6 +431,6 @@ def where_is(self, timeout: Optional[int] = None, keywords = frozenset(kwargs) if keywords not in ProtocolClient._acceptable_kwargs_for_where_is: raise SpallocServerException( - "Invalid arguments: {}".format(", ".join(keywords))) + f"Invalid arguments: {', '.join(keywords)}") kwargs["timeout"] = timeout return self.call("where_is", **kwargs) diff --git a/spalloc_client/scripts/alloc.py b/spalloc_client/scripts/alloc.py index 19192660c..dab1feb72 100644 --- a/spalloc_client/scripts/alloc.py +++ b/spalloc_client/scripts/alloc.py @@ -120,6 +120,7 @@ SpallocServerException) from spalloc_client.term import Terminal, render_definitions +# pylint: disable=invalid-name arguments = None t = None _input = input # This is so we can monkey patch input during testing diff --git a/spalloc_client/scripts/machine.py b/spalloc_client/scripts/machine.py index 390e5d212..e33a0e598 100644 --- a/spalloc_client/scripts/machine.py +++ b/spalloc_client/scripts/machine.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=wrong-spelling-in-docstring +# pylint: disable=wrong-spelling-in-docstring,wrong-spelling-in-comment """ Command-line administrative machine management interface. When called with no arguments the ``spalloc-machine`` command lists all diff --git a/spalloc_client/scripts/ps.py b/spalloc_client/scripts/ps.py index e1a737950..cbad4f33f 100644 --- a/spalloc_client/scripts/ps.py +++ b/spalloc_client/scripts/ps.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=wrong-spelling-in-docstring """ An administrative command-line process listing utility. By default, the ``spalloc-ps`` command lists all running and queued jobs. For @@ -96,7 +97,7 @@ def render_job_list(t, jobs, args): owner = job["owner"] if "keepalivehost" in job and job["keepalivehost"] is not None: - owner += " (%s)" % job["keepalivehost"] + owner += f" ({job['keepalivehost']})" table.append(( job["job_id"], diff --git a/spalloc_client/scripts/support.py b/spalloc_client/scripts/support.py index c0af6d8ea..7e8dddcb1 100644 --- a/spalloc_client/scripts/support.py +++ b/spalloc_client/scripts/support.py @@ -104,10 +104,10 @@ def __call__(self, argv=None): self.body(client, args) return 0 except (IOError, OSError, ProtocolError, ProtocolTimeoutError) as e: - sys.stderr.write("Error communicating with server: {}\n".format(e)) + sys.stderr.write(f"Error communicating with server: {e}\n") return 1 except SpallocServerException as srv_exn: - sys.stderr.write("Error from server: {}\n".format(srv_exn)) + sys.stderr.write(f"Error from server: {srv_exn}\n") return 1 except Terminate as t: t.exit() diff --git a/spalloc_client/term.py b/spalloc_client/term.py index e6c85de2d..9edec238b 100644 --- a/spalloc_client/term.py +++ b/spalloc_client/term.py @@ -22,6 +22,8 @@ from enum import IntEnum from functools import partial +# pylint: disable=wrong-spelling-in-docstring + class ANSIDisplayAttributes(IntEnum): """ Code numbers of ANSI display attributes for use with `ESC[...m`\ @@ -149,8 +151,7 @@ def set_attrs(self, attrs=tuple()): """ if not attrs: return "" - return self("\033[{}m".format( - ";".join(str(attr) for attr in attrs))) + return self(f"\033[{''.join(str(attr) for attr in attrs)}m") def wrap(self, string=None, pre="", post=""): """ Wrap a string in the suppled pre and post strings or just print\ @@ -243,6 +244,8 @@ def render_table(table, column_sep=" "): length = len(str(string)) right_align = True string = f(string) + else: + raise TypeError(f"Unexpected type {column=}") padding = " " * (column_widths[i] - length) if right_align: @@ -271,7 +274,7 @@ def render_definitions(definitions, separator=": "): definitions : :py:class:`collections.OrderedDict` The key/value set to display. separator : str - The seperator inserted between keys and values. + The separator inserted between keys and values. """ # Special case since max would fail if not definitions: @@ -383,7 +386,7 @@ def render_boards(board_groups, dead_links=frozenset(), Lists the groups of boards to display. Label is a 3-character string labelling the boards in the group, edge_inner and edge_outer are the characters to use to draw board edges as a tuple ("___", "\\", "/") - which are to be used for the inner and outer board edges repsectively. + which are to be used for the inner and outer board edges respectively. Board groups are drawn sequentially with later board groups obscuring earlier ones when their edges or boards overlap. dead_links : set([(x, y, z, link), ...]) @@ -411,7 +414,7 @@ def render_boards(board_groups, dead_links=frozenset(), all_boards = set() for boards, label, edge_inner, edge_outer in board_groups: - # Convert to Cartesian coords + # Convert to Cartesian coordinates boards = set(_board_to_cartesian(x, y, z) for x, y, z in boards) all_boards.update(boards) From 3a2ff6b8ae4f9e0008b871f2c233911567e791ea Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 26 Jun 2024 12:12:35 +0100 Subject: [PATCH 06/55] fix f string --- spalloc_client/term.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spalloc_client/term.py b/spalloc_client/term.py index 9edec238b..e4f1392f4 100644 --- a/spalloc_client/term.py +++ b/spalloc_client/term.py @@ -151,7 +151,7 @@ def set_attrs(self, attrs=tuple()): """ if not attrs: return "" - return self(f"\033[{''.join(str(attr) for attr in attrs)}m") + return self(f"\033[{';'.join(str(attr) for attr in attrs)}m") def wrap(self, string=None, pre="", post=""): """ Wrap a string in the suppled pre and post strings or just print\ From 1d60bd26a3154ea265fc3a90a004907f27029976 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 26 Jun 2024 16:07:12 +0100 Subject: [PATCH 07/55] f string --- spalloc_client/scripts/alloc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spalloc_client/scripts/alloc.py b/spalloc_client/scripts/alloc.py index dab1feb72..7ace21c20 100644 --- a/spalloc_client/scripts/alloc.py +++ b/spalloc_client/scripts/alloc.py @@ -141,7 +141,7 @@ def write_ips_to_csv(connections: Dict[Tuple[int, int], str], """ with open(ip_file_filename, "w", encoding="utf-8") as f: f.write("x,y,hostname\n") - f.write("".join("{},{},{}\n".format(x, y, hostname) + f.write("".join(f"{x},{y},{hostname}\n" for (x, y), hostname in sorted(connections.items()))) @@ -439,7 +439,7 @@ def run_job(job_args: List[str], job_kwargs: Dict[str, str], try: job = Job(*job_args, **job_kwargs) except (OSError, IOError, ProtocolError, ProtocolTimeoutError) as e: - info(t.red("Could not connect to server: {}".format(e))) + info(t.red(f"Could not connect to server: {e}")) return 6 try: From 9eee347b00672f0ea6d4ef18f5ab33a2a4bc7333 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 26 Jun 2024 16:07:24 +0100 Subject: [PATCH 08/55] docs --- spalloc_client/protocol_client.py | 82 +++++++++++++++++++++--------- spalloc_client/scripts/machine.py | 17 +++++-- spalloc_client/scripts/ps.py | 13 +++-- spalloc_client/scripts/support.py | 22 ++++++-- spalloc_client/scripts/where_is.py | 3 ++ 5 files changed, 103 insertions(+), 34 deletions(-) diff --git a/spalloc_client/protocol_client.py b/spalloc_client/protocol_client.py index ceb3cdf20..01ab92199 100644 --- a/spalloc_client/protocol_client.py +++ b/spalloc_client/protocol_client.py @@ -18,8 +18,11 @@ import errno import json import socket -from typing import Dict, Optional +from typing import Dict, List, Optional from threading import current_thread, RLock, local + +from spinn_utilities.typing.json import JsonObject + from spalloc_client._utils import time_left, timed_out, make_timeout @@ -199,7 +202,7 @@ def close(self): self._close(key) self._local = _ProtocolThreadLocal() - def _recv_json(self, timeout=None): + def _recv_json(self, timeout=None) -> JsonObject: """ Receive a line of JSON from the server. Parameters @@ -358,63 +361,93 @@ def wait_for_notification(self, timeout=None): # The bindings of the Spalloc protocol methods themselves; simplifies use # from IDEs. - def version(self, timeout=None): # pragma: no cover + def version(self, timeout: Optional[int] = None) -> JsonObject: + """ Ask what version of spalloc is running. """ return self.call("version", timeout=timeout) - def create_job(self, *args, **kwargs): # pragma: no cover + def create_job(self, *args: List[object], + **kwargs: Dict[str, object]) -> JsonObject: + """ + Start a new job + """ # If no owner, don't bother with the call if "owner" not in kwargs: raise SpallocServerException( "owner must be specified for all jobs.") return self.call("create_job", *args, **kwargs) - def job_keepalive(self, job_id, timeout=None): # pragma: no cover + def job_keepalive(self, job_id: int, + timeout: Optional[int] = None) -> JsonObject: + """ + Send s message to keep the job alive. + + Without these the job will be killed after a while. + """ return self.call("job_keepalive", job_id, timeout=timeout) - def get_job_state(self, job_id, timeout=None): # pragma: no cover + def get_job_state(self, job_id: int, + timeout: Optional[int] = None) -> JsonObject: + """Get the state for this job """ return self.call("get_job_state", job_id, timeout=timeout) - def get_job_machine_info(self, job_id, timeout=None): # pragma: no cover + def get_job_machine_info(self, job_id: int, + timeout: Optional[int] = None) -> JsonObject: + """ Get info for this job. """ return self.call("get_job_machine_info", job_id, timeout=timeout) - def power_on_job_boards(self, job_id, timeout=None): # pragma: no cover + def power_on_job_boards(self, job_id: int, + timeout: Optional[int] = None) -> JsonObject: + """ Turn on the power on the jobs baords. """ return self.call("power_on_job_boards", job_id, timeout=timeout) - def power_off_job_boards(self, job_id, timeout=None): # pragma: no cover + def power_off_job_boards(self, job_id: int, + timeout: Optional[int] = None) -> JsonObject: + """ Turn off the power on the jobs baords. """ return self.call("power_off_job_boards", job_id, timeout=timeout) - def destroy_job(self, job_id, reason=None, - timeout=None): # pragma: no cover + def destroy_job(self, job_id: int, reason: Optional[str] = None, + timeout: Optional[int] = None) -> JsonObject: + """ Destroy the job """ return self.call("destroy_job", job_id, reason, timeout=timeout) - def notify_job(self, job_id=None, timeout=None): # pragma: no cover + def notify_job(self, job_id: Optional[int] = None, + timeout: Optional[int]=None) -> JsonObject: + """ Turn on notification of job status changes. """ return self.call("notify_job", job_id, timeout=timeout) - def no_notify_job(self, job_id=None, timeout=None): # pragma: no cover + def no_notify_job(self, job_id: Optional[int] = None, + timeout: Optional[int]=None) -> JsonObject: + """ Turn off notification of job status changes. """ return self.call("no_notify_job", job_id, timeout=timeout) - def notify_machine(self, machine_name=None, - timeout=None): # pragma: no cover + def notify_machine(self, machine_name: Optional[str] = None, + timeout: Optional[int] = None) -> JsonObject: + """ Turn on notification of machine status changes. """ return self.call("notify_machine", machine_name, timeout=timeout) - def no_notify_machine(self, machine_name=None, - timeout=None): # pragma: no cover + def no_notify_machine(self, machine_name: Optional[str] = None, + timeout: Optional[int] = None) -> JsonObject: + """ Turn off notification of machine status changes. """ return self.call("no_notify_machine", machine_name, timeout=timeout) - def list_jobs(self, timeout=None): # pragma: no cover + def list_jobs(self, timeout: Optional[int] = None) -> JsonObject: + """ Obtains a list of jobs currently running. """ return self.call("list_jobs", timeout=timeout) - def list_machines(self, timeout=None): # pragma: no cover + def list_machines(self, timeout: Optional[float] = None) -> JsonObject: + """ Obtains a list of currently supported machines. """ return self.call("list_machines", timeout=timeout) - def get_board_position(self, machine_name, x, y, z, - timeout=None): # pragma: no cover + def get_board_position(self, machine_name: str, x: int, y: int, z: int, + timeout: Optional[int] = None): # pragma: no cover + """ Gets the position of board x, y, z on the given machine. """ # pylint: disable=too-many-arguments return self.call("get_board_position", machine_name, x, y, z, timeout=timeout) - def get_board_at_position(self, machine_name, x, y, z, - timeout=None): # pragma: no cover + def get_board_at_position(self, machine_name: str, x: int, y: int, z: int, + timeout: Optional[int] = None) -> JsonObject: # pragma: no cover + """ Gets the board x, y, z on the requested machine. """ # pylint: disable=too-many-arguments return self.call("get_board_at_position", machine_name, x, y, z, timeout=timeout) @@ -426,7 +459,8 @@ def get_board_at_position(self, machine_name, x, y, z, frozenset("job_id chip_x chip_y".split())]) def where_is(self, timeout: Optional[int] = None, - **kwargs: Dict[str, object]): + **kwargs: Dict[str, object]) -> JsonObject: + """ Reports where ion the Machine a job is running """ # Test for whether sane arguments are passed. keywords = frozenset(kwargs) if keywords not in ProtocolClient._acceptable_kwargs_for_where_is: diff --git a/spalloc_client/scripts/machine.py b/spalloc_client/scripts/machine.py index e33a0e598..18f9b0134 100644 --- a/spalloc_client/scripts/machine.py +++ b/spalloc_client/scripts/machine.py @@ -33,7 +33,9 @@ from collections import defaultdict import argparse import sys -from spalloc_client import __version__ +from typing import List + +from spalloc_client import __version__, ProtocolClient from spalloc_client.term import ( Terminal, render_table, render_definitions, render_boards, render_cells, DEFAULT_BOARD_EDGES) @@ -211,6 +213,9 @@ def show_machine(t, machines, jobs, machine_name, compact=False): class ListMachinesScript(Script): + """ + A Script object to get information from a spalloc machine. + """ def __init__(self): super().__init__() @@ -250,12 +255,18 @@ def verify_arguments(self, args): self.parser.error( "--detailed only works when a specific machine is specified") - def one_shot(self, client, args): + def one_shot(self, client: ProtocolClient, args: List[object]): + """ + Display the machine info once + """ t = Terminal() # Get all information and display accordingly self.get_and_display_machine_info(client, args, t) - def recurring(self, client, args): + def recurring(self, client: ProtocolClient, args: List[object]): + """ + Repeatedly display the machine info + """ t = Terminal() while True: client.notify_machine(args.machine, timeout=args.timeout) diff --git a/spalloc_client/scripts/ps.py b/spalloc_client/scripts/ps.py index cbad4f33f..1efa5d286 100644 --- a/spalloc_client/scripts/ps.py +++ b/spalloc_client/scripts/ps.py @@ -27,7 +27,9 @@ """ import argparse import sys -from spalloc_client import __version__, JobState +from typing import List + +from spalloc_client import __version__, JobState, ProtocolClient from spalloc_client.term import Terminal, render_table from spalloc_client._utils import render_timestamp from .support import Script @@ -113,6 +115,9 @@ def render_job_list(t, jobs, args): class ProcessListScript(Script): + """ + An object form Job scripts. + """ def get_parser(self, cfg): parser = argparse.ArgumentParser(description="List all active jobs.") parser.add_argument( @@ -128,12 +133,14 @@ def get_parser(self, cfg): help="list only jobs belonging to a particular owner") return parser - def one_shot(self, client, args): + def one_shot(self, client: ProtocolClient, args: List[object]): + """ Gets info on the job list once. """ t = Terminal(stream=sys.stderr) jobs = client.list_jobs(timeout=args.timeout) print(render_job_list(t, jobs, args)) - def recurring(self, client, args): + def recurring(self, client: ProtocolClient, args: List[object]): + """ Repeatedly gets info on the job list. """ client.notify_job(timeout=args.timeout) t = Terminal(stream=sys.stderr) while True: diff --git a/spalloc_client/scripts/support.py b/spalloc_client/scripts/support.py index 7e8dddcb1..2f7353b66 100644 --- a/spalloc_client/scripts/support.py +++ b/spalloc_client/scripts/support.py @@ -13,7 +13,7 @@ # limitations under the License. import sys -from typing import List +from typing import Any, Dict, List, Optional, Tuple from spalloc_client import ( config, ProtocolClient, ProtocolError, ProtocolTimeoutError, SpallocServerException) @@ -25,7 +25,10 @@ class Terminate(Exception): - def __init__(self, code, *args): + def __init__(self, code: int, *args: Tuple[object]): + """ + An Exception that can be used to exit the program. + """ super().__init__() self._code = code args = list(args) @@ -38,12 +41,16 @@ def __init__(self, code, *args): self._msg = message def exit(self): + """ Exit the program after priintg and erro msg. """ if self._msg is not None: sys.stderr.write(self._msg + "\n") sys.exit(self._code) -def version_verify(client, timeout): +def version_verify(client: ProtocolClient, timeout: Optional[int]): + """ + Verify that the current version of the client is compatible + """ version = tuple(map(int, client.version(timeout=timeout).split("."))) if not (VERSION_RANGE_START <= version < VERSION_RANGE_STOP): raise Terminate(2, "Incompatible server version ({})", @@ -51,6 +58,7 @@ def version_verify(client, timeout): class Script(object): + """ Base class of various Scopt Objects. """ def __init__(self): self.client_factory = ProtocolClient @@ -69,7 +77,13 @@ def body(self, client: ProtocolClient, args: List[str]): obtained and verified to be compatible. """ - def build_server_arg_group(self, server_args, cfg): + def build_server_arg_group(self, server_args: Any, + cfg: Dict[str, object]): + """ + Adds a few more arguements + + :param argparse._ArguementGroup server_args: + """ server_args.add_argument( "--hostname", "-H", default=cfg["hostname"], help="hostname or IP of the spalloc server (default: %(default)s)") diff --git a/spalloc_client/scripts/where_is.py b/spalloc_client/scripts/where_is.py index 26731b10b..d5e077fd1 100644 --- a/spalloc_client/scripts/where_is.py +++ b/spalloc_client/scripts/where_is.py @@ -72,6 +72,9 @@ class WhereIsScript(Script): + """ + An script object to find where a board is + """ def __init__(self): super().__init__() From 41daf6815e5a71e727be500ce69678ce7151bfd9 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 26 Jun 2024 16:23:40 +0100 Subject: [PATCH 09/55] flake8 --- spalloc_client/protocol_client.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/spalloc_client/protocol_client.py b/spalloc_client/protocol_client.py index 01ab92199..d0f454727 100644 --- a/spalloc_client/protocol_client.py +++ b/spalloc_client/protocol_client.py @@ -396,7 +396,7 @@ def get_job_machine_info(self, job_id: int, return self.call("get_job_machine_info", job_id, timeout=timeout) def power_on_job_boards(self, job_id: int, - timeout: Optional[int] = None) -> JsonObject: + timeout: Optional[int] = None) -> JsonObject: """ Turn on the power on the jobs baords. """ return self.call("power_on_job_boards", job_id, timeout=timeout) @@ -411,12 +411,12 @@ def destroy_job(self, job_id: int, reason: Optional[str] = None, return self.call("destroy_job", job_id, reason, timeout=timeout) def notify_job(self, job_id: Optional[int] = None, - timeout: Optional[int]=None) -> JsonObject: + timeout: Optional[int] = None) -> JsonObject: """ Turn on notification of job status changes. """ return self.call("notify_job", job_id, timeout=timeout) def no_notify_job(self, job_id: Optional[int] = None, - timeout: Optional[int]=None) -> JsonObject: + timeout: Optional[int] = None) -> JsonObject: """ Turn off notification of job status changes. """ return self.call("no_notify_job", job_id, timeout=timeout) @@ -446,7 +446,8 @@ def get_board_position(self, machine_name: str, x: int, y: int, z: int, timeout=timeout) def get_board_at_position(self, machine_name: str, x: int, y: int, z: int, - timeout: Optional[int] = None) -> JsonObject: # pragma: no cover + timeout: Optional[int] = None + ) -> JsonObject: # pragma: no cover """ Gets the board x, y, z on the requested machine. """ # pylint: disable=too-many-arguments return self.call("get_board_at_position", machine_name, x, y, z, From 46e3bae36508aa743de69fd7391b63162f94518b Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 26 Jun 2024 16:29:38 +0100 Subject: [PATCH 10/55] flake8 --- spalloc_client/protocol_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spalloc_client/protocol_client.py b/spalloc_client/protocol_client.py index d0f454727..bf04641f5 100644 --- a/spalloc_client/protocol_client.py +++ b/spalloc_client/protocol_client.py @@ -411,7 +411,7 @@ def destroy_job(self, job_id: int, reason: Optional[str] = None, return self.call("destroy_job", job_id, reason, timeout=timeout) def notify_job(self, job_id: Optional[int] = None, - timeout: Optional[int] = None) -> JsonObject: + timeout: Optional[int] = None) -> JsonObject: """ Turn on notification of job status changes. """ return self.call("notify_job", job_id, timeout=timeout) From 6b41eb2c8fc371496ba9d5e9ee40690df976c912 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Jun 2024 06:42:05 +0100 Subject: [PATCH 11/55] remove not working disable --- spalloc_client/scripts/machine.py | 1 - spalloc_client/scripts/ps.py | 1 - 2 files changed, 2 deletions(-) diff --git a/spalloc_client/scripts/machine.py b/spalloc_client/scripts/machine.py index 18f9b0134..13a664427 100644 --- a/spalloc_client/scripts/machine.py +++ b/spalloc_client/scripts/machine.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=wrong-spelling-in-docstring,wrong-spelling-in-comment """ Command-line administrative machine management interface. When called with no arguments the ``spalloc-machine`` command lists all diff --git a/spalloc_client/scripts/ps.py b/spalloc_client/scripts/ps.py index 1efa5d286..652a2dd6f 100644 --- a/spalloc_client/scripts/ps.py +++ b/spalloc_client/scripts/ps.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=wrong-spelling-in-docstring """ An administrative command-line process listing utility. By default, the ``spalloc-ps`` command lists all running and queued jobs. For From ec0212df2902777f2256ad0dea96f62c3def022d Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Jun 2024 06:42:15 +0100 Subject: [PATCH 12/55] fix spellings --- spalloc_client/protocol_client.py | 4 ++-- spalloc_client/scripts/support.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/spalloc_client/protocol_client.py b/spalloc_client/protocol_client.py index bf04641f5..9e634e6f0 100644 --- a/spalloc_client/protocol_client.py +++ b/spalloc_client/protocol_client.py @@ -397,12 +397,12 @@ def get_job_machine_info(self, job_id: int, def power_on_job_boards(self, job_id: int, timeout: Optional[int] = None) -> JsonObject: - """ Turn on the power on the jobs baords. """ + """ Turn on the power on the jobs boards. """ return self.call("power_on_job_boards", job_id, timeout=timeout) def power_off_job_boards(self, job_id: int, timeout: Optional[int] = None) -> JsonObject: - """ Turn off the power on the jobs baords. """ + """ Turn off the power on the jobs boards. """ return self.call("power_off_job_boards", job_id, timeout=timeout) def destroy_job(self, job_id: int, reason: Optional[str] = None, diff --git a/spalloc_client/scripts/support.py b/spalloc_client/scripts/support.py index 2f7353b66..028e00797 100644 --- a/spalloc_client/scripts/support.py +++ b/spalloc_client/scripts/support.py @@ -41,7 +41,7 @@ def __init__(self, code: int, *args: Tuple[object]): self._msg = message def exit(self): - """ Exit the program after priintg and erro msg. """ + """ Exit the program after printing an error msg. """ if self._msg is not None: sys.stderr.write(self._msg + "\n") sys.exit(self._code) @@ -58,7 +58,7 @@ def version_verify(client: ProtocolClient, timeout: Optional[int]): class Script(object): - """ Base class of various Scopt Objects. """ + """ Base class of various Script Objects. """ def __init__(self): self.client_factory = ProtocolClient @@ -80,7 +80,7 @@ def body(self, client: ProtocolClient, args: List[str]): def build_server_arg_group(self, server_args: Any, cfg: Dict[str, object]): """ - Adds a few more arguements + Adds a few more arguments :param argparse._ArguementGroup server_args: """ From 3b797d2b85d3b5ddadcc7f2a9d1e965989aa140e Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Jun 2024 06:53:12 +0100 Subject: [PATCH 13/55] fix a spelling --- spalloc_client/scripts/ps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spalloc_client/scripts/ps.py b/spalloc_client/scripts/ps.py index 652a2dd6f..f6d70a949 100644 --- a/spalloc_client/scripts/ps.py +++ b/spalloc_client/scripts/ps.py @@ -19,7 +19,7 @@ added. .. image:: _static/spalloc_ps.png - :alt: Jobs being listed by spalloc-ps + :alt: Jobs being listed by spalloc- proces scripts This list may be filtered by owner or machine with the ``--owner`` and ``--machine`` arguments. From 05284f1cd14c886263c08d498bcc123cc433e620 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Jun 2024 06:59:42 +0100 Subject: [PATCH 14/55] docs --- spalloc_client/scripts/machine.py | 4 +++- spalloc_client/scripts/ps.py | 2 +- spalloc_client/scripts/support.py | 9 +++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/spalloc_client/scripts/machine.py b/spalloc_client/scripts/machine.py index 13a664427..3da9efcc4 100644 --- a/spalloc_client/scripts/machine.py +++ b/spalloc_client/scripts/machine.py @@ -220,7 +220,9 @@ def __init__(self): super().__init__() self.parser = None - def get_and_display_machine_info(self, client, args, t): + def get_and_display_machine_info( + self, client: ProtocolClient, args: List[object], t: Terminal): + """ Gets and displays info for the machine(s) """ # Get all information machines = client.list_machines(timeout=args.timeout) jobs = client.list_jobs(timeout=args.timeout) diff --git a/spalloc_client/scripts/ps.py b/spalloc_client/scripts/ps.py index f6d70a949..e3613c29e 100644 --- a/spalloc_client/scripts/ps.py +++ b/spalloc_client/scripts/ps.py @@ -19,7 +19,7 @@ added. .. image:: _static/spalloc_ps.png - :alt: Jobs being listed by spalloc- proces scripts + :alt: Jobs being listed by spalloc- process scripts This list may be filtered by owner or machine with the ``--owner`` and ``--machine`` arguments. diff --git a/spalloc_client/scripts/support.py b/spalloc_client/scripts/support.py index 028e00797..27311d298 100644 --- a/spalloc_client/scripts/support.py +++ b/spalloc_client/scripts/support.py @@ -25,11 +25,12 @@ class Terminate(Exception): + """ + An Exception that can be used to exit the program. + """ + def __init__(self, code: int, *args: Tuple[object]): - """ - An Exception that can be used to exit the program. - """ - super().__init__() + super().__init__() self._code = code args = list(args) message = args.pop(0) if args else None From 277342b617042deb00f5b724f5b0a11b9427bf1d Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Jun 2024 07:00:41 +0100 Subject: [PATCH 15/55] spacing --- spalloc_client/scripts/support.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spalloc_client/scripts/support.py b/spalloc_client/scripts/support.py index 27311d298..ebb500d94 100644 --- a/spalloc_client/scripts/support.py +++ b/spalloc_client/scripts/support.py @@ -30,7 +30,7 @@ class Terminate(Exception): """ def __init__(self, code: int, *args: Tuple[object]): - super().__init__() + super().__init__() self._code = code args = list(args) message = args.pop(0) if args else None From fe94c65914d497f63c2835a42b302e94cb631d66 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Jun 2024 07:29:25 +0100 Subject: [PATCH 16/55] types-appdirs --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 98c24636f..6d9ee7d4b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -62,6 +62,7 @@ test = # pytest will be brought in by pytest-cov pytest-cov mock + types-appdirs [options.entry_points] console_scripts = From 992ab5d87204020349806731329e14f65fcff169 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Jun 2024 07:29:35 +0100 Subject: [PATCH 17/55] typing --- spalloc_client/job.py | 3 +-- spalloc_client/protocol_client.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/spalloc_client/job.py b/spalloc_client/job.py index 8c1b3ff46..1ebb9a7ce 100644 --- a/spalloc_client/job.py +++ b/spalloc_client/job.py @@ -763,8 +763,7 @@ class _JobStateTuple(namedtuple("_JobStateTuple", reason the job was terminated. """ - # Python 3.4 Workaround: https://bugs.python.org/issue24931 - __slots__ = tuple() + __slots__ = () class _JobMachineInfoTuple(namedtuple("_JobMachineInfoTuple", diff --git a/spalloc_client/protocol_client.py b/spalloc_client/protocol_client.py index 9e634e6f0..cd7f7308c 100644 --- a/spalloc_client/protocol_client.py +++ b/spalloc_client/protocol_client.py @@ -459,8 +459,7 @@ def get_board_at_position(self, machine_name: str, x: int, y: int, z: int, frozenset("machine chip_x chip_y".split()), frozenset("job_id chip_x chip_y".split())]) - def where_is(self, timeout: Optional[int] = None, - **kwargs: Dict[str, object]) -> JsonObject: + def where_is(self, timeout: Optional[int] = None, **kwargs) -> JsonObject: """ Reports where ion the Machine a job is running """ # Test for whether sane arguments are passed. keywords = frozenset(kwargs) From e3abf1ae8a24ae53ffb1b9f047512180f57e8d26 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Jun 2024 07:47:19 +0100 Subject: [PATCH 18/55] simplify Terminate exception --- spalloc_client/scripts/job.py | 16 ++++++++-------- spalloc_client/scripts/machine.py | 2 +- spalloc_client/scripts/support.py | 16 +++++----------- 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/spalloc_client/scripts/job.py b/spalloc_client/scripts/job.py index 1447bfbfc..2e877328c 100644 --- a/spalloc_client/scripts/job.py +++ b/spalloc_client/scripts/job.py @@ -241,9 +241,9 @@ def power_job(client, timeout, job_id, power): raise Terminate(7) from exc else: # In an unknown state, perhaps the job was queued etc. - raise Terminate(8, "Error: Cannot power {} job {} in state {}", - "on" if power else "off", - job_id, _state_name(state)) + raise Terminate( + 8, (f"Error: Cannot power {'on' if power else 'off'} " + f"job {job_id} in state {_state_name(state)}")) def list_ips(client, timeout, job_id): @@ -267,7 +267,7 @@ def list_ips(client, timeout, job_id): info = client.get_job_machine_info(job_id, timeout=timeout) connections = info["connections"] if connections is None: - raise Terminate(9, "Job {} is queued or does not exist", job_id) + raise Terminate(9, f"Job {job_id} is queued or does not exist") print("x,y,hostname") for ((x, y), hostname) in sorted(connections): print("{},{},{}".format(x, y, hostname)) @@ -308,11 +308,11 @@ def get_job_id(self, client, args): jobs = client.list_jobs(timeout=args.timeout) job_ids = [job["job_id"] for job in jobs if job["owner"] == args.owner] if not job_ids: - raise Terminate(3, "Owner {} has no live jobs", args.owner) + raise Terminate(3, f"Owner {args.owner} has no live jobs") elif len(job_ids) > 1: - raise Terminate(3, "Ambiguous: {} has {} live jobs: {}", - args.owner, len(job_ids), - ", ".join(map(str, job_ids))) + msg = (f"Ambiguous: {args.owner} has {len(job_ids)} live jobs: " + f"{', '.join(map(str, job_ids))}") + raise Terminate(3, msg) return job_ids[0] def get_parser(self, cfg): diff --git a/spalloc_client/scripts/machine.py b/spalloc_client/scripts/machine.py index 237dc9294..162cc3323 100644 --- a/spalloc_client/scripts/machine.py +++ b/spalloc_client/scripts/machine.py @@ -98,7 +98,7 @@ def _get_machine(machines, machine_name): if machine["name"] == machine_name: return machine # No matching machine - raise Terminate(6, "No machine '{}' was found", machine_name) + raise Terminate(6, f"No machine '{machine_name}' was found") def show_machine(t, machines, jobs, machine_name, compact=False): diff --git a/spalloc_client/scripts/support.py b/spalloc_client/scripts/support.py index af6a3f488..1cc76b271 100644 --- a/spalloc_client/scripts/support.py +++ b/spalloc_client/scripts/support.py @@ -13,6 +13,7 @@ # limitations under the License. import sys +from typing import Optional from spalloc_client import ( config, ProtocolClient, ProtocolError, ProtocolTimeoutError, SpallocServerException) @@ -24,17 +25,10 @@ class Terminate(Exception): - def __init__(self, code, *args): + def __init__(self, code: int, message: Optional[str] = None): super().__init__() self._code = code - args = list(args) - message = args.pop(0) if args else None - if message is None: - self._msg = None - elif args: - self._msg = message.format(*args) - else: - self._msg = message + self._msg = message def exit(self): if self._msg is not None: @@ -45,8 +39,8 @@ def exit(self): def version_verify(client, timeout): version = tuple(map(int, client.version(timeout=timeout).split("."))) if not (VERSION_RANGE_START <= version < VERSION_RANGE_STOP): - raise Terminate(2, "Incompatible server version ({})", - ".".join(map(str, version))) + raise Terminate( + 2, f"Incompatible server version ({'.'.join(map(str, version))})") class Script(object): From f1f086e14766b4dc019d367181f634feb351786c Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Jun 2024 07:56:36 +0100 Subject: [PATCH 19/55] Flake8 --- spalloc_client/scripts/support.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spalloc_client/scripts/support.py b/spalloc_client/scripts/support.py index dda241223..b3f8367c9 100644 --- a/spalloc_client/scripts/support.py +++ b/spalloc_client/scripts/support.py @@ -13,7 +13,7 @@ # limitations under the License. import sys -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional from spalloc_client import ( config, ProtocolClient, ProtocolError, ProtocolTimeoutError, SpallocServerException) From fa611b9670fd56d7a4720807d5404749873246f0 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Jun 2024 07:58:45 +0100 Subject: [PATCH 20/55] docs --- spalloc_client/scripts/support.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spalloc_client/scripts/support.py b/spalloc_client/scripts/support.py index b3f8367c9..ac7ca5a38 100644 --- a/spalloc_client/scripts/support.py +++ b/spalloc_client/scripts/support.py @@ -25,6 +25,7 @@ class Terminate(Exception): + """ Exception that can be used to exit the code """ def __init__(self, code: int, message: Optional[str] = None): super().__init__() self._code = code From a29be98e18e9f6ce5051ce9f5e0ca2161ec26a77 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Jun 2024 09:43:25 +0100 Subject: [PATCH 21/55] fix types --- spalloc_client/config.py | 3 ++- spalloc_client/job.py | 3 +-- spalloc_client/scripts/alloc.py | 17 ++++++++++------- spalloc_client/scripts/job.py | 8 ++++---- spalloc_client/scripts/machine.py | 6 +++--- spalloc_client/scripts/ps.py | 8 ++++---- tests/conftest.py | 2 +- tests/scripts/test_alloc.py | 3 +-- tests/scripts/test_job_script.py | 2 +- tests/scripts/test_machine.py | 2 +- tests/scripts/test_ps.py | 2 +- tests/scripts/test_where_is.py | 2 +- tests/test_job.py | 2 +- tests/test_protocol_client.py | 2 +- 14 files changed, 32 insertions(+), 30 deletions(-) diff --git a/spalloc_client/config.py b/spalloc_client/config.py index cff41673f..c92ed92d7 100644 --- a/spalloc_client/config.py +++ b/spalloc_client/config.py @@ -82,6 +82,7 @@ import configparser import os.path import appdirs +from typing import Any, Dict, List, Optional # The application name to use in config file names _name = "spalloc" @@ -138,7 +139,7 @@ def _read_none_or_str(parser, option): return parser.get(SECTION, option) -def read_config(filenames=None): +def read_config(filenames: Optional[List[str]] = None) -> Dict[str, Any]: """ Attempt to read local configuration files to determine spalloc client settings. diff --git a/spalloc_client/job.py b/spalloc_client/job.py index 1ebb9a7ce..8a5b7acd7 100644 --- a/spalloc_client/job.py +++ b/spalloc_client/job.py @@ -790,5 +790,4 @@ class _JobMachineInfoTuple(namedtuple("_JobMachineInfoTuple", None if none allocated yet. """ - # Python 3.4 Workaround: https://bugs.python.org/issue24931 - __slots__ = tuple() + __slots__ = () diff --git a/spalloc_client/scripts/alloc.py b/spalloc_client/scripts/alloc.py index 7ace21c20..43b67faf2 100644 --- a/spalloc_client/scripts/alloc.py +++ b/spalloc_client/scripts/alloc.py @@ -121,8 +121,8 @@ from spalloc_client.term import Terminal, render_definitions # pylint: disable=invalid-name -arguments = None -t = None +arguments: Optional[argparse.Namespace] = None +t: Optional[Terminal] = None _input = input # This is so we can monkey patch input during testing @@ -231,7 +231,7 @@ def run_command( logging.info("All board IPs listed in: %s", ip_file_filename) # Make substitutions in command arguments - command = [arg.format(root_hostname, + commands = [arg.format(root_hostname, hostname=root_hostname, w=width, width=width, @@ -243,7 +243,7 @@ def run_command( # NB: When using shell=True, commands should be given as a string rather # than the usual list of arguments. - command = " ".join(map(quote, command)) + command = " ".join(map(quote, commands)) p = subprocess.Popen(command, shell=True) # Pass through keyboard interrupts @@ -258,6 +258,7 @@ def info(msg: str): """ Writes a message to the terminal """ + assert t is not None if not arguments.quiet: t.stream.write(f"{msg}\n") @@ -273,6 +274,7 @@ def wait_for_job_ready(job: Job): """ Wait for it to become ready, keeping the user informed along the way """ + assert t is not None old_state = None cur_state = job.state try: @@ -432,6 +434,7 @@ def run_job(job_args: List[str], job_kwargs: Dict[str, str], """ Run a job """ + assert arguments is not None # Reason for destroying the job reason = None @@ -470,14 +473,14 @@ def run_job(job_args: List[str], job_kwargs: Dict[str, str], job.destroy(reason) -def _minzero(value: Optional[float]) -> Optional[float]: +def _minzero(value: float) -> Optional[float]: """ - Makes sure a value is not negative. + Replaces a negative value with None """ return value if value >= 0.0 else None -def main(argv: List[str] = None): +def main(argv: Optional[List[str]] = None): """ The main method run """ diff --git a/spalloc_client/scripts/job.py b/spalloc_client/scripts/job.py index 1a046c840..e0d051d9d 100644 --- a/spalloc_client/scripts/job.py +++ b/spalloc_client/scripts/job.py @@ -76,7 +76,7 @@ """ import argparse import sys -from typing import List +from typing import Dict from spalloc_client import __version__, JobState from spalloc_client.term import ( @@ -307,7 +307,7 @@ def __init__(self): super().__init__() self.parser = None - def get_job_id(self, client: ProtocolClient, args: List[str]): + def get_job_id(self, client: ProtocolClient, args: argparse.Namespace): """ get a job for the owner named in the args """ @@ -324,7 +324,7 @@ def get_job_id(self, client: ProtocolClient, args: List[str]): raise Terminate(3, msg) return job_ids[0] - def get_parser(self, cfg): + def get_parser(self, cfg: Dict[str, Any]) -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Manage running jobs.") parser.add_argument( @@ -359,7 +359,7 @@ def get_parser(self, cfg): self.parser = parser return parser - def verify_arguments(self, args): + def verify_arguments(self, args: argparse.Namespace): if args.job_id is None and args.owner is None: self.parser.error("job ID (or --owner) not specified") diff --git a/spalloc_client/scripts/machine.py b/spalloc_client/scripts/machine.py index af5d42256..f7aa45a75 100644 --- a/spalloc_client/scripts/machine.py +++ b/spalloc_client/scripts/machine.py @@ -220,8 +220,8 @@ def __init__(self): super().__init__() self.parser = None - def get_and_display_machine_info( - self, client: ProtocolClient, args: List[object], t: Terminal): + def get_and_display_machine_info(self, client: ProtocolClient, + args: argparse.Namespace, t: Terminal): """ Gets and displays info for the machine(s) """ # Get all information machines = client.list_machines(timeout=args.timeout) @@ -264,7 +264,7 @@ def one_shot(self, client: ProtocolClient, args: List[object]): # Get all information and display accordingly self.get_and_display_machine_info(client, args, t) - def recurring(self, client: ProtocolClient, args: List[object]): + def recurring(self, client: ProtocolClient, args: argparse.Namespace): """ Repeatedly display the machine info """ diff --git a/spalloc_client/scripts/ps.py b/spalloc_client/scripts/ps.py index e3613c29e..bca7d9037 100644 --- a/spalloc_client/scripts/ps.py +++ b/spalloc_client/scripts/ps.py @@ -26,15 +26,15 @@ """ import argparse import sys -from typing import List from spalloc_client import __version__, JobState, ProtocolClient from spalloc_client.term import Terminal, render_table from spalloc_client._utils import render_timestamp +from spinn_utilities.typing.json import JsonObject from .support import Script -def render_job_list(t, jobs, args): +def render_job_list(t: Terminal, jobs: JsonObject, args: argparse.Namespace): """ Return a human-readable process listing. Parameters @@ -132,13 +132,13 @@ def get_parser(self, cfg): help="list only jobs belonging to a particular owner") return parser - def one_shot(self, client: ProtocolClient, args: List[object]): + def one_shot(self, client: ProtocolClient, args: argparse.Namespace): """ Gets info on the job list once. """ t = Terminal(stream=sys.stderr) jobs = client.list_jobs(timeout=args.timeout) print(render_job_list(t, jobs, args)) - def recurring(self, client: ProtocolClient, args: List[object]): + def recurring(self, client: ProtocolClient, args: argparse.Namespace): """ Repeatedly gets info on the job list. """ client.notify_job(timeout=args.timeout) t = Terminal(stream=sys.stderr) diff --git a/tests/conftest.py b/tests/conftest.py index c93a1557d..38cffe249 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,7 +17,7 @@ import threading import tempfile import pytest -from mock import Mock +from mock import Mock # type: ignore[import] from spalloc_client import ProtocolClient from spalloc_client.config import SEARCH_PATH from .common import MockServer diff --git a/tests/scripts/test_alloc.py b/tests/scripts/test_alloc.py index bdc647a97..c672f3a70 100644 --- a/tests/scripts/test_alloc.py +++ b/tests/scripts/test_alloc.py @@ -15,11 +15,10 @@ import os import tempfile import pytest -from mock import Mock, PropertyMock +from mock import Mock, PropertyMock # type: ignore[import] from spalloc_client import JobState, JobDestroyedError from spalloc_client.scripts.alloc import ( write_ips_to_csv, print_info, run_command, main) -# pylint: disable=redefined-outer-name, unused-argument @pytest.yield_fixture diff --git a/tests/scripts/test_job_script.py b/tests/scripts/test_job_script.py index f0b9808f3..dc95e44d8 100644 --- a/tests/scripts/test_job_script.py +++ b/tests/scripts/test_job_script.py @@ -14,7 +14,7 @@ import datetime import pytest -from mock import Mock, MagicMock +from mock import Mock, MagicMock # type: ignore[import] from spalloc_client import JobState, ProtocolError from spalloc_client.term import Terminal from spalloc_client.scripts.job import ( diff --git a/tests/scripts/test_machine.py b/tests/scripts/test_machine.py index 00dedf49f..5b06119fd 100644 --- a/tests/scripts/test_machine.py +++ b/tests/scripts/test_machine.py @@ -13,7 +13,7 @@ # limitations under the License. import pytest -from mock import Mock, MagicMock +from mock import Mock, MagicMock # type: ignore[import] from spalloc_client.term import Terminal from spalloc_client.scripts.machine import ( main, generate_keys, list_machines, show_machine) diff --git a/tests/scripts/test_ps.py b/tests/scripts/test_ps.py index b52b71700..2d655b2eb 100644 --- a/tests/scripts/test_ps.py +++ b/tests/scripts/test_ps.py @@ -14,7 +14,7 @@ import collections import datetime -from mock import Mock, MagicMock +from mock import Mock, MagicMock# type: ignore[import] import pytest from spalloc_client.scripts.ps import main, render_job_list from spalloc_client.scripts.support import ( diff --git a/tests/scripts/test_where_is.py b/tests/scripts/test_where_is.py index 097598b42..bdead4d71 100644 --- a/tests/scripts/test_where_is.py +++ b/tests/scripts/test_where_is.py @@ -13,7 +13,7 @@ # limitations under the License. import pytest -from mock import Mock, MagicMock +from mock import Mock, MagicMock # type: ignore[import] from spalloc_client.scripts.where_is import main from spalloc_client.scripts.support import ( VERSION_RANGE_START, VERSION_RANGE_STOP) diff --git a/tests/test_job.py b/tests/test_job.py index 098a9ffa4..50e7df9cb 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -15,7 +15,7 @@ import time from threading import Thread, Event import pytest -from mock import Mock +from mock import Mock # type: ignore[import] from spalloc_client import ( Job, JobState, JobDestroyedError, ProtocolTimeoutError) from spalloc_client._keepalive_process import keep_job_alive diff --git a/tests/test_protocol_client.py b/tests/test_protocol_client.py index 0eef573fc..478b7e538 100644 --- a/tests/test_protocol_client.py +++ b/tests/test_protocol_client.py @@ -17,7 +17,7 @@ import time import logging import pytest -from mock import Mock +from mock import Mock # type: ignore[import] from spalloc_client import ( ProtocolClient, SpallocServerException, ProtocolTimeoutError, ProtocolError) From 19b5ba0a33c19a3a47683350d38258ec2d8e34a9 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Jun 2024 09:55:18 +0100 Subject: [PATCH 22/55] flake8 --- spalloc_client/scripts/alloc.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/spalloc_client/scripts/alloc.py b/spalloc_client/scripts/alloc.py index 43b67faf2..2e1b18a6d 100644 --- a/spalloc_client/scripts/alloc.py +++ b/spalloc_client/scripts/alloc.py @@ -232,13 +232,13 @@ def run_command( # Make substitutions in command arguments commands = [arg.format(root_hostname, - hostname=root_hostname, - w=width, - width=width, - h=height, - height=height, - ethernet_ips=ip_file_filename, - id=job_id) + hostname=root_hostname, + w=width, + width=width, + h=height, + height=height, + ethernet_ips=ip_file_filename, + id=job_id) for arg in command] # NB: When using shell=True, commands should be given as a string rather From 6b2fa2c9d109a07673114c6181b669e12392ed01 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Jun 2024 10:01:26 +0100 Subject: [PATCH 23/55] flake8 --- spalloc_client/scripts/alloc.py | 2 +- spalloc_client/scripts/job.py | 2 +- tests/scripts/test_ps.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spalloc_client/scripts/alloc.py b/spalloc_client/scripts/alloc.py index 2e1b18a6d..af55d53e6 100644 --- a/spalloc_client/scripts/alloc.py +++ b/spalloc_client/scripts/alloc.py @@ -239,7 +239,7 @@ def run_command( height=height, ethernet_ips=ip_file_filename, id=job_id) - for arg in command] + for arg in command] # NB: When using shell=True, commands should be given as a string rather # than the usual list of arguments. diff --git a/spalloc_client/scripts/job.py b/spalloc_client/scripts/job.py index e0d051d9d..1a6fdf91d 100644 --- a/spalloc_client/scripts/job.py +++ b/spalloc_client/scripts/job.py @@ -76,7 +76,7 @@ """ import argparse import sys -from typing import Dict +from typing import Any, Dict from spalloc_client import __version__, JobState from spalloc_client.term import ( diff --git a/tests/scripts/test_ps.py b/tests/scripts/test_ps.py index 2d655b2eb..5326f2934 100644 --- a/tests/scripts/test_ps.py +++ b/tests/scripts/test_ps.py @@ -14,7 +14,7 @@ import collections import datetime -from mock import Mock, MagicMock# type: ignore[import] +from mock import Mock, MagicMock # type: ignore[import] import pytest from spalloc_client.scripts.ps import main, render_job_list from spalloc_client.scripts.support import ( From 4508986443e60347a06aa0c57cb1ac44d5d81177 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Jun 2024 10:08:22 +0100 Subject: [PATCH 24/55] reorder imports --- spalloc_client/config.py | 3 ++- spalloc_client/scripts/ps.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/spalloc_client/config.py b/spalloc_client/config.py index c92ed92d7..880afd4a4 100644 --- a/spalloc_client/config.py +++ b/spalloc_client/config.py @@ -81,9 +81,10 @@ """ import configparser import os.path -import appdirs from typing import Any, Dict, List, Optional +import appdirs + # The application name to use in config file names _name = "spalloc" diff --git a/spalloc_client/scripts/ps.py b/spalloc_client/scripts/ps.py index bca7d9037..c48171f34 100644 --- a/spalloc_client/scripts/ps.py +++ b/spalloc_client/scripts/ps.py @@ -27,10 +27,11 @@ import argparse import sys +from spinn_utilities.typing.json import JsonObject + from spalloc_client import __version__, JobState, ProtocolClient from spalloc_client.term import Terminal, render_table from spalloc_client._utils import render_timestamp -from spinn_utilities.typing.json import JsonObject from .support import Script From d4140554b217c68f7f7aca583acaaba52e33e409 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Jun 2024 11:12:34 +0100 Subject: [PATCH 25/55] typing --- spalloc_client/scripts/machine.py | 6 +++--- spalloc_client/scripts/ps.py | 4 ++-- spalloc_client/term.py | 6 +++++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/spalloc_client/scripts/machine.py b/spalloc_client/scripts/machine.py index f7aa45a75..d64d83484 100644 --- a/spalloc_client/scripts/machine.py +++ b/spalloc_client/scripts/machine.py @@ -37,7 +37,7 @@ from spalloc_client import __version__, ProtocolClient from spalloc_client.term import ( Terminal, render_table, render_definitions, render_boards, render_cells, - DEFAULT_BOARD_EDGES) + DEFAULT_BOARD_EDGES, TableList) from .support import Terminate, Script @@ -74,7 +74,7 @@ def list_machines(t, machines, jobs): for job in jobs: machine_jobs[job["allocated_machine_name"]].append(job) - table = [[ + table: TableList = [[ (t.underscore_bright, "Name"), (t.underscore_bright, "Num boards"), (t.underscore_bright, "In-use"), @@ -191,7 +191,7 @@ def show_machine(t, machines, jobs, machine_name, compact=False): print(render_cells(cells)) else: # In non-compact mode, produce a full table of job information - job_table = [[ + job_table: TableList = [[ (t.underscore_bright, "Key"), (t.underscore_bright, "Job ID"), (t.underscore_bright, "Num boards"), diff --git a/spalloc_client/scripts/ps.py b/spalloc_client/scripts/ps.py index c48171f34..60efdccc1 100644 --- a/spalloc_client/scripts/ps.py +++ b/spalloc_client/scripts/ps.py @@ -30,7 +30,7 @@ from spinn_utilities.typing.json import JsonObject from spalloc_client import __version__, JobState, ProtocolClient -from spalloc_client.term import Terminal, render_table +from spalloc_client.term import Terminal, render_table, TableList from spalloc_client._utils import render_timestamp from .support import Script @@ -49,7 +49,7 @@ def render_job_list(t: Terminal, jobs: JsonObject, args: argparse.Namespace): owner : str or None If not None, only list jobs with this owner. """ - table = [] + table: TableList = [] # Add headings table.append(((t.underscore_bright, "ID"), diff --git a/spalloc_client/term.py b/spalloc_client/term.py index e4f1392f4..65f1814d5 100644 --- a/spalloc_client/term.py +++ b/spalloc_client/term.py @@ -21,9 +21,13 @@ from collections import defaultdict from enum import IntEnum from functools import partial +from typing import Callable, List, Tuple, Union +from typing_extensions import TypeAlias # pylint: disable=wrong-spelling-in-docstring +TableList: TypeAlias = List[Union[str, + Tuple[Callable[[Union[int, str]], str]]]] class ANSIDisplayAttributes(IntEnum): """ Code numbers of ANSI display attributes for use with `ESC[...m`\ @@ -180,7 +184,7 @@ def __getattr__(self, name): post=self("\033[0m")) -def render_table(table, column_sep=" "): +def render_table(table: TableList, column_sep: str = " "): """ Render an ASCII table with optional ANSI escape codes. An example table:: From 39bf8bfca20a3673d648b56fa561715f98d251cb Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Jun 2024 11:22:02 +0100 Subject: [PATCH 26/55] flake8 --- spalloc_client/term.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spalloc_client/term.py b/spalloc_client/term.py index 65f1814d5..90fb8d41b 100644 --- a/spalloc_client/term.py +++ b/spalloc_client/term.py @@ -27,7 +27,8 @@ # pylint: disable=wrong-spelling-in-docstring TableList: TypeAlias = List[Union[str, - Tuple[Callable[[Union[int, str]], str]]]] + Tuple[Callable[[Union[int, str]], str]]]] + class ANSIDisplayAttributes(IntEnum): """ Code numbers of ANSI display attributes for use with `ESC[...m`\ From 37f5ca4f2b97e1f110d1a8ecfecc49b7646c0d83 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Jun 2024 12:52:06 +0100 Subject: [PATCH 27/55] more typing --- spalloc_client/protocol_client.py | 10 +++++----- spalloc_client/scripts/machine.py | 6 +++--- spalloc_client/scripts/ps.py | 5 +++-- spalloc_client/term.py | 9 ++++++--- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/spalloc_client/protocol_client.py b/spalloc_client/protocol_client.py index cd7f7308c..740c1c225 100644 --- a/spalloc_client/protocol_client.py +++ b/spalloc_client/protocol_client.py @@ -116,7 +116,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): self.close() return False - def _get_connection(self, timeout): + def _get_connection(self, timeout: Optional[int]) -> socket: if self._dead: raise OSError(errno.ENOTCONN, "not connected") connect_needed = False @@ -141,7 +141,7 @@ def _get_connection(self, timeout): sock.settimeout(timeout) return sock - def _do_connect(self, sock): + def _do_connect(self, sock: socket): success = False try: sock.connect((self._hostname, self._port)) @@ -151,10 +151,10 @@ def _do_connect(self, sock): raise return success - def _has_open_socket(self): + def _has_open_socket(self) -> bool: return self._local.sock is not None - def connect(self, timeout=None): + def connect(self, timeout: Optional[int] = None): """(Re)connect to the server. Raises @@ -168,7 +168,7 @@ def connect(self, timeout=None): self._dead = False self._connect(timeout) - def _connect(self, timeout): + def _connect(self, timeout: Optional[int]) -> socket: """ Try to (re)connect to the server. """ try: diff --git a/spalloc_client/scripts/machine.py b/spalloc_client/scripts/machine.py index d64d83484..4449aed84 100644 --- a/spalloc_client/scripts/machine.py +++ b/spalloc_client/scripts/machine.py @@ -32,13 +32,13 @@ from collections import defaultdict import argparse import sys -from typing import List +from typing import Any, Dict, List from spalloc_client import __version__, ProtocolClient from spalloc_client.term import ( Terminal, render_table, render_definitions, render_boards, render_cells, DEFAULT_BOARD_EDGES, TableList) -from .support import Terminate, Script +from spalloc_client.scripts.support import Terminate, Script def generate_keys(alphabet="ABCDEFGHIJKLMNOPQRSTUVWXYZ"): @@ -233,7 +233,7 @@ def get_and_display_machine_info(self, client: ProtocolClient, else: show_machine(t, machines, jobs, args.machine, not args.detailed) - def get_parser(self, cfg): + def get_parser(self, cfg: Dict[str, Any]): parser = argparse.ArgumentParser( description="Get the state of individual machines.") parser.add_argument( diff --git a/spalloc_client/scripts/ps.py b/spalloc_client/scripts/ps.py index 60efdccc1..d4348f494 100644 --- a/spalloc_client/scripts/ps.py +++ b/spalloc_client/scripts/ps.py @@ -27,7 +27,7 @@ import argparse import sys -from spinn_utilities.typing.json import JsonObject +from spinn_utilities.typing.json import JsonObjectArray from spalloc_client import __version__, JobState, ProtocolClient from spalloc_client.term import Terminal, render_table, TableList @@ -35,7 +35,8 @@ from .support import Script -def render_job_list(t: Terminal, jobs: JsonObject, args: argparse.Namespace): +def render_job_list(t: Terminal, jobs: JsonObjectArray, + args: argparse.Namespace): """ Return a human-readable process listing. Parameters diff --git a/spalloc_client/term.py b/spalloc_client/term.py index 90fb8d41b..e2bd5fa13 100644 --- a/spalloc_client/term.py +++ b/spalloc_client/term.py @@ -26,9 +26,10 @@ # pylint: disable=wrong-spelling-in-docstring -TableList: TypeAlias = List[Union[str, - Tuple[Callable[[Union[int, str]], str]]]] - +TableFunction: TypeAlias = Callable[[Union[int, str]], str] +TableValue: TypeAlias = Union[int, str] +TableItem: TypeAlias = Union[TableValue, Tuple[TableFunction, TableValue]] +TableList: TypeAlias = List[TableItem] class ANSIDisplayAttributes(IntEnum): """ Code numbers of ANSI display attributes for use with `ESC[...m`\ @@ -215,6 +216,7 @@ def render_table(table: TableList, column_sep: str = " "): # Determine maximum column widths column_widths = defaultdict(lambda: 0) for row in table: + column: TableItem for i, column in enumerate(row): if isinstance(column, str): string = column @@ -229,6 +231,7 @@ def render_table(table: TableList, column_sep: str = " "): for row in table: rendered_row = [] out.append(rendered_row) + f: TableFunction for i, column in enumerate(row): # Get string length and formatted string if isinstance(column, str): From c17ea0834f5f39d3e694a1c6c643a424a2d2d68a Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Jun 2024 12:59:00 +0100 Subject: [PATCH 28/55] flake8 --- spalloc_client/term.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spalloc_client/term.py b/spalloc_client/term.py index e2bd5fa13..dc2c9cc39 100644 --- a/spalloc_client/term.py +++ b/spalloc_client/term.py @@ -31,6 +31,7 @@ TableItem: TypeAlias = Union[TableValue, Tuple[TableFunction, TableValue]] TableList: TypeAlias = List[TableItem] + class ANSIDisplayAttributes(IntEnum): """ Code numbers of ANSI display attributes for use with `ESC[...m`\ sequences. From 6fa3d147b7d61ca619bf630493dd0523318d1eab Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Jun 2024 13:06:44 +0100 Subject: [PATCH 29/55] typing --- spalloc_client/term.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/spalloc_client/term.py b/spalloc_client/term.py index dc2c9cc39..b726b6e66 100644 --- a/spalloc_client/term.py +++ b/spalloc_client/term.py @@ -216,6 +216,7 @@ def render_table(table: TableList, column_sep: str = " "): """ # Determine maximum column widths column_widths = defaultdict(lambda: 0) + assert isinstance(table, list) for row in table: column: TableItem for i, column in enumerate(row): @@ -249,10 +250,10 @@ def render_table(table: TableList, column_sep: str = " "): right_align = False string = f(string) elif isinstance(column[1], int): - f, string = column - length = len(str(string)) + f, value = column + length = len(str(value)) right_align = True - string = f(string) + string = f(value) else: raise TypeError(f"Unexpected type {column=}") From 4de60312575f20991713f4853175bd175eb5e96e Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Jun 2024 14:23:58 +0100 Subject: [PATCH 30/55] Terminal TableType --- spalloc_client/scripts/machine.py | 6 +++--- spalloc_client/scripts/ps.py | 4 ++-- spalloc_client/term.py | 12 +++++------- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/spalloc_client/scripts/machine.py b/spalloc_client/scripts/machine.py index 4449aed84..82162cd46 100644 --- a/spalloc_client/scripts/machine.py +++ b/spalloc_client/scripts/machine.py @@ -37,7 +37,7 @@ from spalloc_client import __version__, ProtocolClient from spalloc_client.term import ( Terminal, render_table, render_definitions, render_boards, render_cells, - DEFAULT_BOARD_EDGES, TableList) + DEFAULT_BOARD_EDGES, TableType) from spalloc_client.scripts.support import Terminate, Script @@ -74,7 +74,7 @@ def list_machines(t, machines, jobs): for job in jobs: machine_jobs[job["allocated_machine_name"]].append(job) - table: TableList = [[ + table: TableType = [[ (t.underscore_bright, "Name"), (t.underscore_bright, "Num boards"), (t.underscore_bright, "In-use"), @@ -191,7 +191,7 @@ def show_machine(t, machines, jobs, machine_name, compact=False): print(render_cells(cells)) else: # In non-compact mode, produce a full table of job information - job_table: TableList = [[ + job_table: TableType = [[ (t.underscore_bright, "Key"), (t.underscore_bright, "Job ID"), (t.underscore_bright, "Num boards"), diff --git a/spalloc_client/scripts/ps.py b/spalloc_client/scripts/ps.py index d4348f494..c11d260fa 100644 --- a/spalloc_client/scripts/ps.py +++ b/spalloc_client/scripts/ps.py @@ -30,7 +30,7 @@ from spinn_utilities.typing.json import JsonObjectArray from spalloc_client import __version__, JobState, ProtocolClient -from spalloc_client.term import Terminal, render_table, TableList +from spalloc_client.term import Terminal, render_table, TableType from spalloc_client._utils import render_timestamp from .support import Script @@ -50,7 +50,7 @@ def render_job_list(t: Terminal, jobs: JsonObjectArray, owner : str or None If not None, only list jobs with this owner. """ - table: TableList = [] + table: TableType = [] # Add headings table.append(((t.underscore_bright, "ID"), diff --git a/spalloc_client/term.py b/spalloc_client/term.py index b726b6e66..677df37ab 100644 --- a/spalloc_client/term.py +++ b/spalloc_client/term.py @@ -21,16 +21,16 @@ from collections import defaultdict from enum import IntEnum from functools import partial -from typing import Callable, List, Tuple, Union +from typing import Callable, Iterable, Tuple, Union from typing_extensions import TypeAlias # pylint: disable=wrong-spelling-in-docstring TableFunction: TypeAlias = Callable[[Union[int, str]], str] TableValue: TypeAlias = Union[int, str] -TableItem: TypeAlias = Union[TableValue, Tuple[TableFunction, TableValue]] -TableList: TypeAlias = List[TableItem] - +TableColumn: TypeAlias = Union[TableValue, Tuple[TableFunction, TableValue]] +TableRow: TypeAlias = Iterable[TableColumn] +TableType: TypeError = Iterable[TableRow] class ANSIDisplayAttributes(IntEnum): """ Code numbers of ANSI display attributes for use with `ESC[...m`\ @@ -187,7 +187,7 @@ def __getattr__(self, name): post=self("\033[0m")) -def render_table(table: TableList, column_sep: str = " "): +def render_table(table: TableType, column_sep: str = " "): """ Render an ASCII table with optional ANSI escape codes. An example table:: @@ -216,9 +216,7 @@ def render_table(table: TableList, column_sep: str = " "): """ # Determine maximum column widths column_widths = defaultdict(lambda: 0) - assert isinstance(table, list) for row in table: - column: TableItem for i, column in enumerate(row): if isinstance(column, str): string = column From 2e159d06f0932821588aee4df427401ef8dc90be Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Jun 2024 14:26:21 +0100 Subject: [PATCH 31/55] flake8 --- spalloc_client/term.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spalloc_client/term.py b/spalloc_client/term.py index 677df37ab..60146c6c3 100644 --- a/spalloc_client/term.py +++ b/spalloc_client/term.py @@ -32,6 +32,7 @@ TableRow: TypeAlias = Iterable[TableColumn] TableType: TypeError = Iterable[TableRow] + class ANSIDisplayAttributes(IntEnum): """ Code numbers of ANSI display attributes for use with `ESC[...m`\ sequences. From 765441650bb7e42a0283c7e93e8ef5e4d3873211 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Jun 2024 16:17:20 +0100 Subject: [PATCH 32/55] more typing --- spalloc_client/_utils.py | 2 +- spalloc_client/config.py | 2 +- spalloc_client/protocol_client.py | 15 ++++++++------- spalloc_client/scripts/machine.py | 7 +++++-- spalloc_client/scripts/ps.py | 19 +++++++++++++------ spalloc_client/term.py | 30 ++++++++++++++---------------- 6 files changed, 42 insertions(+), 33 deletions(-) diff --git a/spalloc_client/_utils.py b/spalloc_client/_utils.py index d48801124..9f940e730 100644 --- a/spalloc_client/_utils.py +++ b/spalloc_client/_utils.py @@ -40,7 +40,7 @@ def make_timeout(delay_seconds): return time.time() + delay_seconds -def render_timestamp(timestamp): +def render_timestamp(timestamp) -> str: """ Convert a timestamp (Unix seconds) into a local human-readable\ timestamp string. """ diff --git a/spalloc_client/config.py b/spalloc_client/config.py index 880afd4a4..0583046af 100644 --- a/spalloc_client/config.py +++ b/spalloc_client/config.py @@ -83,7 +83,7 @@ import os.path from typing import Any, Dict, List, Optional -import appdirs +import appdirs # type: ignore[import] # The application name to use in config file names _name = "spalloc" diff --git a/spalloc_client/protocol_client.py b/spalloc_client/protocol_client.py index 740c1c225..65bfc5ebd 100644 --- a/spalloc_client/protocol_client.py +++ b/spalloc_client/protocol_client.py @@ -21,7 +21,7 @@ from typing import Dict, List, Optional from threading import current_thread, RLock, local -from spinn_utilities.typing.json import JsonObject +from spinn_utilities.typing.json import JsonObject, JsonObjectArray from spalloc_client._utils import time_left, timed_out, make_timeout @@ -116,7 +116,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): self.close() return False - def _get_connection(self, timeout: Optional[int]) -> socket: + def _get_connection(self, timeout: Optional[int]) -> socket.socket: if self._dead: raise OSError(errno.ENOTCONN, "not connected") connect_needed = False @@ -141,7 +141,7 @@ def _get_connection(self, timeout: Optional[int]) -> socket: sock.settimeout(timeout) return sock - def _do_connect(self, sock: socket): + def _do_connect(self, sock: socket.socket): success = False try: sock.connect((self._hostname, self._port)) @@ -168,7 +168,7 @@ def connect(self, timeout: Optional[int] = None): self._dead = False self._connect(timeout) - def _connect(self, timeout: Optional[int]) -> socket: + def _connect(self, timeout: Optional[int]) -> socket.socket: """ Try to (re)connect to the server. """ try: @@ -361,7 +361,7 @@ def wait_for_notification(self, timeout=None): # The bindings of the Spalloc protocol methods themselves; simplifies use # from IDEs. - def version(self, timeout: Optional[int] = None) -> JsonObject: + def version(self, timeout: Optional[int] = None) -> str: """ Ask what version of spalloc is running. """ return self.call("version", timeout=timeout) @@ -430,11 +430,12 @@ def no_notify_machine(self, machine_name: Optional[str] = None, """ Turn off notification of machine status changes. """ return self.call("no_notify_machine", machine_name, timeout=timeout) - def list_jobs(self, timeout: Optional[int] = None) -> JsonObject: + def list_jobs(self, timeout: Optional[int] = None) -> JsonObjectArray: """ Obtains a list of jobs currently running. """ return self.call("list_jobs", timeout=timeout) - def list_machines(self, timeout: Optional[float] = None) -> JsonObject: + def list_machines(self, + timeout: Optional[float] = None) -> JsonObjectArray: """ Obtains a list of currently supported machines. """ return self.call("list_machines", timeout=timeout) diff --git a/spalloc_client/scripts/machine.py b/spalloc_client/scripts/machine.py index 82162cd46..6fed0effe 100644 --- a/spalloc_client/scripts/machine.py +++ b/spalloc_client/scripts/machine.py @@ -34,6 +34,8 @@ import sys from typing import Any, Dict, List +from spinn_utilities.typing.json import JsonObjectArray + from spalloc_client import __version__, ProtocolClient from spalloc_client.term import ( Terminal, render_table, render_definitions, render_boards, render_cells, @@ -54,7 +56,8 @@ def generate_keys(alphabet="ABCDEFGHIJKLMNOPQRSTUVWXYZ"): yield prefix + symbol -def list_machines(t, machines, jobs): +def list_machines(t: Terminal, machines: JsonObjectArray, + jobs: JsonObjectArray): """ Display a table summarising the available machines and their load. Parameters @@ -177,7 +180,7 @@ def show_machine(t, machines, jobs, machine_name, compact=False): print("") print(render_boards(board_groups, machine["dead_links"], tuple(map(t.red, DEFAULT_BOARD_EDGES)))) - + compact = False # Produce table showing jobs on machine if compact: # In compact mode, produce column-aligned cells diff --git a/spalloc_client/scripts/ps.py b/spalloc_client/scripts/ps.py index c11d260fa..bc9bfa4f4 100644 --- a/spalloc_client/scripts/ps.py +++ b/spalloc_client/scripts/ps.py @@ -25,12 +25,14 @@ ``--machine`` arguments. """ import argparse +from collections.abc import Sized import sys +from typing import cast, Union from spinn_utilities.typing.json import JsonObjectArray from spalloc_client import __version__, JobState, ProtocolClient -from spalloc_client.term import Terminal, render_table, TableType +from spalloc_client.term import Terminal, render_table, TableColumn, TableType from spalloc_client._utils import render_timestamp from .support import Script @@ -71,6 +73,7 @@ def render_job_list(t: Terminal, jobs: JsonObjectArray, continue # Colourise job states + job_state: TableColumn if job["state"] == JobState.queued: job_state = (t.blue, "queue") elif job["state"] == JobState.power: @@ -81,6 +84,7 @@ def render_job_list(t: Terminal, jobs: JsonObjectArray, job_state = str(job["state"]) # Colourise power states + power_state: TableColumn if job["power"] is not None: power_state = (t.green, "on") if job["power"] else (t.red, "off") if job["state"] == JobState.power: @@ -88,22 +92,25 @@ def render_job_list(t: Terminal, jobs: JsonObjectArray, else: power_state = "" - num_boards = "" if job["boards"] is None else len(job["boards"]) - + num_boards: Union[int, str] + if isinstance(job["boards"], Sized): + num_boards = len(job["boards"]) + else: + num_boards = "" # Format start time timestamp = render_timestamp(job["start_time"]) if job["allocated_machine_name"] is not None: - machine_name = job["allocated_machine_name"] + machine_name = str(job["allocated_machine_name"]) else: machine_name = "" - owner = job["owner"] + owner = str(job["owner"]) if "keepalivehost" in job and job["keepalivehost"] is not None: owner += f" ({job['keepalivehost']})" table.append(( - job["job_id"], + cast(int, job["job_id"]), job_state, power_state, num_boards, diff --git a/spalloc_client/term.py b/spalloc_client/term.py index 60146c6c3..64a1ed0fe 100644 --- a/spalloc_client/term.py +++ b/spalloc_client/term.py @@ -21,7 +21,7 @@ from collections import defaultdict from enum import IntEnum from functools import partial -from typing import Callable, Iterable, Tuple, Union +from typing import Callable, Dict, Iterable, List, Tuple, Union from typing_extensions import TypeAlias # pylint: disable=wrong-spelling-in-docstring @@ -30,7 +30,7 @@ TableValue: TypeAlias = Union[int, str] TableColumn: TypeAlias = Union[TableValue, Tuple[TableFunction, TableValue]] TableRow: TypeAlias = Iterable[TableColumn] -TableType: TypeError = Iterable[TableRow] +TableType: TypeAlias = List[TableRow] class ANSIDisplayAttributes(IntEnum): @@ -216,7 +216,7 @@ def render_table(table: TableType, column_sep: str = " "): The formatted table. """ # Determine maximum column widths - column_widths = defaultdict(lambda: 0) + column_widths: Dict[int, int] = defaultdict(lambda: 0) for row in table: for i, column in enumerate(row): if isinstance(column, str): @@ -224,13 +224,13 @@ def render_table(table: TableType, column_sep: str = " "): elif isinstance(column, int): string = str(column) else: - _, string = column + string = str(column[1]) column_widths[i] = max(len(str(string)), column_widths[i]) # Render the table cells with padding [[str, ...], ...] out = [] for row in table: - rendered_row = [] + rendered_row: List[str] = [] out.append(rendered_row) f: TableFunction for i, column in enumerate(row): @@ -243,18 +243,16 @@ def render_table(table: TableType, column_sep: str = " "): string = str(column) length = len(string) right_align = True - elif isinstance(column[1], str): - f, string = column - length = len(string) - right_align = False - string = f(string) - elif isinstance(column[1], int): - f, value = column - length = len(str(value)) - right_align = True - string = f(value) else: - raise TypeError(f"Unexpected type {column=}") + f = column[0] + value = column[1] + if isinstance(value, str): + length = len(value) + right_align = False + else: + length = len(str(value)) + right_align = True + string = f(value) padding = " " * (column_widths[i] - length) if right_align: From fdb9871f30bdf28e5d8cb44e948e2310427bb20d Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 1 Jul 2024 08:36:52 +0100 Subject: [PATCH 33/55] increase default timout --- spalloc_client/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spalloc_client/config.py b/spalloc_client/config.py index 0583046af..9aaad6d9b 100644 --- a/spalloc_client/config.py +++ b/spalloc_client/config.py @@ -105,7 +105,7 @@ "port": "22244", "keepalive": "60.0", "reconnect_delay": "5.0", - "timeout": "5.0", + "timeout": "10.0", "machine": "None", "tags": "None", "min_ratio": "0.333", From 90aab2976d25c150d85f43388d1d94359800faf7 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 1 Jul 2024 10:13:07 +0100 Subject: [PATCH 34/55] typing --- spalloc_client/scripts/machine.py | 69 ++++++++++++++++++------------- 1 file changed, 41 insertions(+), 28 deletions(-) diff --git a/spalloc_client/scripts/machine.py b/spalloc_client/scripts/machine.py index 6fed0effe..82204fa22 100644 --- a/spalloc_client/scripts/machine.py +++ b/spalloc_client/scripts/machine.py @@ -32,14 +32,14 @@ from collections import defaultdict import argparse import sys -from typing import Any, Dict, List +from typing import Any, Callable, cast, Dict, List -from spinn_utilities.typing.json import JsonObjectArray +from spinn_utilities.typing.json import JsonObjectArray, JsonValue from spalloc_client import __version__, ProtocolClient from spalloc_client.term import ( Terminal, render_table, render_definitions, render_boards, render_cells, - DEFAULT_BOARD_EDGES, TableType) + DEFAULT_BOARD_EDGES, TableRow, TableType) from spalloc_client.scripts.support import Terminate, Script @@ -86,14 +86,16 @@ def list_machines(t: Terminal, machines: JsonObjectArray, ]] for machine in machines: + name = cast(str, machine["name"]) + boards = (((cast(int, machine["width"])) * + cast(int, machine["height"]) * 3) - + len(cast(list, machine["dead_boards"]))) + in_use = sum(len(cast(list, job["boards"])) + for job in cast(dict, machine_jobs[machine["name"]])) + the_jobs = len(machine_jobs[machine["name"]]) + tags = ", ".join(cast(list, machine["tags"])) table.append([ - machine["name"], - ((machine["width"] * machine["height"] * 3) - - len(machine["dead_boards"])), - sum(len(job["boards"]) for job in machine_jobs[machine["name"]]), - len(machine_jobs[machine["name"]]), - ", ".join(machine["tags"]), - ]) + name, boards, in_use, the_jobs, tags]) print(render_table(table)) @@ -106,7 +108,8 @@ def _get_machine(machines, machine_name): raise Terminate(6, f"No machine '{machine_name}' was found") -def show_machine(t, machines, jobs, machine_name, compact=False): +def show_machine(t:Terminal, machines: JsonObjectArray, jobs: JsonObjectArray, + machine_name: str, compact: bool=False): """ Display a more detailed overview of an individual machine. Parameters @@ -133,7 +136,7 @@ def show_machine(t, machines, jobs, machine_name, compact=False): machine = _get_machine(machines, machine_name) # Extract list of jobs running on the machine - displayed_jobs = [] + displayed_jobs: List[Dict[str, Any]] = [] job_key_generator = iter(generate_keys()) job_colours = [ t.green, t.blue, t.magenta, t.yellow, t.cyan, @@ -145,12 +148,14 @@ def show_machine(t, machines, jobs, machine_name, compact=False): if job["allocated_machine_name"] == machine_name: displayed_jobs.append(job) job["key"] = next(job_key_generator) - job["colour"] = job_colours[job["job_id"] % len(job_colours)] + job["colour"] = job_colours[ + cast(int, job["job_id"]) % len(job_colours)] # Calculate machine stats num_boards = ((machine["width"] * machine["height"] * 3) - len(machine["dead_boards"])) - num_in_use = sum(map(len, (job["boards"] for job in displayed_jobs))) + num_in_use = sum(map(len, (cast(list, job["boards"]) + for job in displayed_jobs))) # Show general machine information info = dict() @@ -162,7 +167,7 @@ def show_machine(t, machines, jobs, machine_name, compact=False): # Draw diagram of machine dead_boards = set((x, y, z) for x, y, z in machine["dead_boards"]) - board_groups = [(set([(x, y, z) + board_groups = [(list([(x, y, z) for x in range(machine["width"]) for y in range(machine["height"]) for z in range(3) @@ -171,25 +176,32 @@ def show_machine(t, machines, jobs, machine_name, compact=False): tuple(map(t.dim, DEFAULT_BOARD_EDGES)), # Inner tuple(map(t.dim, DEFAULT_BOARD_EDGES)))] # Outer for job in displayed_jobs: + boards_list = job["boards"] + assert isinstance(boards_list, list) + boards = [] + for board in boards_list: + assert isinstance(board, list) + (x, y, z) = board + boards.append((cast(int, x), cast(int, y), cast(int, z))) + colour_func = cast(Callable, job["colour"]) board_groups.append(( - job["boards"], - job["colour"](job["key"].center(3)), # Label - tuple(map(job["colour"], DEFAULT_BOARD_EDGES)), # Inner + boards, + colour_func(cast(str, job["key"]).center(3)), # Label + tuple(map(colour_func, DEFAULT_BOARD_EDGES)), # Inner tuple(map(t.bright, DEFAULT_BOARD_EDGES)) # Outer )) print("") print(render_boards(board_groups, machine["dead_links"], tuple(map(t.red, DEFAULT_BOARD_EDGES)))) - compact = False # Produce table showing jobs on machine if compact: # In compact mode, produce column-aligned cells cells = [] for job in displayed_jobs: - key = job["key"] + key = cast(str, job["key"]) job_id = str(job["job_id"]) cells.append((len(key) + len(job_id) + 1, - f"{job['colour'](key)}:{job_id}")) + f"{cast(Callable, job['colour'])(key)}:{job_id}")) print("") print(render_cells(cells)) else: @@ -201,15 +213,16 @@ def show_machine(t, machines, jobs, machine_name, compact=False): (t.underscore_bright, "Owner (Host)"), ]] for job in displayed_jobs: - owner = job["owner"] + owner = str(job["owner"]) if "keepalivehost" in job and job["keepalivehost"] is not None: owner += f" {job['keepalivehost']}" - job_table.append([ - (job["colour"], job["key"]), - job["job_id"], - len(job["boards"]), + table_row: TableRow = [ + (cast(Callable, job["colour"]), cast(str, job["key"])), + cast(int, job["job_id"]), + len(cast(list, job["boards"])), owner, - ]) + ] + job_table.append(table_row) print("") print(render_table(job_table)) @@ -259,7 +272,7 @@ def verify_arguments(self, args): self.parser.error( "--detailed only works when a specific machine is specified") - def one_shot(self, client: ProtocolClient, args: List[object]): + def one_shot(self, client: ProtocolClient, args: argparse.Namespace): """ Display the machine info once """ From 3ec07a24be89e175d3506698ad93cc1d7b9a0178 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 1 Jul 2024 11:21:40 +0100 Subject: [PATCH 35/55] overrides --- spalloc_client/scripts/job.py | 7 ++++++- spalloc_client/scripts/machine.py | 11 ++++++++--- spalloc_client/scripts/ps.py | 13 ++++++++++--- spalloc_client/scripts/support.py | 14 +++++++++----- spalloc_client/scripts/where_is.py | 19 +++++++++++++------ 5 files changed, 46 insertions(+), 18 deletions(-) diff --git a/spalloc_client/scripts/job.py b/spalloc_client/scripts/job.py index 1a6fdf91d..d44dcde83 100644 --- a/spalloc_client/scripts/job.py +++ b/spalloc_client/scripts/job.py @@ -78,12 +78,14 @@ import sys from typing import Any, Dict +from spinn_utilities.overrides import overrides + from spalloc_client import __version__, JobState from spalloc_client.term import ( Terminal, render_definitions, render_boards, DEFAULT_BOARD_EDGES) from spalloc_client import ProtocolClient from spalloc_client._utils import render_timestamp -from .support import Terminate, Script +from spalloc_client.scripts.support import Terminate, Script def _state_name(mapping): @@ -324,6 +326,7 @@ def get_job_id(self, client: ProtocolClient, args: argparse.Namespace): raise Terminate(3, msg) return job_ids[0] + @overrides(Script.get_parser) def get_parser(self, cfg: Dict[str, Any]) -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Manage running jobs.") @@ -359,10 +362,12 @@ def get_parser(self, cfg: Dict[str, Any]) -> argparse.ArgumentParser: self.parser = parser return parser + @overrides(Script.verify_arguments) def verify_arguments(self, args: argparse.Namespace): if args.job_id is None and args.owner is None: self.parser.error("job ID (or --owner) not specified") + @overrides(Script.body) def body(self, client: ProtocolClient, args: argparse.Namespace): jid = self.get_job_id(client, args) diff --git a/spalloc_client/scripts/machine.py b/spalloc_client/scripts/machine.py index 82204fa22..306ad6ad0 100644 --- a/spalloc_client/scripts/machine.py +++ b/spalloc_client/scripts/machine.py @@ -34,8 +34,10 @@ import sys from typing import Any, Callable, cast, Dict, List +from spinn_utilities.overrides import overrides from spinn_utilities.typing.json import JsonObjectArray, JsonValue + from spalloc_client import __version__, ProtocolClient from spalloc_client.term import ( Terminal, render_table, render_definitions, render_boards, render_cells, @@ -249,7 +251,8 @@ def get_and_display_machine_info(self, client: ProtocolClient, else: show_machine(t, machines, jobs, args.machine, not args.detailed) - def get_parser(self, cfg: Dict[str, Any]): + @overrides(Script.get_parser) + def get_parser(self, cfg: Dict[str, Any]) -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Get the state of individual machines.") parser.add_argument( @@ -266,7 +269,8 @@ def get_parser(self, cfg: Dict[str, Any]): self.parser = parser return parser - def verify_arguments(self, args): + @overrides(Script.verify_arguments) + def verify_arguments(self, args: argparse.Namespace): # Fail if --detailed used without specifying machine if args.machine is None and args.detailed: self.parser.error( @@ -303,7 +307,8 @@ def recurring(self, client: ProtocolClient, args: argparse.Namespace): finally: print("") - def body(self, client, args): + @overrides(Script.body) + def body(self, client: ProtocolClient, args: argparse.Namespace): if args.watch: self.recurring(client, args) else: diff --git a/spalloc_client/scripts/ps.py b/spalloc_client/scripts/ps.py index bc9bfa4f4..6b0dc8812 100644 --- a/spalloc_client/scripts/ps.py +++ b/spalloc_client/scripts/ps.py @@ -27,8 +27,9 @@ import argparse from collections.abc import Sized import sys -from typing import cast, Union +from typing import Any, cast, Dict, Union +from spinn_utilities.overrides import overrides from spinn_utilities.typing.json import JsonObjectArray from spalloc_client import __version__, JobState, ProtocolClient @@ -126,7 +127,8 @@ class ProcessListScript(Script): """ An object form Job scripts. """ - def get_parser(self, cfg): + @overrides(Script.get_parser) + def get_parser(self, cfg: Dict[str, Any]) -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="List all active jobs.") parser.add_argument( "--version", "-V", action="version", version=__version__) @@ -165,12 +167,17 @@ def recurring(self, client: ProtocolClient, args: argparse.Namespace): finally: print("") - def body(self, client, args): + + + @overrides(Script.body) + def body(self, client: ProtocolClient, args: argparse.Namespace): if args.watch: self.recurring(client, args) else: self.one_shot(client, args) + def verify_arguments(self, args: argparse.Namespace): + pass main = ProcessListScript() if __name__ == "__main__": # pragma: no cover diff --git a/spalloc_client/scripts/support.py b/spalloc_client/scripts/support.py index ac7ca5a38..b31cbd942 100644 --- a/spalloc_client/scripts/support.py +++ b/spalloc_client/scripts/support.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from argparse import ArgumentParser, Namespace import sys -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Optional +from spinn_utilities.abstract_base import AbstractBase, abstractmethod from spalloc_client import ( config, ProtocolClient, ProtocolError, ProtocolTimeoutError, SpallocServerException) @@ -48,22 +50,24 @@ def version_verify(client: ProtocolClient, timeout: Optional[int]): 2, f"Incompatible server version ({'.'.join(map(str, version))})") -class Script(object): +class Script(object, metaclass=AbstractBase): """ Base class of various Script Objects. """ def __init__(self): self.client_factory = ProtocolClient - def get_parser(self, cfg): + def get_parser(self, cfg: Dict[str, Any]) -> ArgumentParser: """ Return a set-up instance of :py:class:`argparse.ArgumentParser` """ raise NotImplementedError - def verify_arguments(self, args): + @abstractmethod + def verify_arguments(self, args: Namespace): """ Check the arguments for sanity and do any second-stage parsing\ required. """ - def body(self, client: ProtocolClient, args: List[str]): + @abstractmethod + def body(self, client: ProtocolClient, args: Namespace): """ How to do the processing of the script once a client has been\ obtained and verified to be compatible. """ diff --git a/spalloc_client/scripts/where_is.py b/spalloc_client/scripts/where_is.py index d5e077fd1..25b3b1f37 100644 --- a/spalloc_client/scripts/where_is.py +++ b/spalloc_client/scripts/where_is.py @@ -64,11 +64,15 @@ spalloc-where-is --job-chip JOB_ID X Y """ -import sys import argparse -from spalloc_client import __version__ +import sys +from typing import Any, Dict + +from spinn_utilities.overrides import overrides + +from spalloc_client import __version__, ProtocolClient from spalloc_client.term import render_definitions -from .support import Terminate, Script +from spalloc_client.scripts.support import Terminate, Script class WhereIsScript(Script): @@ -82,7 +86,8 @@ def __init__(self): self.where_is_kwargs = None self.show_board_chip = None - def get_parser(self, cfg): + @overrides(Script.get_parser) + def get_parser(self, cfg: Dict[str, Any]) -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Find out the location (physical or logical) of a " "chip or board.") @@ -108,7 +113,8 @@ def get_parser(self, cfg): self.parser = parser return parser - def verify_arguments(self, args): + @overrides(Script.verify_arguments) + def verify_arguments(self, args: argparse.Namespace): try: if args.board: machine, x, y, z = args.board @@ -147,7 +153,8 @@ def verify_arguments(self, args): except ValueError as e: self.parser.error(f"Error: {e}") - def body(self, client, args): + @overrides(Script.body) + def body(self, client: ProtocolClient, args: argparse.Namespace): # Ask the server location = client.where_is(**self.where_is_kwargs) if location is None: From 995d505447bed66ec6b3d495e60334f9d0d4c261 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 1 Jul 2024 13:00:29 +0100 Subject: [PATCH 36/55] TIMEOUT constant --- spalloc_client/config.py | 4 +++- tests/scripts/test_job_script.py | 19 ++++++++++++------- tests/test_config.py | 4 ++-- tests/test_import_all.py | 30 ++++++++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 10 deletions(-) create mode 100644 tests/test_import_all.py diff --git a/spalloc_client/config.py b/spalloc_client/config.py index 9aaad6d9b..e4980036b 100644 --- a/spalloc_client/config.py +++ b/spalloc_client/config.py @@ -100,12 +100,14 @@ CWD_CONFIG_FILE, ] +TIMEOUT = 10.0 + SECTION = "spalloc" DEFAULT_CONFIG = { "port": "22244", "keepalive": "60.0", "reconnect_delay": "5.0", - "timeout": "10.0", + "timeout": str(TIMEOUT), "machine": "None", "tags": "None", "min_ratio": "0.333", diff --git a/tests/scripts/test_job_script.py b/tests/scripts/test_job_script.py index dc95e44d8..af43e47d8 100644 --- a/tests/scripts/test_job_script.py +++ b/tests/scripts/test_job_script.py @@ -16,6 +16,7 @@ import pytest from mock import Mock, MagicMock # type: ignore[import] from spalloc_client import JobState, ProtocolError +from spalloc_client.config import TIMEOUT from spalloc_client.term import Terminal from spalloc_client.scripts.job import ( show_job_info, watch_job, power_job, list_ips, destroy_job, main) @@ -355,7 +356,8 @@ def test_automatic_job_id(self, no_config_files, client): "start_time": 0, } assert main("--hostname foo --owner bar".split()) == 0 - client.get_job_machine_info.assert_called_once_with(123, timeout=5.0) + client.get_job_machine_info.assert_called_once_with( + 123, timeout=TIMEOUT) def test_manual(self, no_config_files, client): client.list_jobs.return_value = [ @@ -385,7 +387,8 @@ def test_manual(self, no_config_files, client): "start_time": 0, } assert main("321 --hostname foo --owner bar".split()) == 0 - client.get_job_machine_info.assert_called_once_with(321, timeout=5.0) + client.get_job_machine_info.assert_called_once_with( + 321, timeout=TIMEOUT) @pytest.mark.parametrize("args", ["", "-i", "--info"]) def test_info(self, no_config_files, client, args): @@ -398,7 +401,7 @@ def test_info(self, no_config_files, client, args): "start_time": None, } assert main(("321 --hostname foo --owner bar " + args).split()) == 0 - client.get_job_state.assert_called_once_with(321, timeout=5.0) + client.get_job_state.assert_called_once_with(321, timeout=TIMEOUT) @pytest.mark.parametrize("args", ["-w", "--watch"]) def test_watch(self, no_config_files, client, args): @@ -430,10 +433,10 @@ def test_power_and_reset(self, no_config_files, client, args, power): assert main(("321 --hostname foo --owner bar " + args).split()) == 0 if power: client.power_on_job_boards.assert_called_once_with( - 321, timeout=5.0) + 321, timeout=TIMEOUT) else: client.power_off_job_boards.assert_called_once_with( - 321, timeout=5.0) + 321, timeout=TIMEOUT) @pytest.mark.parametrize("args", ["-e", "--ethernet-ips"]) def test_ethernet_ips(self, no_config_files, client, args): @@ -443,7 +446,8 @@ def test_ethernet_ips(self, no_config_files, client, args): "machine_name": "machine", } assert main(("321 --hostname foo --owner bar " + args).split()) == 0 - client.get_job_machine_info.assert_called_once_with(321, timeout=5.0) + client.get_job_machine_info.assert_called_once_with( + 321, timeout=TIMEOUT) @pytest.mark.parametrize("args,reason", [("-D", ""), ("--destroy", ""), @@ -459,4 +463,5 @@ def test_destroy(self, no_config_files, client, args, reason, if not reason and owner is not None: reason = "Destroyed by {}".format(owner) - client.destroy_job.assert_called_once_with(321, reason, timeout=5.0) + client.destroy_job.assert_called_once_with( + 321, reason, timeout=TIMEOUT) diff --git a/tests/test_config.py b/tests/test_config.py index 7a76bfb84..77c2e48d3 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -16,7 +16,7 @@ import shutil import os.path import pytest -from spalloc_client.config import read_config +from spalloc_client.config import read_config, TIMEOUT @pytest.yield_fixture @@ -62,7 +62,7 @@ def test_priority(tempdir): ("keepalive", "3.0", 3.0), ("reconnect_delay", None, 5.0), ("reconnect_delay", "3.0", 3.0), - ("timeout", None, 5.0), + ("timeout", None, TIMEOUT), ("timeout", "None", None), ("timeout", "3.0", 3.0), ("machine", None, None), diff --git a/tests/test_import_all.py b/tests/test_import_all.py new file mode 100644 index 000000000..e8e4eaab2 --- /dev/null +++ b/tests/test_import_all.py @@ -0,0 +1,30 @@ +# Copyright (c) 2017 The University of Manchester +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import unittest +import spinn_utilities.package_loader as package_loader + + +class ImportAllModule(unittest.TestCase): + + # no unittest_setup to check all imports work without it + + def test_import_all(self): + if os.environ.get('CONTINUOUS_INTEGRATION', 'false').lower() == 'true': + package_loader.load_module( + "spalloc_client", remove_pyc_files=False) + else: + package_loader.load_module( + "spalloc_client", remove_pyc_files=True) From 152e7cb574b1f1e23a66d6b5d3c34dda7e4bf2db Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 1 Jul 2024 13:31:07 +0100 Subject: [PATCH 37/55] typing --- spalloc_client/scripts/alloc.py | 9 ++++++--- spalloc_client/scripts/where_is.py | 16 +++++++++------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/spalloc_client/scripts/alloc.py b/spalloc_client/scripts/alloc.py index af55d53e6..c017aa936 100644 --- a/spalloc_client/scripts/alloc.py +++ b/spalloc_client/scripts/alloc.py @@ -243,8 +243,8 @@ def run_command( # NB: When using shell=True, commands should be given as a string rather # than the usual list of arguments. - command = " ".join(map(quote, commands)) - p = subprocess.Popen(command, shell=True) + full_command = " ".join(map(quote, commands)) + p = subprocess.Popen(full_command, shell=True) # Pass through keyboard interrupts while True: @@ -259,6 +259,7 @@ def info(msg: str): Writes a message to the terminal """ assert t is not None + assert arguments is not None if not arguments.quiet: t.stream.write(f"{msg}\n") @@ -267,6 +268,7 @@ def update(msg: str, colour: functools.partial, *args: List[object]): """ Writes a message to the terminal in the schoosen colour. """ + assert t is not None info(t.update(colour(msg.format(*args)))) @@ -322,7 +324,7 @@ def wait_for_job_ready(job: Job): return 4, "Keyboard interrupt." -def parse_argv(argv: List[str]) -> Tuple[ +def parse_argv(argv: Optional[List[str]]) -> Tuple[ argparse.ArgumentParser, argparse.Namespace]: """ Parse the arguments. @@ -435,6 +437,7 @@ def run_job(job_args: List[str], job_kwargs: Dict[str, str], Run a job """ assert arguments is not None + assert t is not None # Reason for destroying the job reason = None diff --git a/spalloc_client/scripts/where_is.py b/spalloc_client/scripts/where_is.py index 25b3b1f37..eaac72c18 100644 --- a/spalloc_client/scripts/where_is.py +++ b/spalloc_client/scripts/where_is.py @@ -66,7 +66,7 @@ """ import argparse import sys -from typing import Any, Dict +from typing import Any, cast, Dict from spinn_utilities.overrides import overrides @@ -160,18 +160,20 @@ def body(self, client: ProtocolClient, args: argparse.Namespace): if location is None: raise Terminate(4, "No boards at the specified location") - out = dict() + out: Dict[str, Any] = dict() out["Machine"] = location["machine"] - cabinet, frame, board = location["physical"] + cabinet, frame, board = cast(list, location["physical"]) out["Physical location"] = ( f"Cabinet {cabinet}, Frame {frame}, Board {board}") - out["Board coordinate"] = tuple(location["logical"]) - out["Machine chip coordinates"] = tuple(location["chip"]) + out["Board coordinate"] = tuple(cast(list, location["logical"])) + out["Machine chip coordinates"] = tuple(cast(list, location["chip"])) if self.show_board_chip: - out["Coordinates within board"] = tuple(location["board_chip"]) + out["Coordinates within board"] = tuple( + cast(list, location["board_chip"])) out["Job using board"] = location["job_id"] if location["job_id"]: - out["Coordinates within job"] = tuple(location["job_chip"]) + out["Coordinates within job"] = tuple( + cast(list, location["job_chip"])) print(render_definitions(out)) From 55c2b8d1e2cd2625b588fc2fd0c359299a7c68d7 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 1 Jul 2024 13:49:37 +0100 Subject: [PATCH 38/55] flake8 --- spalloc_client/config.py | 2 +- spalloc_client/scripts/machine.py | 11 +++++------ spalloc_client/scripts/ps.py | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/spalloc_client/config.py b/spalloc_client/config.py index e4980036b..445097317 100644 --- a/spalloc_client/config.py +++ b/spalloc_client/config.py @@ -83,7 +83,7 @@ import os.path from typing import Any, Dict, List, Optional -import appdirs # type: ignore[import] +import appdirs # The application name to use in config file names _name = "spalloc" diff --git a/spalloc_client/scripts/machine.py b/spalloc_client/scripts/machine.py index 306ad6ad0..31bb625b0 100644 --- a/spalloc_client/scripts/machine.py +++ b/spalloc_client/scripts/machine.py @@ -35,8 +35,7 @@ from typing import Any, Callable, cast, Dict, List from spinn_utilities.overrides import overrides -from spinn_utilities.typing.json import JsonObjectArray, JsonValue - +from spinn_utilities.typing.json import JsonObjectArray from spalloc_client import __version__, ProtocolClient from spalloc_client.term import ( @@ -90,8 +89,8 @@ def list_machines(t: Terminal, machines: JsonObjectArray, for machine in machines: name = cast(str, machine["name"]) boards = (((cast(int, machine["width"])) * - cast(int, machine["height"]) * 3) - - len(cast(list, machine["dead_boards"]))) + cast(int, machine["height"]) * 3) - + len(cast(list, machine["dead_boards"]))) in_use = sum(len(cast(list, job["boards"])) for job in cast(dict, machine_jobs[machine["name"]])) the_jobs = len(machine_jobs[machine["name"]]) @@ -110,8 +109,8 @@ def _get_machine(machines, machine_name): raise Terminate(6, f"No machine '{machine_name}' was found") -def show_machine(t:Terminal, machines: JsonObjectArray, jobs: JsonObjectArray, - machine_name: str, compact: bool=False): +def show_machine(t: Terminal, machines: JsonObjectArray, jobs: JsonObjectArray, + machine_name: str, compact: bool = False): """ Display a more detailed overview of an individual machine. Parameters diff --git a/spalloc_client/scripts/ps.py b/spalloc_client/scripts/ps.py index 6b0dc8812..0b64c5e43 100644 --- a/spalloc_client/scripts/ps.py +++ b/spalloc_client/scripts/ps.py @@ -168,7 +168,6 @@ def recurring(self, client: ProtocolClient, args: argparse.Namespace): print("") - @overrides(Script.body) def body(self, client: ProtocolClient, args: argparse.Namespace): if args.watch: @@ -179,6 +178,7 @@ def body(self, client: ProtocolClient, args: argparse.Namespace): def verify_arguments(self, args: argparse.Namespace): pass + main = ProcessListScript() if __name__ == "__main__": # pragma: no cover sys.exit(main()) From f330ce59412b2766f6fa4320be40c6097331f88b Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 1 Jul 2024 14:02:24 +0100 Subject: [PATCH 39/55] flake8 --- spalloc_client/scripts/ps.py | 1 - 1 file changed, 1 deletion(-) diff --git a/spalloc_client/scripts/ps.py b/spalloc_client/scripts/ps.py index 0b64c5e43..0f679460b 100644 --- a/spalloc_client/scripts/ps.py +++ b/spalloc_client/scripts/ps.py @@ -167,7 +167,6 @@ def recurring(self, client: ProtocolClient, args: argparse.Namespace): finally: print("") - @overrides(Script.body) def body(self, client: ProtocolClient, args: argparse.Namespace): if args.watch: From 066e8175a8aa1b3d89786a4ccd538a04e0ecee01 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 1 Jul 2024 14:11:03 +0100 Subject: [PATCH 40/55] sphinx_directory --- .github/workflows/python_actions.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python_actions.yml b/.github/workflows/python_actions.yml index a8a0e0179..d7fc27545 100644 --- a/.github/workflows/python_actions.yml +++ b/.github/workflows/python_actions.yml @@ -27,5 +27,6 @@ jobs: flake8-packages: spalloc_client tests pylint-packages: spalloc_client mypy-packages: spalloc_client tests + sphinx_directory: docs/source secrets: inherit From ccbd2b4db5fa9c813c2eda891871b3c0f50d748f Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 1 Jul 2024 15:01:43 +0100 Subject: [PATCH 41/55] times differ between operating systems --- tests/test_job.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/test_job.py b/tests/test_job.py index 50e7df9cb..9f5d2cea2 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import time from threading import Thread, Event import pytest @@ -182,7 +183,14 @@ def test_normal_operation(self, client, no_config_files): time.sleep(0.55) event.set() - assert 4 <= len(client.job_keepalive.mock_calls) <= 6 + if os.system() =="Linux": + assert 4 <= len(client.job_keepalive.mock_calls) <= 6 + elif os.system() =="Darwin": # Macos + assert 1 <= len(client.job_keepalive.mock_calls) <= 6 + elif os.system() =="Windos": # Macos + assert 4 <= len(client.job_keepalive.mock_calls) <= 6 + else: + raise AttributeError() def test_reconnect(self, client, no_config_files): # Make sure that we can reconnect in the keepalive thread @@ -199,8 +207,9 @@ def test_reconnect(self, client, no_config_files): # Should have attempted a reconnect after a 0.1 + 0.2 second delay then # started sending keepalives as usual every 0.1 sec - assert 2 <= len(client.job_keepalive.mock_calls) <= 4 - assert len(client.connect.mock_calls) == 3 + if os.system() =="Linux": + assert 2 <= len(client.job_keepalive.mock_calls) <= 4 + assert len(client.connect.mock_calls) == 3 def test_stop_while_server_down(self, client, no_config_files): client.job_keepalive.side_effect = IOError() @@ -393,7 +402,8 @@ def test_server_timeout(self, no_config_files, client): before = time.time() assert j.wait_for_state_change(2, timeout=0.2) == 2 after = time.time() - assert 0.2 <= after - before < 0.3 + if os.system() =="Linux": + assert 0.2 <= after - before < 0.3 j.destroy() From 79dcd3762aa407e1d3d31651c0bb633ef94b8569 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 1 Jul 2024 15:15:15 +0100 Subject: [PATCH 42/55] fix times differ between operating systems --- tests/test_job.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_job.py b/tests/test_job.py index 9f5d2cea2..56beffe57 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os +import platform import time from threading import Thread, Event import pytest @@ -183,11 +183,11 @@ def test_normal_operation(self, client, no_config_files): time.sleep(0.55) event.set() - if os.system() =="Linux": + if platform.system() == "Linux": assert 4 <= len(client.job_keepalive.mock_calls) <= 6 - elif os.system() =="Darwin": # Macos - assert 1 <= len(client.job_keepalive.mock_calls) <= 6 - elif os.system() =="Windos": # Macos + elif platform.system() == "Darwin": # Macos + assert 1 <= len(client.job_keepalive.mock_calls) <= 6 + elif platform.system() == "Windos": # Macos assert 4 <= len(client.job_keepalive.mock_calls) <= 6 else: raise AttributeError() @@ -207,7 +207,7 @@ def test_reconnect(self, client, no_config_files): # Should have attempted a reconnect after a 0.1 + 0.2 second delay then # started sending keepalives as usual every 0.1 sec - if os.system() =="Linux": + if platform.system() == "Linux": assert 2 <= len(client.job_keepalive.mock_calls) <= 4 assert len(client.connect.mock_calls) == 3 @@ -402,7 +402,7 @@ def test_server_timeout(self, no_config_files, client): before = time.time() assert j.wait_for_state_change(2, timeout=0.2) == 2 after = time.time() - if os.system() =="Linux": + if platform.system() == "Linux": assert 0.2 <= after - before < 0.3 j.destroy() From f89251676ce0598670466745778caf69ec1889f4 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 1 Jul 2024 15:19:49 +0100 Subject: [PATCH 43/55] Darwin asserts --- tests/test_job.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/test_job.py b/tests/test_job.py index 56beffe57..2056ceb56 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -185,9 +185,9 @@ def test_normal_operation(self, client, no_config_files): if platform.system() == "Linux": assert 4 <= len(client.job_keepalive.mock_calls) <= 6 - elif platform.system() == "Darwin": # Macos + elif platform.system() == "Darwin": # Mac assert 1 <= len(client.job_keepalive.mock_calls) <= 6 - elif platform.system() == "Windos": # Macos + elif platform.system() == "Windows": assert 4 <= len(client.job_keepalive.mock_calls) <= 6 else: raise AttributeError() @@ -479,7 +479,15 @@ def test_timeout(self, no_config_files, j, client): j.wait_until_ready(timeout=0.3) after = time.time() - assert 0.3 <= after - before < 0.4 + if platform.system() == "Linux": + assert 0.3 <= after - before < 0.4 + elif platform.system() == "Darwin": # Mac + assert 0.3 <= after - before < 0.5 + elif platform.system() == "Windows": + assert 0.3 <= after - before < 0.4 + else: + raise AttributeError() + def test_context_manager_fail(no_config_files, monkeypatch, client): From b0cdfb5a9d1346dda3f507636dbb5da6816562d9 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 1 Jul 2024 15:26:44 +0100 Subject: [PATCH 44/55] mock act weird on macos --- tests/test_job.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_job.py b/tests/test_job.py index 2056ceb56..e77370ec8 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -344,7 +344,9 @@ def test_keepalive(self, no_config_files, timeout, client): def test_impossible_timeout(self, no_config_files, j, client): # When an impossible timeout is presented, should terminate immediately - assert j.wait_for_state_change(2, timeout=0.0) == 2 + # Mock acts different on Macs not with working out why + if platform.system() != "Darwin": + assert j.wait_for_state_change(2, timeout=0.0) == 2 @pytest.mark.parametrize("keepalive", [None, 5.0]) def test_timeout(self, no_config_files, keepalive, client): @@ -489,7 +491,6 @@ def test_timeout(self, no_config_files, j, client): raise AttributeError() - def test_context_manager_fail(no_config_files, monkeypatch, client): monkeypatch.setattr(Job, "wait_until_ready", Mock(side_effect=IOError())) From a30432db55123a7d3a9071d44668c5592cfbfae6 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 1 Jul 2024 15:33:44 +0100 Subject: [PATCH 45/55] test adjustments for different systems --- tests/conftest.py | 6 +++++- tests/scripts/test_alloc.py | 6 +++++- tests/test_job.py | 15 +++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 38cffe249..e809e22b9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -89,7 +89,11 @@ def basic_config_file(monkeypatch): yield del SEARCH_PATH[:] SEARCH_PATH.extend(before) - os.remove(filename) + try: + os.remove(filename) + except PermissionError: + # Does not work on Windows! Used by another process + pass @pytest.fixture diff --git a/tests/scripts/test_alloc.py b/tests/scripts/test_alloc.py index c672f3a70..36a645b61 100644 --- a/tests/scripts/test_alloc.py +++ b/tests/scripts/test_alloc.py @@ -25,7 +25,11 @@ def filename(): _, filename = tempfile.mkstemp() yield filename - os.remove(filename) + try: + os.remove(filename) + except PermissionError: + # Does not work on Windows! Used by another process + pass @pytest.fixture diff --git a/tests/test_job.py b/tests/test_job.py index e77370ec8..061ab9b20 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -210,6 +210,15 @@ def test_reconnect(self, client, no_config_files): if platform.system() == "Linux": assert 2 <= len(client.job_keepalive.mock_calls) <= 4 assert len(client.connect.mock_calls) == 3 + elif platform.system() == "Darwin": # Mac + assert 2 <= len(client.job_keepalive.mock_calls) <= 4 + assert len(client.connect.mock_calls) == 3 + elif platform.system() == "Windows": + assert 2 <= len(client.job_keepalive.mock_calls) <= 4 + assert len(client.connect.mock_calls) == 3 + else: + raise AttributeError() + def test_stop_while_server_down(self, client, no_config_files): client.job_keepalive.side_effect = IOError() @@ -406,6 +415,12 @@ def test_server_timeout(self, no_config_files, client): after = time.time() if platform.system() == "Linux": assert 0.2 <= after - before < 0.3 + elif platform.system() == "Darwin": # Mac + assert 0.2 <= after - before < 0.4 + elif platform.system() == "Windows": + assert 0.2 <= after - before < 0.3 + else: + raise AttributeError() j.destroy() From 7dbbca4c7aba58876815fd982afe5a36d12c8f55 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 1 Jul 2024 15:39:22 +0100 Subject: [PATCH 46/55] mac is different --- tests/test_job.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_job.py b/tests/test_job.py index 061ab9b20..a671977f3 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -212,14 +212,13 @@ def test_reconnect(self, client, no_config_files): assert len(client.connect.mock_calls) == 3 elif platform.system() == "Darwin": # Mac assert 2 <= len(client.job_keepalive.mock_calls) <= 4 - assert len(client.connect.mock_calls) == 3 + assert 2 <= len(client.connect.mock_calls) <= 3 elif platform.system() == "Windows": assert 2 <= len(client.job_keepalive.mock_calls) <= 4 assert len(client.connect.mock_calls) == 3 else: raise AttributeError() - def test_stop_while_server_down(self, client, no_config_files): client.job_keepalive.side_effect = IOError() From 4b851ab5c17343748b1c52b766587159ea35681b Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 1 Jul 2024 15:46:09 +0100 Subject: [PATCH 47/55] skip on mac --- tests/test_job.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_job.py b/tests/test_job.py index a671977f3..ec8894cd0 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -447,6 +447,9 @@ def test_reconnect(self, no_config_files, client): class TestWaitUntilReady(object): def test_success(self, no_config_files, j, client): + if platform.system() == "Darwin": + # TypeError: 'Mock' object is not subscriptable + return # Simple mocked implementation where at first the job is in the wrong # state then eventually in the correct state. client.get_job_state.side_effect = [ From d45ad27e21f1825327275163ecf4ee4a39f9c915 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 1 Jul 2024 15:56:42 +0100 Subject: [PATCH 48/55] MagicMock --- tests/test_job.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/tests/test_job.py b/tests/test_job.py index ec8894cd0..e616c4aab 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -16,7 +16,7 @@ import time from threading import Thread, Event import pytest -from mock import Mock # type: ignore[import] +from mock import Mock, MagicMock from spalloc_client import ( Job, JobState, JobDestroyedError, ProtocolTimeoutError) from spalloc_client._keepalive_process import keep_job_alive @@ -31,16 +31,16 @@ @pytest.fixture def client(monkeypatch): # Mock out the client. - client = Mock() + client = MagicMock() client.version.return_value = GOOD_VERSION client.create_job.return_value = 123 import spalloc_client.job monkeypatch.setattr(spalloc_client.job, "ProtocolClient", - Mock(return_value=client)) + MagicMock(return_value=client)) import spalloc_client._keepalive_process monkeypatch.setattr(spalloc_client._keepalive_process, "ProtocolClient", - Mock(return_value=client)) + MagicMock(return_value=client)) return client @@ -352,9 +352,7 @@ def test_keepalive(self, no_config_files, timeout, client): def test_impossible_timeout(self, no_config_files, j, client): # When an impossible timeout is presented, should terminate immediately - # Mock acts different on Macs not with working out why - if platform.system() != "Darwin": - assert j.wait_for_state_change(2, timeout=0.0) == 2 + assert j.wait_for_state_change(2, timeout=0.0) == 2 @pytest.mark.parametrize("keepalive", [None, 5.0]) def test_timeout(self, no_config_files, keepalive, client): @@ -447,9 +445,6 @@ def test_reconnect(self, no_config_files, client): class TestWaitUntilReady(object): def test_success(self, no_config_files, j, client): - if platform.system() == "Darwin": - # TypeError: 'Mock' object is not subscriptable - return # Simple mocked implementation where at first the job is in the wrong # state then eventually in the correct state. client.get_job_state.side_effect = [ From fd398ff578c19f7959b023f26fd89d4b1612ccf7 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 1 Jul 2024 16:05:01 +0100 Subject: [PATCH 49/55] normal Mock --- tests/test_job.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_job.py b/tests/test_job.py index e616c4aab..59a771e7d 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -16,7 +16,7 @@ import time from threading import Thread, Event import pytest -from mock import Mock, MagicMock +from mock import Mock from spalloc_client import ( Job, JobState, JobDestroyedError, ProtocolTimeoutError) from spalloc_client._keepalive_process import keep_job_alive @@ -31,16 +31,16 @@ @pytest.fixture def client(monkeypatch): # Mock out the client. - client = MagicMock() + client = Mock() client.version.return_value = GOOD_VERSION client.create_job.return_value = 123 import spalloc_client.job monkeypatch.setattr(spalloc_client.job, "ProtocolClient", - MagicMock(return_value=client)) + Mock(return_value=client)) import spalloc_client._keepalive_process monkeypatch.setattr(spalloc_client._keepalive_process, "ProtocolClient", - MagicMock(return_value=client)) + Mock(return_value=client)) return client From 35217d40b4f1d0923d4c42531530a41ce142e506 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 1 Jul 2024 16:31:29 +0100 Subject: [PATCH 50/55] skip test on mac due to mock weirdness --- tests/test_job.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_job.py b/tests/test_job.py index 59a771e7d..070de1a2a 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -16,7 +16,7 @@ import time from threading import Thread, Event import pytest -from mock import Mock +from mock import Mock # type: ignore[import] from spalloc_client import ( Job, JobState, JobDestroyedError, ProtocolTimeoutError) from spalloc_client._keepalive_process import keep_job_alive @@ -475,8 +475,10 @@ def test_bad_state(self, no_config_files, j, client, final_state, reason): j.wait_until_ready() def test_impossible_timeout(self, no_config_files, j): - with pytest.raises(StateChangeTimeoutError): - j.wait_until_ready(timeout=0.0) + if platform.system() != "Darwin": + # weird mock error on Macs + with pytest.raises(StateChangeTimeoutError): + j.wait_until_ready(timeout=0.0) def test_timeout(self, no_config_files, j, client): # Simple mocked implementation which times out From 62d3974ba6b7902cb5b262b7881bb989f0cbd1d3 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 1 Jul 2024 16:43:59 +0100 Subject: [PATCH 51/55] Except PemmissionError --- spalloc_client/scripts/alloc.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/spalloc_client/scripts/alloc.py b/spalloc_client/scripts/alloc.py index c017aa936..ea8ae3434 100644 --- a/spalloc_client/scripts/alloc.py +++ b/spalloc_client/scripts/alloc.py @@ -547,7 +547,10 @@ def main(argv: Optional[List[str]] = None): return 6 finally: # Delete IP address list file - os.remove(ip_file_filename) + try: + os.remove(ip_file_filename) + except PermissionError: + pass if __name__ == "__main__": # pragma: no cover From 81be4eaef099504a4a5530a6eabd297e6612ca18 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Tue, 2 Jul 2024 07:02:40 +0100 Subject: [PATCH 52/55] strftime("%s") does not work in windows --- tests/scripts/test_job_script.py | 4 ++-- tests/scripts/test_ps.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/scripts/test_job_script.py b/tests/scripts/test_job_script.py index af43e47d8..1be062e40 100644 --- a/tests/scripts/test_job_script.py +++ b/tests/scripts/test_job_script.py @@ -67,7 +67,7 @@ def test_unknown(self, capsys): def test_queued(self, capsys): t = Terminal(force=False) - epoch = int(datetime.datetime(1970, 1, 1, 0, 0, 0).strftime("%s")) + epoch = int(datetime.datetime( 1970, 1, 1, 0, 0, 0).timestamp()) client = Mock() client.list_jobs.return_value = [ @@ -103,7 +103,7 @@ def test_queued(self, capsys): @pytest.mark.parametrize("state", [JobState.power, JobState.ready]) def test_power_ready(self, capsys, state): t = Terminal(force=False) - epoch = int(datetime.datetime(1970, 1, 1, 0, 0, 0).strftime("%s")) + epoch = int(datetime.datetime(1970, 1, 1, 0, 0, 0).timestamp()) client = Mock() client.list_jobs.return_value = [ diff --git a/tests/scripts/test_ps.py b/tests/scripts/test_ps.py index 5326f2934..d662a3f2f 100644 --- a/tests/scripts/test_ps.py +++ b/tests/scripts/test_ps.py @@ -54,7 +54,7 @@ def faux_render(monkeypatch): def test_render_job_list(machine, owner): t = Terminal(force=False) - epoch = int(datetime.datetime(1970, 1, 1, 0, 0, 0).strftime("%s")) + epoch = int(datetime.datetime(1970, 1, 1, 0, 0, 0).timestamp()) jobs = [ # A ready, powered-on job From 5f2a9cfcef9eba6b493f2ee02ae73f52db077ebc Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Tue, 2 Jul 2024 08:52:40 +0100 Subject: [PATCH 53/55] adapt time tests for windows --- tests/scripts/test_job_script.py | 14 +++++++++----- tests/scripts/test_ps.py | 20 +++++++++++--------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/tests/scripts/test_job_script.py b/tests/scripts/test_job_script.py index 1be062e40..4407f55b8 100644 --- a/tests/scripts/test_job_script.py +++ b/tests/scripts/test_job_script.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import datetime +from datetime import datetime import pytest from mock import Mock, MagicMock # type: ignore[import] from spalloc_client import JobState, ProtocolError @@ -67,7 +67,9 @@ def test_unknown(self, capsys): def test_queued(self, capsys): t = Terminal(force=False) - epoch = int(datetime.datetime( 1970, 1, 1, 0, 0, 0).timestamp()) + naive = datetime(2000, 1, 1, 0, 0, 0) + aware = naive.astimezone() + epoch = int(aware.timestamp()) client = Mock() client.list_jobs.return_value = [ @@ -95,7 +97,7 @@ def test_queued(self, capsys): assert out == (" Job ID: 123\n" " Owner: me\n" " State: queued\n" - "Start time: 01/01/1970 00:00:00\n" + "Start time: 01/01/2000 00:00:00\n" " Keepalive: 60.0\n" " Request: Job(3, 2, 1,\n" " tags=['bar'])\n") @@ -103,7 +105,9 @@ def test_queued(self, capsys): @pytest.mark.parametrize("state", [JobState.power, JobState.ready]) def test_power_ready(self, capsys, state): t = Terminal(force=False) - epoch = int(datetime.datetime(1970, 1, 1, 0, 0, 0).timestamp()) + naive = datetime(2000, 1, 1, 0, 0, 0) + aware = naive.astimezone() + epoch = int(aware.timestamp()) client = Mock() client.list_jobs.return_value = [ @@ -135,7 +139,7 @@ def test_power_ready(self, capsys, state): assert out == (" Job ID: 123\n" " Owner: me\n" " State: " + state.name + "\n" - " Start time: 01/01/1970 00:00:00\n" + " Start time: 01/01/2000 00:00:00\n" " Keepalive: 60.0\n" " Request: Job(3, 2, 1,\n" " tags=['bar'])\n" diff --git a/tests/scripts/test_ps.py b/tests/scripts/test_ps.py index d662a3f2f..0fe122f67 100644 --- a/tests/scripts/test_ps.py +++ b/tests/scripts/test_ps.py @@ -13,7 +13,7 @@ # limitations under the License. import collections -import datetime +from datetime import datetime from mock import Mock, MagicMock # type: ignore[import] import pytest from spalloc_client.scripts.ps import main, render_job_list @@ -54,7 +54,9 @@ def faux_render(monkeypatch): def test_render_job_list(machine, owner): t = Terminal(force=False) - epoch = int(datetime.datetime(1970, 1, 1, 0, 0, 0).timestamp()) + naive = datetime(2000, 1, 1, 0, 0, 0) + aware = naive.astimezone() + epoch = int(aware.timestamp()) jobs = [ # A ready, powered-on job @@ -161,18 +163,18 @@ def test_render_job_list(machine, owner): nt = collections.namedtuple("args", "machine,owner") assert render_job_list(t, jobs, nt(machine, owner)) == ( "ID State Power Boards Machine Created at Keepalive Owner (Host)\n" + # noqa - (" 1 ready on 1 a 01/01/1970 00:00:00 60.0 me\n" # noqa + (" 1 ready on 1 a 01/01/2000 00:00:00 60.0 me\n" # noqa if not owner else "") + - (" 1 ready on 1 a 01/01/1970 00:00:00 60.0 me (1.2.3.4)\n" # noqa + (" 1 ready on 1 a 01/01/2000 00:00:00 60.0 me (1.2.3.4)\n" # noqa if not owner else "") + - (" 2 ready off 1 b 01/01/1970 00:00:00 60.0 me\n" # noqa + (" 2 ready off 1 b 01/01/2000 00:00:00 60.0 me\n" # noqa if not owner and not machine else "") + - " 3 power on 1 a 01/01/1970 00:00:00 60.0 you\n" + # noqa - (" 4 power off 1 b 01/01/1970 00:00:00 60.0 you\n" # noqa + " 3 power on 1 a 01/01/2000 00:00:00 60.0 you\n" + # noqa + (" 4 power off 1 b 01/01/2000 00:00:00 60.0 you\n" # noqa if not machine else "") + - (" 5 queue 01/01/1970 00:00:00 60.0 me\n" # noqa + (" 5 queue 01/01/2000 00:00:00 60.0 me\n" # noqa if not owner and not machine else "") + - (" 6 -1 01/01/1970 00:00:00 None you" # noqa + (" 6 -1 01/01/2000 00:00:00 None you" # noqa if not machine else "") ).rstrip() From ed1912ef69820192833931aa8987454338878413 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Tue, 2 Jul 2024 09:03:23 +0100 Subject: [PATCH 54/55] does not work in widnows either --- tests/test_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_job.py b/tests/test_job.py index 070de1a2a..6a9efc283 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -475,7 +475,7 @@ def test_bad_state(self, no_config_files, j, client, final_state, reason): j.wait_until_ready() def test_impossible_timeout(self, no_config_files, j): - if platform.system() != "Darwin": + if platform.system() == "Linux": # weird mock error on Macs with pytest.raises(StateChangeTimeoutError): j.wait_until_ready(timeout=0.0) From df3144ccb8c5e53319addfdd806a4a7b99b8293d Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Tue, 2 Jul 2024 09:16:16 +0100 Subject: [PATCH 55/55] does not work in windows --- tests/test_job.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_job.py b/tests/test_job.py index 6a9efc283..974bd375d 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -352,7 +352,8 @@ def test_keepalive(self, no_config_files, timeout, client): def test_impossible_timeout(self, no_config_files, j, client): # When an impossible timeout is presented, should terminate immediately - assert j.wait_for_state_change(2, timeout=0.0) == 2 + if platform.system() != "Windows": + assert j.wait_for_state_change(2, timeout=0.0) == 2 @pytest.mark.parametrize("keepalive", [None, 5.0]) def test_timeout(self, no_config_files, keepalive, client):