From 29283e952917a2462d42cbc8ba9c5633e9edca04 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Tue, 10 Oct 2023 13:16:08 +0200 Subject: [PATCH 01/18] Added test for cd context manager (imported from monty). --- pyproject.toml | 4 ++-- tests/test_utils.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 tests/test_utils.py diff --git a/pyproject.toml b/pyproject.toml index c4651dd..2a92ed0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,14 +91,14 @@ filterwarnings = [ "ignore:.*input structure.*:UserWarning", "ignore::DeprecationWarning", ] +addopts = "--cov=src/qtoolkit" [tool.coverage.run] -include = ["src/*"] parallel = true branch = true [tool.coverage.paths] -source = ["src/"] +source = ["src/qtoolkit"] [tool.coverage.report] skip_covered = true diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..1bfa362 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,18 @@ +import os +from pathlib import Path + +from monty.tempfile import ScratchDir + +from qtoolkit.utils import cd + + +def test_cd(): + with ScratchDir("."): + dirpath = Path("mydir") + dirpath.mkdir() + filepath = dirpath / "empty_file.txt" + filepath.touch() + + with cd(dirpath): + assert os.path.exists("empty_file.txt") + assert not os.path.exists("empty_file.txt") From f5c174d8c4752d6b5da1b65ff7bbe538d6b60fb5 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Tue, 10 Oct 2023 15:01:00 +0200 Subject: [PATCH 02/18] Added unit tests for ShellIO. --- pyproject.toml | 1 + src/qtoolkit/io/shell.py | 17 +++++++++++++++- tests/io/test_shell.py | 43 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2a92ed0..cb5334c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -107,6 +107,7 @@ exclude_lines = [ '^\s*assert False(,|$)', 'if typing.TYPE_CHECKING:', '^\s*@overload( |$)', + '# pragma: no cover', ] [tool.autoflake] diff --git a/src/qtoolkit/io/shell.py b/src/qtoolkit/io/shell.py index 8da03fc..6c758ff 100644 --- a/src/qtoolkit/io/shell.py +++ b/src/qtoolkit/io/shell.py @@ -158,6 +158,21 @@ def _get_job_cmd(self, job_id: str): return cmd def parse_job_output(self, exit_code, stdout, stderr) -> QJob | None: + """Parse the output of the ps command and return the corresponding QJob object. + + If the ps command returns multiple shell jobs, only the first corresponding + QJob is returned. + #TODO: should we check that there is only one job here ? + + Parameters + ---------- + exit_code : int + Exit code of the ps command. + stdout : str + Standard output of the ps command. + stderr : str + Standard error of the ps command. + """ out = self.parse_jobs_list_output(exit_code, stdout, stderr) if out: return out[0] @@ -242,7 +257,7 @@ def _convert_qresources(self, resources: QResources) -> dict: header of the submission script. Not implemented for ShellIO """ - raise UnsupportedResourcesError + raise UnsupportedResourcesError # pragma: no cover @property def supported_qresources_keys(self) -> list: diff --git a/tests/io/test_shell.py b/tests/io/test_shell.py index d00b434..d6ba530 100644 --- a/tests/io/test_shell.py +++ b/tests/io/test_shell.py @@ -5,7 +5,6 @@ except ModuleNotFoundError: monty = None - from qtoolkit.core.data_objects import ( CancelResult, CancelStatus, @@ -145,6 +144,16 @@ def test_get_jobs_list_cmd(self, shell_io): jobs=[QJob(job_id=125), 126, "127"], user=None ) assert get_jobs_list_cmd == "ps -o pid,user,etime,state,comm -p 125,126,127" + get_jobs_list_cmd = shell_io.get_jobs_list_cmd(jobs=None, user="johndoe") + assert get_jobs_list_cmd == "ps -o pid,user,etime,state,comm -U johndoe" + with pytest.raises( + ValueError, + match=r"Cannot query by user and job\(s\) with ps, " + r"as the user option will override the ids list", + ): + shell_io.get_jobs_list_cmd( + jobs=[QJob(job_id=125), 126, "127"], user="johndoe" + ) def test_parse_jobs_list_output(self, shell_io): joblist = shell_io.parse_jobs_list_output( @@ -184,6 +193,12 @@ def test_parse_jobs_list_output(self, shell_io): stdout=b" PID USER ELAPSED S COMMAND\n 18092 davidwa+ 04:52 S bash\n 18112 davidwa+ 01:12 K bash\n", stderr=b"", ) + joblist = shell_io.parse_jobs_list_output( + exit_code=0, + stdout=b" PID USER ELAPSED S COMMAND\n\n", + stderr=b"", + ) + assert joblist == [] def test_check_convert_qresources(self, shell_io): qr = QResources(processes=1) @@ -204,3 +219,29 @@ def test_header(self, shell_io): assert "exec > /some/file" in header assert "exec 2> /some/file" in header assert "NAME" in header + + def test_parse_job_output(self, shell_io): + job = shell_io.parse_job_output( + exit_code=0, + stdout=" PID USER ELAPSED S COMMAND\n 18092 davidwa+ 04:52 S bash\n 18112 davidwa+ 01:12 S bash\n", + stderr="", + ) + assert isinstance(job, QJob) + assert job.job_id == "18092" + assert job.name == "bash" + assert job.runtime == 292 + job = shell_io.parse_job_output( + exit_code=0, + stdout=" PID USER ELAPSED S COMMAND\n", + stderr="", + ) + assert job is None + + def test_convert_str_to_time(self, shell_io): + assert shell_io._convert_str_to_time("") is None + assert shell_io._convert_str_to_time(None) is None + assert shell_io._convert_str_to_time("2-11:21:32") == 213692 + with pytest.raises(OutputParsingError): + shell_io._convert_str_to_time("2-11:21:32:5") + with pytest.raises(OutputParsingError): + shell_io._convert_str_to_time("2-11:21:hello") From 7fb366fe8002363103819302d07f99a8d7d2b597 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Tue, 10 Oct 2023 15:21:08 +0200 Subject: [PATCH 03/18] Added unit tests for data_objects (mainly get_processes_distribution). --- src/qtoolkit/core/data_objects.py | 2 +- tests/core/test_data_objects.py | 178 ++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+), 1 deletion(-) diff --git a/src/qtoolkit/core/data_objects.py b/src/qtoolkit/core/data_objects.py index eb0bdc2..b762e8d 100644 --- a/src/qtoolkit/core/data_objects.py +++ b/src/qtoolkit/core/data_objects.py @@ -100,7 +100,7 @@ def __repr__(self): @property @abc.abstractmethod def qstate(self) -> QState: - raise NotImplementedError + raise NotImplementedError # pragma: no cover class ProcessPlacement(QTKEnum): diff --git a/tests/core/test_data_objects.py b/tests/core/test_data_objects.py index 8d628b8..d0e63bd 100644 --- a/tests/core/test_data_objects.py +++ b/tests/core/test_data_objects.py @@ -355,6 +355,184 @@ def test_get_processes_distribution(self): qr = QResources.no_constraints(processes=14) proc_distr = qr.get_processes_distribution() assert proc_distr == [None, 14, None] + qr = QResources( + process_placement=ProcessPlacement.SCATTERED, nodes=None, processes=4 + ) + proc_distr = qr.get_processes_distribution() + assert proc_distr == [4, 4, 1] + qr = QResources( + process_placement=ProcessPlacement.SCATTERED, nodes=3, processes=None + ) + proc_distr = qr.get_processes_distribution() + assert proc_distr == [3, 3, 1] + with pytest.raises( + UnsupportedResourcesError, + match=r"ProcessPlacement.SCATTERED is incompatible " + r"with different values of nodes and processes", + ): + qr = QResources( + process_placement=ProcessPlacement.SCATTERED, nodes=3, processes=4 + ) + qr.get_processes_distribution() + qr = QResources( + process_placement=ProcessPlacement.SCATTERED, nodes=None, processes=None + ) + proc_distr = qr.get_processes_distribution() + assert proc_distr == [1, 1, 1] + with pytest.raises( + UnsupportedResourcesError, + match=r"ProcessPlacement.SCATTERED is incompatible " + r"with 2 processes_per_node", + ): + qr = QResources( + process_placement=ProcessPlacement.SCATTERED, + nodes=4, + processes=4, + processes_per_node=2, + ) + qr.get_processes_distribution() + with pytest.raises( + UnsupportedResourcesError, + match=r"ProcessPlacement.SAME_NODE is incompatible " r"with 4 nodes", + ): + qr = QResources(process_placement=ProcessPlacement.SAME_NODE, nodes=4) + qr.get_processes_distribution() + qr = QResources( + process_placement=ProcessPlacement.SAME_NODE, + nodes=None, + processes=None, + processes_per_node=4, + ) + proc_distr = qr.get_processes_distribution() + assert proc_distr == [1, 4, 4] + qr = QResources( + process_placement=ProcessPlacement.SAME_NODE, + nodes=1, + processes=6, + processes_per_node=None, + ) + proc_distr = qr.get_processes_distribution() + assert proc_distr == [1, 6, 6] + with pytest.raises( + UnsupportedResourcesError, + match=r"ProcessPlacement.SAME_NODE is incompatible with " + r"different values of nodes and processes", + ): + qr = QResources( + process_placement=ProcessPlacement.SAME_NODE, + nodes=1, + processes=2, + processes_per_node=6, + ) + qr.get_processes_distribution() + qr = QResources( + process_placement=ProcessPlacement.SAME_NODE, + nodes=1, + processes=None, + processes_per_node=None, + ) + proc_distr = qr.get_processes_distribution() + assert proc_distr == [1, 1, 1] + qr = QResources( + process_placement=ProcessPlacement.SAME_NODE, + nodes=None, + processes=None, + processes_per_node=None, + ) + proc_distr = qr.get_processes_distribution() + assert proc_distr == [1, 1, 1] + qr = QResources( + process_placement=ProcessPlacement.SAME_NODE, + nodes=None, + processes=3, + processes_per_node=3, + ) + proc_distr = qr.get_processes_distribution() + assert proc_distr == [1, 3, 3] + qr = QResources( + process_placement=ProcessPlacement.EVENLY_DISTRIBUTED, + nodes=None, + processes=None, + processes_per_node=3, + ) + proc_distr = qr.get_processes_distribution() + assert proc_distr == [1, None, 3] + qr = QResources( + process_placement=ProcessPlacement.EVENLY_DISTRIBUTED, + nodes=4, + processes=None, + processes_per_node=3, + ) + proc_distr = qr.get_processes_distribution() + assert proc_distr == [4, None, 3] + qr = QResources( + process_placement=ProcessPlacement.EVENLY_DISTRIBUTED, + nodes=4, + processes=None, + processes_per_node=None, + ) + proc_distr = qr.get_processes_distribution() + assert proc_distr == [4, None, 1] + with pytest.raises( + UnsupportedResourcesError, + match=r"ProcessPlacement.EVENLY_DISTRIBUTED " + r"is incompatible with processes attribute", + ): + qr = QResources( + process_placement=ProcessPlacement.EVENLY_DISTRIBUTED, + nodes=1, + processes=2, + processes_per_node=6, + ) + qr.get_processes_distribution() + with pytest.raises( + UnsupportedResourcesError, + match=r"ProcessPlacement.NO_CONSTRAINTS is incompatible " + r"with processes_per_node and nodes attribute", + ): + qr = QResources( + process_placement=ProcessPlacement.NO_CONSTRAINTS, + nodes=1, + processes=2, + processes_per_node=None, + ) + qr.get_processes_distribution() + with pytest.raises( + UnsupportedResourcesError, + match=r"ProcessPlacement.NO_CONSTRAINTS is incompatible " + r"with processes_per_node and nodes attribute", + ): + qr = QResources( + process_placement=ProcessPlacement.NO_CONSTRAINTS, + nodes=None, + processes=2, + processes_per_node=2, + ) + qr.get_processes_distribution() + qr = QResources( + process_placement=ProcessPlacement.NO_CONSTRAINTS, + nodes=None, + processes=None, + processes_per_node=None, + ) + proc_distr = qr.get_processes_distribution() + assert proc_distr == [None, 1, None] + qr = QResources( + process_placement=ProcessPlacement.NO_CONSTRAINTS, + nodes=None, + processes=8, + processes_per_node=None, + ) + proc_distr = qr.get_processes_distribution() + assert proc_distr == [None, 8, None] + qr = QResources( + process_placement="No placement", + nodes="a", + processes="b", + processes_per_node="c", + ) + proc_distr = qr.get_processes_distribution() + assert proc_distr == ["a", "b", "c"] class TestQJobInfo: From 887056e9c391120c55212d9243f91172f2a3d4fc Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Wed, 11 Oct 2023 08:51:46 +0200 Subject: [PATCH 04/18] Refactor existing unit tests of slurm. --- tests/io/test_base.py | 7 --- tests/io/test_slurm.py | 55 +++++++++++++++---- ...ut.yaml => parse_submit_output_inout.yaml} | 0 3 files changed, 45 insertions(+), 17 deletions(-) rename tests/test_data/io/slurm/{parse_submit_cmd_inout.yaml => parse_submit_output_inout.yaml} (100%) diff --git a/tests/io/test_base.py b/tests/io/test_base.py index e549ef8..41b295b 100644 --- a/tests/io/test_base.py +++ b/tests/io/test_base.py @@ -1,17 +1,10 @@ from __future__ import annotations -from pathlib import Path - import pytest -from monty.serialization import loadfn from qtoolkit.core.data_objects import CancelResult, QJob, QResources, SubmissionResult from qtoolkit.io.base import BaseSchedulerIO, QTemplate -TEST_DIR = Path(__file__).resolve().parents[1] / "test_data" -ref_file = TEST_DIR / "io" / "slurm" / "parse_submit_cmd_inout.yaml" -in_out_ref_list = loadfn(ref_file) - def test_qtemplate(): template_str = """This is a template diff --git a/tests/io/test_slurm.py b/tests/io/test_slurm.py index 3456629..d20bb85 100644 --- a/tests/io/test_slurm.py +++ b/tests/io/test_slurm.py @@ -3,18 +3,53 @@ import pytest from monty.serialization import loadfn -from qtoolkit.io.slurm import SlurmIO +from qtoolkit.core.data_objects import QState +from qtoolkit.io.slurm import SlurmIO, SlurmState TEST_DIR = Path(__file__).resolve().parents[1] / "test_data" -ref_file = TEST_DIR / "io" / "slurm" / "parse_submit_cmd_inout.yaml" +ref_file = TEST_DIR / "io" / "slurm" / "parse_submit_output_inout.yaml" in_out_ref_list = loadfn(ref_file) -@pytest.mark.parametrize("in_out_ref", in_out_ref_list) -def test_parse_submit_cmd_output(in_out_ref, test_utils): - parse_cmd_output, sr_ref = test_utils.inkwargs_outref( - in_out_ref, inkey="parse_submit_kwargs", outkey="submission_result_ref" - ) - slurm_io = SlurmIO() - sr = slurm_io.parse_submit_output(**parse_cmd_output) - assert sr == sr_ref +@pytest.fixture(scope="module") +def slurm_io(): + return SlurmIO() + + +class TestSlurmState: + @pytest.mark.parametrize("slurm_state", [s for s in SlurmState]) + def test_qstate(self, slurm_state): + assert isinstance(slurm_state.qstate, QState) + assert SlurmState("CA") == SlurmState.CANCELLED + assert SlurmState("CG") == SlurmState.COMPLETING + assert SlurmState("CD") == SlurmState.COMPLETED + assert SlurmState("CF") == SlurmState.CONFIGURING + assert SlurmState("DL") == SlurmState.DEADLINE + assert SlurmState("F") == SlurmState.FAILED + assert SlurmState("OOM") == SlurmState.OUT_OF_MEMORY + assert SlurmState("PD") == SlurmState.PENDING + assert SlurmState("R") == SlurmState.RUNNING + assert SlurmState("S") == SlurmState.SUSPENDED + assert SlurmState("TO") == SlurmState.TIMEOUT + + +class TestSlurmIO: + @pytest.mark.parametrize("in_out_ref", in_out_ref_list) + def test_parse_submit_output(self, slurm_io, in_out_ref, test_utils): + parse_cmd_output, sr_ref = test_utils.inkwargs_outref( + in_out_ref, inkey="parse_submit_kwargs", outkey="submission_result_ref" + ) + sr = slurm_io.parse_submit_output(**parse_cmd_output) + assert sr == sr_ref + sr = slurm_io.parse_submit_output( + exit_code=parse_cmd_output["exit_code"], + stdout=bytes(parse_cmd_output["stdout"], "utf-8"), + stderr=bytes(parse_cmd_output["stderr"], "utf-8"), + ) + assert sr == sr_ref + sr = slurm_io.parse_submit_output( + exit_code=parse_cmd_output["exit_code"], + stdout=bytes(parse_cmd_output["stdout"], "ascii"), + stderr=bytes(parse_cmd_output["stderr"], "ascii"), + ) + assert sr == sr_ref diff --git a/tests/test_data/io/slurm/parse_submit_cmd_inout.yaml b/tests/test_data/io/slurm/parse_submit_output_inout.yaml similarity index 100% rename from tests/test_data/io/slurm/parse_submit_cmd_inout.yaml rename to tests/test_data/io/slurm/parse_submit_output_inout.yaml From 52d3652de13a0df868ea6c9f72c2131b327524d7 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Wed, 11 Oct 2023 09:28:47 +0200 Subject: [PATCH 05/18] Added scripts to create yaml reference files. --- .../slurm/create_parse_cancel_output_inout.py | 30 +++++ .../slurm/create_parse_submit_output_inout.py | 126 ++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 tests/test_data/io/slurm/create_parse_cancel_output_inout.py create mode 100644 tests/test_data/io/slurm/create_parse_submit_output_inout.py diff --git a/tests/test_data/io/slurm/create_parse_cancel_output_inout.py b/tests/test_data/io/slurm/create_parse_cancel_output_inout.py new file mode 100644 index 0000000..8cb67ee --- /dev/null +++ b/tests/test_data/io/slurm/create_parse_cancel_output_inout.py @@ -0,0 +1,30 @@ +import json + +import yaml + +from qtoolkit.io.slurm import SlurmIO + +slurm_io = SlurmIO() + +mylist = [] + +return_code = 1 +stdout = b"" +stderr = ( + b"sbatch: error: invalid partition specified: abcd\n" + b"sbatch: error: Batch job submission failed: Invalid partition name specified\n" +) + +cr = slurm_io.parse_cancel_output(exit_code=return_code, stdout=stdout, stderr=stderr) + +a = { + "parse_cancel_kwargs": json.dumps( + {"exit_code": return_code, "stdout": stdout.decode(), "stderr": stderr.decode()} + ), + "cancel_result_ref": json.dumps(cr.as_dict()), +} +mylist.append(a) + + +with open("parse_cancel_output_inout.yaml", "w") as f: + yaml.dump(mylist, f) diff --git a/tests/test_data/io/slurm/create_parse_submit_output_inout.py b/tests/test_data/io/slurm/create_parse_submit_output_inout.py new file mode 100644 index 0000000..cb6796c --- /dev/null +++ b/tests/test_data/io/slurm/create_parse_submit_output_inout.py @@ -0,0 +1,126 @@ +import json + +import yaml + +from qtoolkit.io.slurm import SlurmIO + +slurm_io = SlurmIO() + +mylist = [] + +return_code = 1 +stdout = b"" +stderr = ( + b"sbatch: error: invalid partition specified: abcd\n" + b"sbatch: error: Batch job submission failed: Invalid partition name specified\n" +) + +sr = slurm_io.parse_submit_output(exit_code=return_code, stdout=stdout, stderr=stderr) + +a = { + "parse_submit_kwargs": json.dumps( + {"exit_code": return_code, "stdout": stdout.decode(), "stderr": stderr.decode()} + ), + "submission_result_ref": json.dumps(sr.as_dict()), +} +mylist.append(a) + +return_code = 0 +stdout = b"Submitted batch job 24\n" +stderr = b"" +sr = slurm_io.parse_submit_output(exit_code=return_code, stdout=stdout, stderr=stderr) +a = { + "parse_submit_kwargs": json.dumps( + {"exit_code": return_code, "stdout": stdout.decode(), "stderr": stderr.decode()} + ), + "submission_result_ref": json.dumps(sr.as_dict()), +} +mylist.append(a) + +return_code = 0 +stdout = b"submitted batch job 15\n" +stderr = b"" +sr = slurm_io.parse_submit_output(exit_code=return_code, stdout=stdout, stderr=stderr) +a = { + "parse_submit_kwargs": json.dumps( + {"exit_code": return_code, "stdout": stdout.decode(), "stderr": stderr.decode()} + ), + "submission_result_ref": json.dumps(sr.as_dict()), +} +mylist.append(a) + +return_code = 0 +stdout = b"Granted job allocation 10\n" +stderr = b"" +sr = slurm_io.parse_submit_output(exit_code=return_code, stdout=stdout, stderr=stderr) +a = { + "parse_submit_kwargs": json.dumps( + {"exit_code": return_code, "stdout": stdout.decode(), "stderr": stderr.decode()} + ), + "submission_result_ref": json.dumps(sr.as_dict()), +} +mylist.append(a) + +return_code = 0 +stdout = b"granted job allocation 124\n" +stderr = b"" +sr = slurm_io.parse_submit_output(exit_code=return_code, stdout=stdout, stderr=stderr) +a = { + "parse_submit_kwargs": json.dumps( + {"exit_code": return_code, "stdout": stdout.decode(), "stderr": stderr.decode()} + ), + "submission_result_ref": json.dumps(sr.as_dict()), +} +mylist.append(a) + +return_code = 0 +stdout = b"sbatch: Submitted batch job 24\n" +stderr = b"" +sr = slurm_io.parse_submit_output(exit_code=return_code, stdout=stdout, stderr=stderr) +a = { + "parse_submit_kwargs": json.dumps( + {"exit_code": return_code, "stdout": stdout.decode(), "stderr": stderr.decode()} + ), + "submission_result_ref": json.dumps(sr.as_dict()), +} +mylist.append(a) + +return_code = 0 +stdout = b"sbatch: submitted batch job 15\n" +stderr = b"" +sr = slurm_io.parse_submit_output(exit_code=return_code, stdout=stdout, stderr=stderr) +a = { + "parse_submit_kwargs": json.dumps( + {"exit_code": return_code, "stdout": stdout.decode(), "stderr": stderr.decode()} + ), + "submission_result_ref": json.dumps(sr.as_dict()), +} +mylist.append(a) + +return_code = 0 +stdout = b"salloc: Granted job allocation 10\n" +stderr = b"" +sr = slurm_io.parse_submit_output(exit_code=return_code, stdout=stdout, stderr=stderr) +a = { + "parse_submit_kwargs": json.dumps( + {"exit_code": return_code, "stdout": stdout.decode(), "stderr": stderr.decode()} + ), + "submission_result_ref": json.dumps(sr.as_dict()), +} +mylist.append(a) + +return_code = 0 +stdout = b"salloc: granted job allocation 124\n" +stderr = b"" +sr = slurm_io.parse_submit_output(exit_code=return_code, stdout=stdout, stderr=stderr) +a = { + "parse_submit_kwargs": json.dumps( + {"exit_code": return_code, "stdout": stdout.decode(), "stderr": stderr.decode()} + ), + "submission_result_ref": json.dumps(sr.as_dict()), +} +mylist.append(a) + + +with open("parse_submit_output_inout.yaml", "w") as f: + yaml.dump(mylist, f) From dd8e211a54a565465d1ff05e20088d8114770a79 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Wed, 11 Oct 2023 10:58:14 +0200 Subject: [PATCH 06/18] Added more unit tests for SlurmIO. --- src/qtoolkit/io/slurm.py | 2 + tests/io/test_slurm.py | 28 ++++++- .../slurm/create_parse_cancel_output_inout.py | 82 +++++++++++++++++-- .../slurm/create_parse_submit_output_inout.py | 2 +- .../io/slurm/parse_cancel_output_inout.yaml | 42 ++++++++++ 5 files changed, 147 insertions(+), 9 deletions(-) create mode 100644 tests/test_data/io/slurm/parse_cancel_output_inout.yaml diff --git a/src/qtoolkit/io/slurm.py b/src/qtoolkit/io/slurm.py index a21e4d1..d04f168 100644 --- a/src/qtoolkit/io/slurm.py +++ b/src/qtoolkit/io/slurm.py @@ -237,6 +237,8 @@ def parse_cancel_output(self, exit_code, stdout, stderr) -> CancelResult: status = ( CancelStatus("SUCCESSFUL") if job_id else CancelStatus("JOB_ID_UNKNOWN") ) + # TODO: when cancelling a job already completed or cancelled, exit_code is 0 + # should we set the CancelStatus to FAILED ? Same if the job does not exist. return CancelResult( job_id=job_id, exit_code=exit_code, diff --git a/tests/io/test_slurm.py b/tests/io/test_slurm.py index d20bb85..730b847 100644 --- a/tests/io/test_slurm.py +++ b/tests/io/test_slurm.py @@ -7,8 +7,10 @@ from qtoolkit.io.slurm import SlurmIO, SlurmState TEST_DIR = Path(__file__).resolve().parents[1] / "test_data" -ref_file = TEST_DIR / "io" / "slurm" / "parse_submit_output_inout.yaml" -in_out_ref_list = loadfn(ref_file) +submit_ref_file = TEST_DIR / "io" / "slurm" / "parse_submit_output_inout.yaml" +in_out_submit_ref_list = loadfn(submit_ref_file) +cancel_ref_file = TEST_DIR / "io" / "slurm" / "parse_cancel_output_inout.yaml" +in_out_cancel_ref_list = loadfn(cancel_ref_file) @pytest.fixture(scope="module") @@ -34,7 +36,7 @@ def test_qstate(self, slurm_state): class TestSlurmIO: - @pytest.mark.parametrize("in_out_ref", in_out_ref_list) + @pytest.mark.parametrize("in_out_ref", in_out_submit_ref_list) def test_parse_submit_output(self, slurm_io, in_out_ref, test_utils): parse_cmd_output, sr_ref = test_utils.inkwargs_outref( in_out_ref, inkey="parse_submit_kwargs", outkey="submission_result_ref" @@ -53,3 +55,23 @@ def test_parse_submit_output(self, slurm_io, in_out_ref, test_utils): stderr=bytes(parse_cmd_output["stderr"], "ascii"), ) assert sr == sr_ref + + @pytest.mark.parametrize("in_out_ref", in_out_cancel_ref_list) + def test_parse_cancel_output(self, slurm_io, in_out_ref, test_utils): + parse_cmd_output, cr_ref = test_utils.inkwargs_outref( + in_out_ref, inkey="parse_cancel_kwargs", outkey="cancel_result_ref" + ) + cr = slurm_io.parse_cancel_output(**parse_cmd_output) + assert cr == cr_ref + cr = slurm_io.parse_cancel_output( + exit_code=parse_cmd_output["exit_code"], + stdout=bytes(parse_cmd_output["stdout"], "utf-8"), + stderr=bytes(parse_cmd_output["stderr"], "utf-8"), + ) + assert cr == cr_ref + cr = slurm_io.parse_cancel_output( + exit_code=parse_cmd_output["exit_code"], + stdout=bytes(parse_cmd_output["stdout"], "ascii"), + stderr=bytes(parse_cmd_output["stderr"], "ascii"), + ) + assert cr == cr_ref diff --git a/tests/test_data/io/slurm/create_parse_cancel_output_inout.py b/tests/test_data/io/slurm/create_parse_cancel_output_inout.py index 8cb67ee..b66ecef 100644 --- a/tests/test_data/io/slurm/create_parse_cancel_output_inout.py +++ b/tests/test_data/io/slurm/create_parse_cancel_output_inout.py @@ -8,12 +8,84 @@ mylist = [] +return_code = 0 +stdout = b"" +stderr = b"scancel: Terminating job 267\n" + +cr = slurm_io.parse_cancel_output(exit_code=return_code, stdout=stdout, stderr=stderr) + +a = { + "parse_cancel_kwargs": json.dumps( + {"exit_code": return_code, "stdout": stdout.decode(), "stderr": stderr.decode()} + ), + "cancel_result_ref": json.dumps(cr.as_dict()), +} +mylist.append(a) + + +return_code = 1 +stdout = b"" +stderr = b"scancel: error: No job identification provided\n" + +cr = slurm_io.parse_cancel_output(exit_code=return_code, stdout=stdout, stderr=stderr) + +a = { + "parse_cancel_kwargs": json.dumps( + {"exit_code": return_code, "stdout": stdout.decode(), "stderr": stderr.decode()} + ), + "cancel_result_ref": json.dumps(cr.as_dict()), +} +mylist.append(a) + + +return_code = 210 +stdout = b"" +stderr = b"scancel: error: Kill job error on job id 1: Access/permission denied\n" + +cr = slurm_io.parse_cancel_output(exit_code=return_code, stdout=stdout, stderr=stderr) + +a = { + "parse_cancel_kwargs": json.dumps( + {"exit_code": return_code, "stdout": stdout.decode(), "stderr": stderr.decode()} + ), + "cancel_result_ref": json.dumps(cr.as_dict()), +} +mylist.append(a) + + return_code = 1 stdout = b"" -stderr = ( - b"sbatch: error: invalid partition specified: abcd\n" - b"sbatch: error: Batch job submission failed: Invalid partition name specified\n" -) +stderr = b"scancel: error: Invalid job id a\n" + +cr = slurm_io.parse_cancel_output(exit_code=return_code, stdout=stdout, stderr=stderr) + +a = { + "parse_cancel_kwargs": json.dumps( + {"exit_code": return_code, "stdout": stdout.decode(), "stderr": stderr.decode()} + ), + "cancel_result_ref": json.dumps(cr.as_dict()), +} +mylist.append(a) + + +return_code = 0 +stdout = b"" +stderr = b"scancel: Terminating job 269\nscancel: error: Kill job error on job id 269: Job/step already completing or completed\n" + +cr = slurm_io.parse_cancel_output(exit_code=return_code, stdout=stdout, stderr=stderr) + +a = { + "parse_cancel_kwargs": json.dumps( + {"exit_code": return_code, "stdout": stdout.decode(), "stderr": stderr.decode()} + ), + "cancel_result_ref": json.dumps(cr.as_dict()), +} +mylist.append(a) + + +return_code = 0 +stdout = b"" +stderr = b"scancel: Terminating job 2675\nscancel: error: Kill job error on job id 2675: Invalid job id specified\n" cr = slurm_io.parse_cancel_output(exit_code=return_code, stdout=stdout, stderr=stderr) @@ -27,4 +99,4 @@ with open("parse_cancel_output_inout.yaml", "w") as f: - yaml.dump(mylist, f) + yaml.dump(mylist, f, sort_keys=False) diff --git a/tests/test_data/io/slurm/create_parse_submit_output_inout.py b/tests/test_data/io/slurm/create_parse_submit_output_inout.py index cb6796c..feb24f8 100644 --- a/tests/test_data/io/slurm/create_parse_submit_output_inout.py +++ b/tests/test_data/io/slurm/create_parse_submit_output_inout.py @@ -123,4 +123,4 @@ with open("parse_submit_output_inout.yaml", "w") as f: - yaml.dump(mylist, f) + yaml.dump(mylist, f, sort_keys=False) diff --git a/tests/test_data/io/slurm/parse_cancel_output_inout.yaml b/tests/test_data/io/slurm/parse_cancel_output_inout.yaml new file mode 100644 index 0000000..1f6c0b2 --- /dev/null +++ b/tests/test_data/io/slurm/parse_cancel_output_inout.yaml @@ -0,0 +1,42 @@ +- parse_cancel_kwargs: '{"exit_code": 0, "stdout": "", "stderr": "scancel: Terminating + job 267\n"}' + cancel_result_ref: '{"@module": "qtoolkit.core.data_objects", "@class": "CancelResult", + "@version": "0.1.1", "job_id": "267", "step_id": null, "exit_code": 0, "stdout": + "", "stderr": "scancel: Terminating job 267\n", "status": {"@module": "qtoolkit.core.data_objects", + "@class": "CancelStatus", "@version": "0.1.1", "value": "SUCCESSFUL"}}' +- parse_cancel_kwargs: '{"exit_code": 1, "stdout": "", "stderr": "scancel: error: + No job identification provided\n"}' + cancel_result_ref: '{"@module": "qtoolkit.core.data_objects", "@class": "CancelResult", + "@version": "0.1.1", "job_id": null, "step_id": null, "exit_code": 1, "stdout": + "", "stderr": "scancel: error: No job identification provided\n", "status": {"@module": + "qtoolkit.core.data_objects", "@class": "CancelStatus", "@version": "0.1.1", "value": + "FAILED"}}' +- parse_cancel_kwargs: '{"exit_code": 210, "stdout": "", "stderr": "scancel: error: + Kill job error on job id 1: Access/permission denied\n"}' + cancel_result_ref: '{"@module": "qtoolkit.core.data_objects", "@class": "CancelResult", + "@version": "0.1.1", "job_id": null, "step_id": null, "exit_code": 210, "stdout": + "", "stderr": "scancel: error: Kill job error on job id 1: Access/permission denied\n", + "status": {"@module": "qtoolkit.core.data_objects", "@class": "CancelStatus", + "@version": "0.1.1", "value": "FAILED"}}' +- parse_cancel_kwargs: '{"exit_code": 1, "stdout": "", "stderr": "scancel: error: + Invalid job id a\n"}' + cancel_result_ref: '{"@module": "qtoolkit.core.data_objects", "@class": "CancelResult", + "@version": "0.1.1", "job_id": null, "step_id": null, "exit_code": 1, "stdout": + "", "stderr": "scancel: error: Invalid job id a\n", "status": {"@module": "qtoolkit.core.data_objects", + "@class": "CancelStatus", "@version": "0.1.1", "value": "FAILED"}}' +- parse_cancel_kwargs: '{"exit_code": 0, "stdout": "", "stderr": "scancel: Terminating + job 269\nscancel: error: Kill job error on job id 269: Job/step already completing + or completed\n"}' + cancel_result_ref: '{"@module": "qtoolkit.core.data_objects", "@class": "CancelResult", + "@version": "0.1.1", "job_id": "269", "step_id": null, "exit_code": 0, "stdout": + "", "stderr": "scancel: Terminating job 269\nscancel: error: Kill job error on + job id 269: Job/step already completing or completed\n", "status": {"@module": + "qtoolkit.core.data_objects", "@class": "CancelStatus", "@version": "0.1.1", "value": + "SUCCESSFUL"}}' +- parse_cancel_kwargs: '{"exit_code": 0, "stdout": "", "stderr": "scancel: Terminating + job 2675\nscancel: error: Kill job error on job id 2675: Invalid job id specified\n"}' + cancel_result_ref: '{"@module": "qtoolkit.core.data_objects", "@class": "CancelResult", + "@version": "0.1.1", "job_id": "2675", "step_id": null, "exit_code": 0, "stdout": + "", "stderr": "scancel: Terminating job 2675\nscancel: error: Kill job error on + job id 2675: Invalid job id specified\n", "status": {"@module": "qtoolkit.core.data_objects", + "@class": "CancelStatus", "@version": "0.1.1", "value": "SUCCESSFUL"}}' From 59e557c03c03d291d37cdb065f457da642f79545 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Wed, 11 Oct 2023 12:04:08 +0200 Subject: [PATCH 07/18] Small fix of parsing of slurm outputs. --- src/qtoolkit/io/slurm.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/qtoolkit/io/slurm.py b/src/qtoolkit/io/slurm.py index d04f168..46137d5 100644 --- a/src/qtoolkit/io/slurm.py +++ b/src/qtoolkit/io/slurm.py @@ -261,7 +261,7 @@ def _get_job_cmd(self, job_id: str): if self.get_job_executable == "scontrol": # -o is to get the output as a one-liner cmd = f"SLURM_TIME_FORMAT='standard' scontrol show job -o {job_id}" - elif self.get_job_executable == "sacct": + elif self.get_job_executable == "sacct": # pragma: no cover raise NotImplementedError("sacct for get_job not yet implemented.") else: raise RuntimeError( @@ -296,27 +296,27 @@ def parse_job_output(self, exit_code, stdout, stderr) -> QJob | None: try: memory_per_cpu = self._convert_memory_str(parsed_output["MinMemoryCPU"]) - except OutputParsingError: + except (OutputParsingError, KeyError): memory_per_cpu = None try: nodes = int(parsed_output["NumNodes"]) - except ValueError: + except (ValueError, KeyError): nodes = None try: cpus = int(parsed_output["NumCPUs"]) - except ValueError: + except (ValueError, KeyError): cpus = None try: cpus_task = int(parsed_output["CPUs/Task"]) - except ValueError: + except (ValueError, KeyError): cpus_task = None try: time_limit = self._convert_str_to_time(parsed_output["TimeLimit"]) - except OutputParsingError: + except (OutputParsingError, KeyError): time_limit = None info = QJobInfo( From 961b39a4202f7718f742e16cfe0f4dab226e74e5 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Wed, 11 Oct 2023 14:18:03 +0200 Subject: [PATCH 08/18] Added more unit tests for slurm. --- src/qtoolkit/io/slurm.py | 4 +- tests/io/test_slurm.py | 44 +++++++++++++++++++ .../io/slurm/create_parse_job_output_inout.py | 25 +++++++++++ .../io/slurm/parse_job_output_inout.yaml | 23 ++++++++++ 4 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 tests/test_data/io/slurm/create_parse_job_output_inout.py create mode 100644 tests/test_data/io/slurm/parse_job_output_inout.yaml diff --git a/src/qtoolkit/io/slurm.py b/src/qtoolkit/io/slurm.py index 46137d5..dd46993 100644 --- a/src/qtoolkit/io/slurm.py +++ b/src/qtoolkit/io/slurm.py @@ -263,7 +263,7 @@ def _get_job_cmd(self, job_id: str): cmd = f"SLURM_TIME_FORMAT='standard' scontrol show job -o {job_id}" elif self.get_job_executable == "sacct": # pragma: no cover raise NotImplementedError("sacct for get_job not yet implemented.") - else: + else: # pragma: no cover raise RuntimeError( f'"{self.get_job_executable}" is not a valid get_job_executable.' ) @@ -281,7 +281,7 @@ def parse_job_output(self, exit_code, stdout, stderr) -> QJob | None: if self.get_job_executable == "scontrol": parsed_output = self._parse_scontrol_cmd_output(stdout=stdout) - elif self.get_job_executable == "sacct": + elif self.get_job_executable == "sacct": # pragma: no cover raise NotImplementedError("sacct for get_job not yet implemented.") else: raise RuntimeError( diff --git a/tests/io/test_slurm.py b/tests/io/test_slurm.py index 730b847..535c9b6 100644 --- a/tests/io/test_slurm.py +++ b/tests/io/test_slurm.py @@ -11,6 +11,8 @@ in_out_submit_ref_list = loadfn(submit_ref_file) cancel_ref_file = TEST_DIR / "io" / "slurm" / "parse_cancel_output_inout.yaml" in_out_cancel_ref_list = loadfn(cancel_ref_file) +job_ref_file = TEST_DIR / "io" / "slurm" / "parse_job_output_inout.yaml" +in_out_job_ref_list = loadfn(job_ref_file) @pytest.fixture(scope="module") @@ -75,3 +77,45 @@ def test_parse_cancel_output(self, slurm_io, in_out_ref, test_utils): stderr=bytes(parse_cmd_output["stderr"], "ascii"), ) assert cr == cr_ref + + @pytest.mark.parametrize("in_out_ref", in_out_job_ref_list) + def test_parse_job_output(self, slurm_io, in_out_ref, test_utils): + parse_cmd_output, job_ref = test_utils.inkwargs_outref( + in_out_ref, inkey="parse_job_kwargs", outkey="job_ref" + ) + job = slurm_io.parse_job_output(**parse_cmd_output) + assert job == job_ref + job = slurm_io.parse_job_output( + exit_code=parse_cmd_output["exit_code"], + stdout=bytes(parse_cmd_output["stdout"], "utf-8"), + stderr=bytes(parse_cmd_output["stderr"], "utf-8"), + ) + assert job == job_ref + job = slurm_io.parse_job_output( + exit_code=parse_cmd_output["exit_code"], + stdout=bytes(parse_cmd_output["stdout"], "ascii"), + stderr=bytes(parse_cmd_output["stderr"], "ascii"), + ) + assert job == job_ref + + def test_get_job_cmd(self, slurm_io): + cmd = slurm_io._get_job_cmd(3) + assert cmd == "SLURM_TIME_FORMAT='standard' scontrol show job -o 3" + cmd = slurm_io._get_job_cmd("56") + assert cmd == "SLURM_TIME_FORMAT='standard' scontrol show job -o 56" + + def test_get_jobs_list_cmd(self, slurm_io): + with pytest.raises( + ValueError, match=r"Cannot query by user and job\(s\) in SLURM" + ): + slurm_io._get_jobs_list_cmd(job_ids=["1"], user="johndoe") + cmd = slurm_io._get_jobs_list_cmd(user="johndoe") + assert ( + cmd + == "SLURM_TIME_FORMAT='standard' squeue --noheader -o '%i<><> %t<><> %r<><> %j<><> %u<><> %P<><> %l<><> %D<><> %C<><> %M<><> %m' -u johndoe" + ) + cmd = slurm_io._get_jobs_list_cmd(job_ids=["1", "3", "56", "15"]) + assert ( + cmd + == "SLURM_TIME_FORMAT='standard' squeue --noheader -o '%i<><> %t<><> %r<><> %j<><> %u<><> %P<><> %l<><> %D<><> %C<><> %M<><> %m' --jobs=1,3,56,15" + ) diff --git a/tests/test_data/io/slurm/create_parse_job_output_inout.py b/tests/test_data/io/slurm/create_parse_job_output_inout.py new file mode 100644 index 0000000..e840518 --- /dev/null +++ b/tests/test_data/io/slurm/create_parse_job_output_inout.py @@ -0,0 +1,25 @@ +import json + +import yaml + +from qtoolkit.io.slurm import SlurmIO + +slurm_io = SlurmIO() + +mylist = [] + +return_code = 0 +stdout = b"JobId=270 JobName=submit.script UserId=matgenix-dwa(1001) GroupId=matgenix-dwa(1002) MCS_label=N/A Priority=4294901497 Nice=0 Account=(null) QOS=normal JobState=COMPLETED Reason=None Dependency=(null) Requeue=1 Restarts=0 BatchFlag=1 Reboot=0 ExitCode=0:0 RunTime=00:05:00 TimeLimit=UNLIMITED TimeMin=N/A SubmitTime=2023-10-11T11:08:17 EligibleTime=2023-10-11T11:08:17 AccrueTime=2023-10-11T11:08:17 StartTime=2023-10-11T11:08:17 EndTime=2023-10-11T11:13:17 Deadline=N/A SuspendTime=None SecsPreSuspend=0 LastSchedEval=2023-10-11T11:08:17 Scheduler=Main Partition=main AllocNode:Sid=matgenixdb:2556938 ReqNodeList=(null) ExcNodeList=(null) NodeList=matgenixdb BatchHost=matgenixdb NumNodes=1 NumCPUs=1 NumTasks=1 CPUs/Task=1 ReqB:S:C:T=0:0:*:* TRES=cpu=1,mem=96G,node=1,billing=1 Socks/Node=* NtasksPerN:B:S:C=0:0:*:* CoreSpec=* MinCPUsNode=1 MinMemoryNode=0 MinTmpDiskNode=0 Features=(null) DelayBoot=00:00:00 OverSubscribe=OK Contiguous=0 Licenses=(null) Network=(null) Command=/home/matgenix-dwa/software/qtoolkit/tests/test_data/io/slurm/submit.script WorkDir=/home/matgenix-dwa/software/qtoolkit/tests/test_data/io/slurm StdErr=/home/matgenix-dwa/software/qtoolkit/tests/test_data/io/slurm/slurm-270.out StdIn=/dev/null StdOut=/home/matgenix-dwa/software/qtoolkit/tests/test_data/io/slurm/slurm-270.out Power= \n" +stderr = b"" +job = slurm_io.parse_job_output(exit_code=return_code, stdout=stdout, stderr=stderr) +a = { + "parse_job_kwargs": json.dumps( + {"exit_code": return_code, "stdout": stdout.decode(), "stderr": stderr.decode()} + ), + "job_ref": json.dumps(job.as_dict()), +} +mylist.append(a) + + +with open("parse_job_output_inout.yaml", "w") as f: + yaml.dump(mylist, f, sort_keys=False) diff --git a/tests/test_data/io/slurm/parse_job_output_inout.yaml b/tests/test_data/io/slurm/parse_job_output_inout.yaml new file mode 100644 index 0000000..d37eb84 --- /dev/null +++ b/tests/test_data/io/slurm/parse_job_output_inout.yaml @@ -0,0 +1,23 @@ +- parse_job_kwargs: '{"exit_code": 0, "stdout": "JobId=270 JobName=submit.script UserId=matgenix-dwa(1001) + GroupId=matgenix-dwa(1002) MCS_label=N/A Priority=4294901497 Nice=0 Account=(null) + QOS=normal JobState=COMPLETED Reason=None Dependency=(null) Requeue=1 Restarts=0 + BatchFlag=1 Reboot=0 ExitCode=0:0 RunTime=00:05:00 TimeLimit=UNLIMITED TimeMin=N/A + SubmitTime=2023-10-11T11:08:17 EligibleTime=2023-10-11T11:08:17 AccrueTime=2023-10-11T11:08:17 + StartTime=2023-10-11T11:08:17 EndTime=2023-10-11T11:13:17 Deadline=N/A SuspendTime=None + SecsPreSuspend=0 LastSchedEval=2023-10-11T11:08:17 Scheduler=Main Partition=main + AllocNode:Sid=matgenixdb:2556938 ReqNodeList=(null) ExcNodeList=(null) NodeList=matgenixdb + BatchHost=matgenixdb NumNodes=1 NumCPUs=1 NumTasks=1 CPUs/Task=1 ReqB:S:C:T=0:0:*:* + TRES=cpu=1,mem=96G,node=1,billing=1 Socks/Node=* NtasksPerN:B:S:C=0:0:*:* CoreSpec=* + MinCPUsNode=1 MinMemoryNode=0 MinTmpDiskNode=0 Features=(null) DelayBoot=00:00:00 + OverSubscribe=OK Contiguous=0 Licenses=(null) Network=(null) Command=/home/matgenix-dwa/software/qtoolkit/tests/test_data/io/slurm/submit.script + WorkDir=/home/matgenix-dwa/software/qtoolkit/tests/test_data/io/slurm StdErr=/home/matgenix-dwa/software/qtoolkit/tests/test_data/io/slurm/slurm-270.out + StdIn=/dev/null StdOut=/home/matgenix-dwa/software/qtoolkit/tests/test_data/io/slurm/slurm-270.out + Power= \n", "stderr": ""}' + job_ref: '{"@module": "qtoolkit.core.data_objects", "@class": "QJob", "@version": + "0.1.1", "name": "submit.script", "job_id": "270", "exit_status": null, "state": + {"@module": "qtoolkit.core.data_objects", "@class": "QState", "@version": "0.1.1", + "value": "DONE"}, "sub_state": {"@module": "qtoolkit.io.slurm", "@class": "SlurmState", + "@version": "0.1.1", "value": "COMPLETED"}, "info": {"@module": "qtoolkit.core.data_objects", + "@class": "QJobInfo", "@version": "0.1.1", "memory": null, "memory_per_cpu": null, + "nodes": 1, "cpus": 1, "threads_per_process": 1, "time_limit": null}, "account": + "matgenix-dwa(1001)", "runtime": null, "queue_name": "main"}' From e794b970a0b3fea81afcd974d46ea5776dab915c Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Wed, 11 Oct 2023 14:21:21 +0200 Subject: [PATCH 09/18] Added more unit tests to slurm. --- tests/io/test_slurm.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/io/test_slurm.py b/tests/io/test_slurm.py index 535c9b6..033a4c3 100644 --- a/tests/io/test_slurm.py +++ b/tests/io/test_slurm.py @@ -110,12 +110,23 @@ def test_get_jobs_list_cmd(self, slurm_io): ): slurm_io._get_jobs_list_cmd(job_ids=["1"], user="johndoe") cmd = slurm_io._get_jobs_list_cmd(user="johndoe") - assert ( - cmd - == "SLURM_TIME_FORMAT='standard' squeue --noheader -o '%i<><> %t<><> %r<><> %j<><> %u<><> %P<><> %l<><> %D<><> %C<><> %M<><> %m' -u johndoe" + assert cmd == ( + "SLURM_TIME_FORMAT='standard' " + "squeue --noheader -o '%i<><> %t<><> %r<><> " + "%j<><> %u<><> %P<><> %l<><> %D<><> %C<><> " + "%M<><> %m' -u johndoe" ) cmd = slurm_io._get_jobs_list_cmd(job_ids=["1", "3", "56", "15"]) - assert ( - cmd - == "SLURM_TIME_FORMAT='standard' squeue --noheader -o '%i<><> %t<><> %r<><> %j<><> %u<><> %P<><> %l<><> %D<><> %C<><> %M<><> %m' --jobs=1,3,56,15" + assert cmd == ( + "SLURM_TIME_FORMAT='standard' " + "squeue --noheader -o '%i<><> %t<><> %r<><> " + "%j<><> %u<><> %P<><> %l<><> %D<><> %C<><> " + "%M<><> %m' --jobs=1,3,56,15" + ) + cmd = slurm_io._get_jobs_list_cmd(job_ids=["1"]) + assert cmd == ( + "SLURM_TIME_FORMAT='standard' " + "squeue --noheader -o '%i<><> %t<><> %r<><> " + "%j<><> %u<><> %P<><> %l<><> %D<><> %C<><> " + "%M<><> %m' --jobs=1,1" ) From f653476137ef3e06582f6f75d42c0d40e1496c9c Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Wed, 11 Oct 2023 15:16:49 +0200 Subject: [PATCH 10/18] More tests --- src/qtoolkit/io/slurm.py | 3 ++ tests/io/test_slurm.py | 102 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 104 insertions(+), 1 deletion(-) diff --git a/src/qtoolkit/io/slurm.py b/src/qtoolkit/io/slurm.py index dd46993..d4029bd 100644 --- a/src/qtoolkit/io/slurm.py +++ b/src/qtoolkit/io/slurm.py @@ -499,6 +499,9 @@ def _convert_memory_str(memory: str | None) -> int | None: if not memory: return None + # TODO: @GP not sure I get what is this line here + # Shouldn't it be all(u not in memory for u in ("K", "M", "G", "T"))? + # Or not any(u in memory for u in ("K", "M", "G", "T"))? if all(u in memory for u in ("K", "M", "G", "T")): # assume Mb units = "M" diff --git a/tests/io/test_slurm.py b/tests/io/test_slurm.py index 033a4c3..41e3557 100644 --- a/tests/io/test_slurm.py +++ b/tests/io/test_slurm.py @@ -1,9 +1,11 @@ +from datetime import timedelta from pathlib import Path import pytest from monty.serialization import loadfn -from qtoolkit.core.data_objects import QState +from qtoolkit.core.data_objects import ProcessPlacement, QResources, QState +from qtoolkit.core.exceptions import OutputParsingError from qtoolkit.io.slurm import SlurmIO, SlurmState TEST_DIR = Path(__file__).resolve().parents[1] / "test_data" @@ -130,3 +132,101 @@ def test_get_jobs_list_cmd(self, slurm_io): "%j<><> %u<><> %P<><> %l<><> %D<><> %C<><> " "%M<><> %m' --jobs=1,1" ) + + def test_convert_str_to_time(self, slurm_io): + time_seconds = slurm_io._convert_str_to_time(None) + assert time_seconds is None + time_seconds = slurm_io._convert_str_to_time("UNLIMITED") + assert time_seconds is None + time_seconds = slurm_io._convert_str_to_time("NOT_SET") + assert time_seconds is None + + time_seconds = slurm_io._convert_str_to_time("3-10:51:13") + assert time_seconds == 298273 + time_seconds = slurm_io._convert_str_to_time("2:10:02") + assert time_seconds == 7802 + time_seconds = slurm_io._convert_str_to_time("10:02") + assert time_seconds == 602 + time_seconds = slurm_io._convert_str_to_time("45") + assert time_seconds == 2700 + + with pytest.raises(OutputParsingError): + slurm_io._convert_str_to_time("2:10:02:10") + + with pytest.raises(OutputParsingError): + slurm_io._convert_str_to_time("2:10:a") + + def test_convert_memory_str(self, slurm_io): + memory_kb = slurm_io._convert_memory_str(None) + assert memory_kb is None + memory_kb = slurm_io._convert_memory_str("") + assert memory_kb is None + + memory_kb = slurm_io._convert_memory_str("12M") + assert memory_kb == 12288 + memory_kb = slurm_io._convert_memory_str("13K") + assert memory_kb == 13 + memory_kb = slurm_io._convert_memory_str("5G") + assert memory_kb == 5242880 + memory_kb = slurm_io._convert_memory_str("1T") + assert memory_kb == 1073741824 + + with pytest.raises(OutputParsingError): + slurm_io._convert_memory_str("aT") + + def test_convert_time_to_str(self, slurm_io): + time_str = slurm_io._convert_time_to_str(10) + assert time_str == "0-0:0:10" + time_str = slurm_io._convert_time_to_str(298273) + assert time_str == "3-10:51:13" + time_str = slurm_io._convert_time_to_str(7802) + assert time_str == "0-2:10:2" + time_str = slurm_io._convert_time_to_str(602) + assert time_str == "0-0:10:2" + + time_str = slurm_io._convert_time_to_str(timedelta(seconds=298273)) + assert time_str == "3-10:51:13" + time_str = slurm_io._convert_time_to_str( + timedelta(days=15, hours=21, minutes=19, seconds=32) + ) + assert time_str == "15-21:19:32" + + def test_convert_qresources(self, slurm_io): + res = QResources( + queue_name="myqueue", + job_name="myjob", + memory_per_thread=2048, + account="myaccount", + qos="myqos", + output_filepath="someoutputpath", + error_filepath="someerrorpath", + njobs=4, + time_limit=298273, + process_placement=ProcessPlacement.EVENLY_DISTRIBUTED, + nodes=4, + processes_per_node=3, + threads_per_process=2, + gpus_per_job=4, + email_address="john.doe@submit.qtk", + kwargs={"tata": "toto", "titi": "tutu"}, + ) + header_dict = slurm_io._convert_qresources(resources=res) + assert header_dict == { + "partition": "myqueue", + "job_name": "myjob", + "mem-per-cpu": 2048, + "account": "myaccount", + "qos": "myqos", + "qout_path": "someoutputpath", + "qerr_path": "someerrorpath", + "array": "1-4", + "time": "3-10:51:13", + "ntasks_per_node": 3, + "nodes": 4, + "cpus_per_task": 2, + "gres": "gpu:4", + "mail_user": "john.doe@submit.qtk", + "mail_type": "ALL", + "tata": "toto", + "titi": "tutu", + } From a8b3228d9baadd45ba359ec0a66b7c7583650471 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Wed, 11 Oct 2023 15:42:59 +0200 Subject: [PATCH 11/18] More tests. --- tests/io/test_slurm.py | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/tests/io/test_slurm.py b/tests/io/test_slurm.py index 41e3557..f1b32d0 100644 --- a/tests/io/test_slurm.py +++ b/tests/io/test_slurm.py @@ -5,7 +5,7 @@ from monty.serialization import loadfn from qtoolkit.core.data_objects import ProcessPlacement, QResources, QState -from qtoolkit.core.exceptions import OutputParsingError +from qtoolkit.core.exceptions import OutputParsingError, UnsupportedResourcesError from qtoolkit.io.slurm import SlurmIO, SlurmState TEST_DIR = Path(__file__).resolve().parents[1] / "test_data" @@ -191,7 +191,7 @@ def test_convert_time_to_str(self, slurm_io): ) assert time_str == "15-21:19:32" - def test_convert_qresources(self, slurm_io): + def test_check_convert_qresources(self, slurm_io): res = QResources( queue_name="myqueue", job_name="myjob", @@ -210,7 +210,7 @@ def test_convert_qresources(self, slurm_io): email_address="john.doe@submit.qtk", kwargs={"tata": "toto", "titi": "tutu"}, ) - header_dict = slurm_io._convert_qresources(resources=res) + header_dict = slurm_io.check_convert_qresources(resources=res) assert header_dict == { "partition": "myqueue", "job_name": "myjob", @@ -230,3 +230,33 @@ def test_convert_qresources(self, slurm_io): "tata": "toto", "titi": "tutu", } + + res = QResources( + time_limit=298273, + processes=24, + ) + header_dict = slurm_io.check_convert_qresources(resources=res) + assert header_dict == { + "time": "3-10:51:13", + "ntasks": 24, + } + + res = QResources( + njobs=1, + processes=24, + gpus_per_job=4, + ) + header_dict = slurm_io.check_convert_qresources(resources=res) + assert header_dict == { + "ntasks": 24, + "gres": "gpu:4", + } + + res = QResources( + processes=5, + rerunnable=True, + ) + with pytest.raises( + UnsupportedResourcesError, match=r"Keys not supported: rerunnable" + ): + slurm_io.check_convert_qresources(res) From 4806e694104a6fd06f0fe11754e292b8ae421f33 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Wed, 11 Oct 2023 16:12:22 +0200 Subject: [PATCH 12/18] More unit tests. --- src/qtoolkit/io/slurm.py | 2 +- .../io/slurm/create_parse_job_output_inout.py | 26 +++++++++++++++++++ .../io/slurm/parse_job_output_inout.yaml | 25 ++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/qtoolkit/io/slurm.py b/src/qtoolkit/io/slurm.py index d4029bd..e01aa14 100644 --- a/src/qtoolkit/io/slurm.py +++ b/src/qtoolkit/io/slurm.py @@ -283,7 +283,7 @@ def parse_job_output(self, exit_code, stdout, stderr) -> QJob | None: parsed_output = self._parse_scontrol_cmd_output(stdout=stdout) elif self.get_job_executable == "sacct": # pragma: no cover raise NotImplementedError("sacct for get_job not yet implemented.") - else: + else: # pragma: no cover raise RuntimeError( f'"{self.get_job_executable}" is not a valid get_job_executable.' ) diff --git a/tests/test_data/io/slurm/create_parse_job_output_inout.py b/tests/test_data/io/slurm/create_parse_job_output_inout.py index e840518..187c38e 100644 --- a/tests/test_data/io/slurm/create_parse_job_output_inout.py +++ b/tests/test_data/io/slurm/create_parse_job_output_inout.py @@ -21,5 +21,31 @@ mylist.append(a) +return_code = 0 +stdout = b"JobId=270 JobName=submit.script UserId=matgenix-dwa(1001) GroupId=matgenix-dwa(1002) MCS_label=N/A Priority=4294901497 Nice=0 Account=(null) QOS=normal JobState=COMPLETED Reason=None Dependency=(null) Requeue=1 Restarts=0 BatchFlag=1 Reboot=0 ExitCode=0:0 RunTime=00:05:00 TimeLimit=a TimeMin=N/A SubmitTime=2023-10-11T11:08:17 EligibleTime=2023-10-11T11:08:17 AccrueTime=2023-10-11T11:08:17 StartTime=2023-10-11T11:08:17 EndTime=2023-10-11T11:13:17 Deadline=N/A SuspendTime=None SecsPreSuspend=0 LastSchedEval=2023-10-11T11:08:17 Scheduler=Main Partition=main AllocNode:Sid=matgenixdb:2556938 ReqNodeList=(null) ExcNodeList=(null) NodeList=matgenixdb BatchHost=matgenixdb NumNodes=a NumCPUs=a NumTasks=1 CPUs/Task=a ReqB:S:C:T=0:0:*:* TRES=cpu=1,mem=96G,node=1,billing=1 Socks/Node=* NtasksPerN:B:S:C=0:0:*:* CoreSpec=* MinCPUsNode=1 MinMemoryNode=0 MinTmpDiskNode=0 Features=(null) DelayBoot=00:00:00 OverSubscribe=OK Contiguous=0 Licenses=(null) Network=(null) Command=/home/matgenix-dwa/software/qtoolkit/tests/test_data/io/slurm/submit.script WorkDir=/home/matgenix-dwa/software/qtoolkit/tests/test_data/io/slurm StdErr=/home/matgenix-dwa/software/qtoolkit/tests/test_data/io/slurm/slurm-270.out StdIn=/dev/null StdOut=/home/matgenix-dwa/software/qtoolkit/tests/test_data/io/slurm/slurm-270.out Power= \n" +stderr = b"" +job = slurm_io.parse_job_output(exit_code=return_code, stdout=stdout, stderr=stderr) +a = { + "parse_job_kwargs": json.dumps( + {"exit_code": return_code, "stdout": stdout.decode(), "stderr": stderr.decode()} + ), + "job_ref": json.dumps(job.as_dict()), +} +mylist.append(a) + + +return_code = 0 +stdout = b"" +stderr = b"" +job = slurm_io.parse_job_output(exit_code=return_code, stdout=stdout, stderr=stderr) +a = { + "parse_job_kwargs": json.dumps( + {"exit_code": return_code, "stdout": stdout.decode(), "stderr": stderr.decode()} + ), + "job_ref": json.dumps(job.as_dict() if job is not None else None), +} +mylist.append(a) + + with open("parse_job_output_inout.yaml", "w") as f: yaml.dump(mylist, f, sort_keys=False) diff --git a/tests/test_data/io/slurm/parse_job_output_inout.yaml b/tests/test_data/io/slurm/parse_job_output_inout.yaml index d37eb84..4beeecf 100644 --- a/tests/test_data/io/slurm/parse_job_output_inout.yaml +++ b/tests/test_data/io/slurm/parse_job_output_inout.yaml @@ -21,3 +21,28 @@ "@class": "QJobInfo", "@version": "0.1.1", "memory": null, "memory_per_cpu": null, "nodes": 1, "cpus": 1, "threads_per_process": 1, "time_limit": null}, "account": "matgenix-dwa(1001)", "runtime": null, "queue_name": "main"}' +- parse_job_kwargs: '{"exit_code": 0, "stdout": "JobId=270 JobName=submit.script UserId=matgenix-dwa(1001) + GroupId=matgenix-dwa(1002) MCS_label=N/A Priority=4294901497 Nice=0 Account=(null) + QOS=normal JobState=COMPLETED Reason=None Dependency=(null) Requeue=1 Restarts=0 + BatchFlag=1 Reboot=0 ExitCode=0:0 RunTime=00:05:00 TimeLimit=a TimeMin=N/A SubmitTime=2023-10-11T11:08:17 + EligibleTime=2023-10-11T11:08:17 AccrueTime=2023-10-11T11:08:17 StartTime=2023-10-11T11:08:17 + EndTime=2023-10-11T11:13:17 Deadline=N/A SuspendTime=None SecsPreSuspend=0 LastSchedEval=2023-10-11T11:08:17 + Scheduler=Main Partition=main AllocNode:Sid=matgenixdb:2556938 ReqNodeList=(null) + ExcNodeList=(null) NodeList=matgenixdb BatchHost=matgenixdb NumNodes=a NumCPUs=a + NumTasks=1 CPUs/Task=a ReqB:S:C:T=0:0:*:* TRES=cpu=1,mem=96G,node=1,billing=1 + Socks/Node=* NtasksPerN:B:S:C=0:0:*:* CoreSpec=* MinCPUsNode=1 MinMemoryNode=0 + MinTmpDiskNode=0 Features=(null) DelayBoot=00:00:00 OverSubscribe=OK Contiguous=0 + Licenses=(null) Network=(null) Command=/home/matgenix-dwa/software/qtoolkit/tests/test_data/io/slurm/submit.script + WorkDir=/home/matgenix-dwa/software/qtoolkit/tests/test_data/io/slurm StdErr=/home/matgenix-dwa/software/qtoolkit/tests/test_data/io/slurm/slurm-270.out + StdIn=/dev/null StdOut=/home/matgenix-dwa/software/qtoolkit/tests/test_data/io/slurm/slurm-270.out + Power= \n", "stderr": ""}' + job_ref: '{"@module": "qtoolkit.core.data_objects", "@class": "QJob", "@version": + "0.1.1", "name": "submit.script", "job_id": "270", "exit_status": null, "state": + {"@module": "qtoolkit.core.data_objects", "@class": "QState", "@version": "0.1.1", + "value": "DONE"}, "sub_state": {"@module": "qtoolkit.io.slurm", "@class": "SlurmState", + "@version": "0.1.1", "value": "COMPLETED"}, "info": {"@module": "qtoolkit.core.data_objects", + "@class": "QJobInfo", "@version": "0.1.1", "memory": null, "memory_per_cpu": null, + "nodes": null, "cpus": null, "threads_per_process": null, "time_limit": null}, + "account": "matgenix-dwa(1001)", "runtime": null, "queue_name": "main"}' +- parse_job_kwargs: '{"exit_code": 0, "stdout": "", "stderr": ""}' + job_ref: 'null' From 5da1fa5f6a9f2c9fe945c13f5b03be63501c3d0a Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Wed, 11 Oct 2023 16:32:56 +0200 Subject: [PATCH 13/18] Skip coverage for part of testing get_identifiers as we are not using any complex patterns in qtoolkit... And in any case, get_identifiers is only there because it's not part of the standard library for python < 3.11. --- src/qtoolkit/io/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/qtoolkit/io/base.py b/src/qtoolkit/io/base.py index bebe3ca..d0e5908 100644 --- a/src/qtoolkit/io/base.py +++ b/src/qtoolkit/io/base.py @@ -30,7 +30,7 @@ def get_identifiers(self) -> list: named is None and mo.group("invalid") is None and mo.group("escaped") is None - ): + ): # pragma: no cover - no complex patterns, part of python stdlib 3.11 # If all the groups are None, there must be # another group we're not expecting raise ValueError("Unrecognized named group in pattern", self.pattern) From f04d127c53bd9940ee484dcdd274bece816e19de Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Wed, 11 Oct 2023 16:36:11 +0200 Subject: [PATCH 14/18] Removed old comments in io/base. --- src/qtoolkit/io/base.py | 132 +--------------------------------------- 1 file changed, 1 insertion(+), 131 deletions(-) diff --git a/src/qtoolkit/io/base.py b/src/qtoolkit/io/base.py index d0e5908..b14ea39 100644 --- a/src/qtoolkit/io/base.py +++ b/src/qtoolkit/io/base.py @@ -47,142 +47,12 @@ class BaseSchedulerIO(QTKObject, abc.ABC): shebang: str = "#!/bin/bash" - # config: QueueConfig = None - - # scheduler = None, - # name = None, - # cores = None, - # memory = None, - # processes = None, - # nanny = True, - # protocol = None, - # security = None, - # interface = None, - # death_timeout = None, - # local_directory = None, - # extra = None, - # worker_extra_args = None, - # job_extra = None, - # job_extra_directives = None, - # env_extra = None, - # job_script_prologue = None, - # header_skip = None, - # job_directives_skip = None, - # log_directory = None, - # shebang = None, - # python = sys.executable, - # job_name = None, - # config_name = None, - - """ABIPY - - Args: - qname: Name of the queue. - qparams: Dictionary with the parameters used in the template. - setup: String or list of commands to execute during the initial setup. - modules: String or list of modules to load before running the application. - shell_env: Dictionary with the environment variables to export before - running the application. - omp_env: Dictionary with the OpenMP variables. - pre_run: String or list of commands to execute before launching the - calculation. - post_run: String or list of commands to execute once the calculation is - completed. - mpi_runner: Path to the MPI runner or :class:`MpiRunner` instance. - None if not used - mpi_runner_options: Optional string with options passed to the mpi_runner. - max_num_launches: Maximum number of submissions that can be done for a - specific task. Defaults to 5 - qverbatim: - min_cores, max_cores, hint_cores: Minimum, maximum, and hint limits of - number of cores that can be used - min_mem_per_proc=Minimum memory per process in megabytes. - max_mem_per_proc=Maximum memory per process in megabytes. - timelimit: initial time limit in seconds - timelimit_hard: hard limelimit for this queue - priority: Priority level, integer number > 0 - condition: Condition object (dictionary) - - """ - def get_submission_script( self, commands: str | list[str], options: dict | QResources | None = None, ) -> str: - """ - This is roughly what/how it is done in the existing solutions. - - abipy: done with a str template (using $$ as a delimiter). - Remaining "$$" delimiters are then removed at the end. - It uses a ScriptEditor object to add/modify things to the templated script. - The different steps of "get_script_str(...)" in abipy are summarized here: - - _header, based on the str template (includes the shebang line and all - #SBATCH, #PBS, ... directives) - - change directory (added by the script editor) - - setup section, list of commands executed before running (added - by the script editor) - - load modules section, list of modules to be loaded before running - (added by the script editor) - - setting of openmp environment variables (added by the script editor) - - setting of shell environment variables (added by the script editor) - - prerun, i.e. commands to run before execution, again? (added by - the script editor) - - run line (added by the script editor) - - postrun (added by the script editor) - - aiida: done with a class template (JobTemplate) that should contain - the required info to generate the job header. Other class templates - are also used inside the generation, e.g. JobTemplateCodesInfo, which - defines the command(s) to be run. The JobTemplate is only used as a - container of the information and the script is generated not using - templating but rather directly using python methods based on that - "JobTemplate" container. Actually this JobTemplate is based on the - DRMAA v2 specifications and many other objects are based on that too - (e.g. machine, slots, etc ...). - The different steps of "get_submit_script(...)" in aiida are summarized here: - - shebang line - - _header - - all #SBATCH, #PBS etc ... lines defining the resources and other - info for the queuing system - - some custom lines if it is not dealt with by the template - - environment variables - - prepend_text (something to be written before the run lines) - - _run_line (defines the code execution(s) based on a CodeInfo object). - There can be several codes run. - - append_text (something to be written after the run lines) - - _footer (some post commands done after the run) [note this is only - done/needed for LSF in aiida] - - fireworks: done with a str template. similar to abipy (actually abipy took - its initial concept from fireworks) - - dask_jobqueue: quite obscure ... the job header is done in the init of a - given JobCluster (e.g. SLURMCluster) based on something in the actual - Job object itself. Dask is not really meant to be our use case anyway. - - dpdispatcher: uses python's format() with 5 templates, combined into - another python's format "script" template. - Here are the steps: - - header (includes shebang and #SBATCH, #PBS, ... directives) - - custom directives - - script environment (modules, environment variables, source - somefiles, ...) - - run command - - append script lines - In the templates of the different steps, there are some dpdispatcher's - specific things (e.g. tag a job as finished by touching a file, ...) - - jobqueues: Some queues are using pure python (PBS, LSF, ...), some are - using jinja2 templates (SLURM and SGE). Directly written to file. - - myqueue: the job queue directives are directly passed to the submit - command (no #SBATCH, #PBS, ...). - - troika: uses a generic generator with a list of directives as well - as a directive prefix. These directives are defined in specific - files for each type of job queue. - """ + """Get the submission script for the given commands and options.""" script_blocks = [self.shebang] if header := self.generate_header(options): script_blocks.append(header) From 2fac00b431952f54d5c3d6531bed8d1f7aff0f4e Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Thu, 12 Oct 2023 08:57:19 +0200 Subject: [PATCH 15/18] Refactored tests for BaseScheduler into a class. --- tests/io/test_base.py | 123 ++++++++++++++++++++++-------------------- 1 file changed, 66 insertions(+), 57 deletions(-) diff --git a/tests/io/test_base.py b/tests/io/test_base.py index 41b295b..8c1c8c1 100644 --- a/tests/io/test_base.py +++ b/tests/io/test_base.py @@ -51,74 +51,83 @@ def test_qtemplate(): ) -def test_base_scheduler(): - class MyScheduler(BaseSchedulerIO): - pass - - with pytest.raises(TypeError): - MyScheduler() - - class MyScheduler(BaseSchedulerIO): - header_template = """#SPECCMD --option1=$${option1} +class TestBaseScheduler: + @pytest.fixture(scope="module") + def scheduler(self): + class MyScheduler(BaseSchedulerIO): + header_template = """#SPECCMD --option1=$${option1} #SPECCMD --option2=$${option2} #SPECCMD --option3=$${option3}""" - SUBMIT_CMD = "mysubmit" - CANCEL_CMD = "mycancel" + SUBMIT_CMD = "mysubmit" + CANCEL_CMD = "mycancel" - def parse_submit_output(self, exit_code, stdout, stderr) -> SubmissionResult: - pass + def parse_submit_output( + self, exit_code, stdout, stderr + ) -> SubmissionResult: + pass - def parse_cancel_output(self, exit_code, stdout, stderr) -> CancelResult: - pass + def parse_cancel_output(self, exit_code, stdout, stderr) -> CancelResult: + pass - def _get_job_cmd(self, job_id: str) -> str: - pass + def _get_job_cmd(self, job_id: str) -> str: + pass - def parse_job_output(self, exit_code, stdout, stderr) -> QJob | None: - pass + def parse_job_output(self, exit_code, stdout, stderr) -> QJob | None: + pass - def _convert_qresources(self, resources: QResources) -> dict: - pass + def _convert_qresources(self, resources: QResources) -> dict: + pass - @property - def supported_qresources_keys(self) -> list: - return [] + @property + def supported_qresources_keys(self) -> list: + return [] - def _get_jobs_list_cmd( - self, job_ids: list[str] | None = None, user: str | None = None - ) -> str: - pass + def _get_jobs_list_cmd( + self, job_ids: list[str] | None = None, user: str | None = None + ) -> str: + pass - def parse_jobs_list_output(self, exit_code, stdout, stderr) -> list[QJob]: - pass + def parse_jobs_list_output(self, exit_code, stdout, stderr) -> list[QJob]: + pass - scheduler = MyScheduler() + return MyScheduler() - header = scheduler.generate_header({"option2": "value_option2"}) - assert header == """#SPECCMD --option2=value_option2""" + def test_subclass_base_scheduler(self, scheduler): + class MyScheduler(BaseSchedulerIO): + pass - ids_list = scheduler.generate_ids_list( - [QJob(job_id=4), QJob(job_id="job_id_abc1"), 215, "job12345"] - ) - assert ids_list == ["4", "job_id_abc1", "215", "job12345"] - - submit_cmd = scheduler.get_submit_cmd() - assert submit_cmd == "mysubmit submit.script" - submit_cmd = scheduler.get_submit_cmd(script_file="sub.sh") - assert submit_cmd == "mysubmit sub.sh" - - cancel_cmd = scheduler.get_cancel_cmd(QJob(job_id=5)) - assert cancel_cmd == "mycancel 5" - cancel_cmd = scheduler.get_cancel_cmd(QJob(job_id="abc1")) - assert cancel_cmd == "mycancel abc1" - cancel_cmd = scheduler.get_cancel_cmd("jobid2") - assert cancel_cmd == "mycancel jobid2" - cancel_cmd = scheduler.get_cancel_cmd(632) - assert cancel_cmd == "mycancel 632" - - with pytest.raises( - ValueError, - match=r"The id of the job to be cancelled should be defined. Received: None", - ): - scheduler.get_cancel_cmd(job=None) + with pytest.raises(TypeError): + MyScheduler() + + def test_generate_header(self, scheduler): + header = scheduler.generate_header({"option2": "value_option2"}) + assert header == """#SPECCMD --option2=value_option2""" + + def test_generate_ids_list(self, scheduler): + ids_list = scheduler.generate_ids_list( + [QJob(job_id=4), QJob(job_id="job_id_abc1"), 215, "job12345"] + ) + assert ids_list == ["4", "job_id_abc1", "215", "job12345"] + + def test_get_submit_cmd(self, scheduler): + submit_cmd = scheduler.get_submit_cmd() + assert submit_cmd == "mysubmit submit.script" + submit_cmd = scheduler.get_submit_cmd(script_file="sub.sh") + assert submit_cmd == "mysubmit sub.sh" + + def test_get_cancel_cmd(self, scheduler): + cancel_cmd = scheduler.get_cancel_cmd(QJob(job_id=5)) + assert cancel_cmd == "mycancel 5" + cancel_cmd = scheduler.get_cancel_cmd(QJob(job_id="abc1")) + assert cancel_cmd == "mycancel abc1" + cancel_cmd = scheduler.get_cancel_cmd("jobid2") + assert cancel_cmd == "mycancel jobid2" + cancel_cmd = scheduler.get_cancel_cmd(632) + assert cancel_cmd == "mycancel 632" + + with pytest.raises( + ValueError, + match=r"The id of the job to be cancelled should be defined. Received: None", + ): + scheduler.get_cancel_cmd(job=None) From 62811f7b820415220944a5990190d628370c743a Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Thu, 16 Nov 2023 12:31:39 +0100 Subject: [PATCH 16/18] Small fix + added more unit tests. --- src/qtoolkit/io/base.py | 3 +- tests/io/test_base.py | 63 ++++++++++++++++++++++++++++++++++++++--- 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/src/qtoolkit/io/base.py b/src/qtoolkit/io/base.py index b14ea39..a533e4b 100644 --- a/src/qtoolkit/io/base.py +++ b/src/qtoolkit/io/base.py @@ -141,8 +141,9 @@ def get_cancel_cmd(self, job: QJob | int | str) -> str: """ job_id = job.job_id if isinstance(job, QJob) else job if job_id is None or job_id == "": + received = None if job_id is None else "'' (empty string)" raise ValueError( - f"The id of the job to be cancelled should be defined. Received: {job_id}" + f"The id of the job to be cancelled should be defined. Received: {received}" ) return f"{self.CANCEL_CMD} {job_id}" diff --git a/tests/io/test_base.py b/tests/io/test_base.py index 8c1c8c1..fc09386 100644 --- a/tests/io/test_base.py +++ b/tests/io/test_base.py @@ -57,7 +57,10 @@ def scheduler(self): class MyScheduler(BaseSchedulerIO): header_template = """#SPECCMD --option1=$${option1} #SPECCMD --option2=$${option2} -#SPECCMD --option3=$${option3}""" +#SPECCMD --option3=$${option3} +#SPECCMD --processes=$${processes} +#SPECCMD --processes_per_node=$${processes_per_node} +#SPECCMD --nodes=$${nodes}""" SUBMIT_CMD = "mysubmit" CANCEL_CMD = "mycancel" @@ -77,11 +80,34 @@ def parse_job_output(self, exit_code, stdout, stderr) -> QJob | None: pass def _convert_qresources(self, resources: QResources) -> dict: - pass + header_dict = {} + + ( + nodes, + processes, + processes_per_node, + ) = resources.get_processes_distribution() + if processes: + header_dict["processes"] = processes + if processes_per_node: + header_dict["processes_per_node"] = processes_per_node + if nodes: + header_dict["nodes"] = nodes + + if resources.kwargs: + header_dict.update(resources.kwargs) + + return header_dict @property def supported_qresources_keys(self) -> list: - return [] + return [ + "kwargs", + "nodes", + "processes_per_node", + "process_placement", + "processes", + ] def _get_jobs_list_cmd( self, job_ids: list[str] | None = None, user: str | None = None @@ -104,6 +130,27 @@ def test_generate_header(self, scheduler): header = scheduler.generate_header({"option2": "value_option2"}) assert header == """#SPECCMD --option2=value_option2""" + res = QResources(processes=8) + header = scheduler.generate_header(res) + assert header == """#SPECCMD --processes=8""" + res = QResources(nodes=4, processes_per_node=16, kwargs={"option2": "myopt2"}) + header = scheduler.generate_header(res) + assert ( + header + == """#SPECCMD --option2=myopt2 +#SPECCMD --processes_per_node=16 +#SPECCMD --nodes=4""" + ) + + with pytest.raises( + ValueError, + match=r"The following keys are not present in the template: tata, titi", + ): + res = QResources( + nodes=4, processes_per_node=16, kwargs={"tata": "tata", "titi": "titi"} + ) + scheduler.generate_header(res) + def test_generate_ids_list(self, scheduler): ids_list = scheduler.generate_ids_list( [QJob(job_id=4), QJob(job_id="job_id_abc1"), 215, "job12345"] @@ -128,6 +175,14 @@ def test_get_cancel_cmd(self, scheduler): with pytest.raises( ValueError, - match=r"The id of the job to be cancelled should be defined. Received: None", + match=r"The id of the job to be cancelled should be defined. " + r"Received: None", ): scheduler.get_cancel_cmd(job=None) + + with pytest.raises( + ValueError, + match=r"The id of the job to be cancelled should be defined. " + r"Received: '' \(empty string\)", + ): + scheduler.get_cancel_cmd(job="") From a9fba7aaa9bafcc769e1112da4b9f58e44f300cd Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Thu, 16 Nov 2023 12:36:21 +0100 Subject: [PATCH 17/18] Alphabetic order of parameters in error message. --- src/qtoolkit/io/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/qtoolkit/io/base.py b/src/qtoolkit/io/base.py index a533e4b..b4ad46e 100644 --- a/src/qtoolkit/io/base.py +++ b/src/qtoolkit/io/base.py @@ -83,7 +83,7 @@ def generate_header(self, options: dict | QResources | None) -> str: keys = set(options.keys()) extra = keys.difference(template.get_identifiers()) if extra: - msg = f"The following keys are not present in the template: {', '.join(extra)}" + msg = f"The following keys are not present in the template: {', '.join(sorted(extra))}" raise ValueError(msg) unclean_header = template.safe_substitute(options) From 10b8838cde1c14a3541003b4725f088eaeca4199 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Thu, 16 Nov 2023 14:05:39 +0100 Subject: [PATCH 18/18] Update pyproject.toml --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cb5334c..f7a1c0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,14 +91,13 @@ filterwarnings = [ "ignore:.*input structure.*:UserWarning", "ignore::DeprecationWarning", ] -addopts = "--cov=src/qtoolkit" [tool.coverage.run] parallel = true branch = true [tool.coverage.paths] -source = ["src/qtoolkit"] +source = ["src/"] [tool.coverage.report] skip_covered = true