Implement default submission template

pysqa/utils/basic.py

@@ -13,7 +13,7 @@
 from pysqa.utils.execute import execute_command
 from pysqa.utils.queues import Queues
 from pysqa.utils.validate import value_error_if_none, value_in_range
-from pysqa.wrapper.generic import SchedulerCommands
+from pysqa.wrapper.abstract import SchedulerCommands
 
 queue_type_dict = {
     "SGE": {
@@ -511,14 +511,14 @@ def _job_submission_template(
             memory_max=memory_max,
             active_queue=active_queue,
         )
-        template = active_queue["template"]
-        return template.render(
-            job_name=job_name,
+        return self._commands.render_submission_template(
+            command=command,
+            submission_template=active_queue["template"],
             working_directory=working_directory,
+            job_name=job_name,
             cores=cores,
             memory_max=memory_max,
             run_time_max=run_time_max,
-            command=command,
             dependency_list=dependency_list,
             **kwargs,
         )

pysqa/wrapper/generic.py → pysqa/wrapper/abstract.py

@@ -2,9 +2,12 @@
 # Copyright (c) Max-Planck-Institut für Eisenforschung GmbH - Computational Materials Design (CM) Department
 # Distributed under the terms of "New BSD License", see the LICENSE file.
 
+import os
 from abc import ABC, abstractmethod
+from typing import List, Optional, Union
 
 import pandas
+from jinja2 import Template
 
 __author__ = "Niklas Siemer"
 __copyright__ = (
@@ -19,6 +22,16 @@
 
 
 class SchedulerCommands(ABC):
+    @property
+    def enable_reservation_command(self) -> list[str]:
+        """
+        Returns the command to enable job reservation on the scheduler.
+
+        Returns:
+            list[str]: The command to enable job reservation.
+        """
+        raise NotImplementedError()
+
     @property
     @abstractmethod
     def submit_job_command(self) -> list[str]:
@@ -41,16 +54,6 @@ def delete_job_command(self) -> list[str]:
         """
         pass
 
-    @property
-    def enable_reservation_command(self) -> list[str]:
-        """
-        Returns the command to enable job reservation on the scheduler.
-
-        Returns:
-            list[str]: The command to enable job reservation.
-        """
-        raise NotImplementedError()
-
     @property
     @abstractmethod
     def get_queue_status_command(self) -> list[str]:
@@ -62,6 +65,47 @@ def get_queue_status_command(self) -> list[str]:
         """
         pass
 
+    @staticmethod
+    def render_submission_template(
+        command: str,
+        submission_template: Union[str, Template],
+        job_name: str = "pysqa",
+        working_directory: str = os.path.abspath("."),
+        cores: int = 1,
+        memory_max: Optional[int] = None,
+        run_time_max: Optional[int] = None,
+        dependency_list: Optional[List[int]] = None,
+        **kwargs,
+    ) -> str:
+        """
+        Generate the job submission template.
+
+        Args:
+            command (str, optional): The command to be executed.
+            job_name (str, optional): The job name. Defaults to "pysqa".
+            working_directory (str, optional): The working directory. Defaults to ".".
+            cores (int, optional): The number of cores. Defaults to 1.
+            memory_max (int, optional): The maximum memory. Defaults to None.
+            run_time_max (int, optional): The maximum run time. Defaults to None.
+            dependency_list (list[int], optional): The list of dependency job IDs. Defaults to None.
+            submission_template (str): Submission script template pysqa.wrapper.torque.template
+
+        Returns:
+            str: The rendered job submission template.
+        """
+        if not isinstance(submission_template, Template):
+            submission_template = Template(submission_template)
+        return submission_template.render(
+            command=command,
+            job_name=job_name,
+            working_directory=working_directory,
+            cores=cores,
+            memory_max=memory_max,
+            run_time_max=run_time_max,
+            dependency_list=dependency_list,
+            **kwargs,
+        )
+
     @staticmethod
     def dependencies(dependency_list: list[str]) -> list:
         """

pysqa/wrapper/flux.py

@@ -1,8 +1,31 @@
 # coding: utf-8
+import os
+from typing import List, Optional, Union
+
 import pandas
 from flux.job import JobID
+from jinja2 import Template
+
+from pysqa.wrapper.abstract import SchedulerCommands
 
-from pysqa.wrapper.generic import SchedulerCommands
+template = """\
+#!/bin/bash
+# flux: --job-name={{job_name}}
+# flux: --env=CORES={{cores}}
+# flux: --output=time.out
+# flux: --error=error.out
+# flux: -n {{cores}}
+{%- if run_time_max %}
+# flux: -t {{ [1, run_time_max // 60]|max }}
+{%- endif %}
+{%- if dependency %}
+{%- for jobid in dependency %}
+# flux: --dependency=afterok:{{jobid}}
+{%- endfor %}
+{%- endif %}
+
+{{command}}
+"""
 
 
 class FluxCommands(SchedulerCommands):
@@ -51,3 +74,43 @@ def convert_queue_status(queue_status_output: str) -> pandas.DataFrame:
         df.loc[df.status == "C", "status"] = "error"
         df.loc[df.status == "CD", "status"] = "finished"
         return df
+
+    def render_submission_template(
+        self,
+        command: str,
+        job_name: str = "pysqa",
+        working_directory: str = os.path.abspath("."),
+        cores: int = 1,
+        memory_max: Optional[int] = None,
+        run_time_max: Optional[int] = None,
+        dependency_list: Optional[List[int]] = None,
+        submission_template: Union[str, Template] = template,
+        **kwargs,
+    ) -> str:
+        """
+        Generate the job submission template.
+
+        Args:
+            command (str, optional): The command to be executed.
+            job_name (str, optional): The job name. Defaults to "pysqa".
+            working_directory (str, optional): The working directory. Defaults to ".".
+            cores (int, optional): The number of cores. Defaults to 1.
+            memory_max (int, optional): The maximum memory. Defaults to None.
+            run_time_max (int, optional): The maximum run time. Defaults to None.
+            dependency_list (list[int], optional): The list of dependency job IDs. Defaults to None.
+            submission_template (str): Submission script template pysqa.wrapper.flux.template
+
+        Returns:
+            str: The rendered job submission template.
+        """
+        return super().render_submission_template(
+            command=command,
+            job_name=job_name,
+            working_directory=working_directory,
+            cores=cores,
+            memory_max=memory_max,
+            run_time_max=run_time_max,
+            dependency_list=dependency_list,
+            submission_template=submission_template,
+            **kwargs,
+        )

pysqa/wrapper/lsf.py

@@ -2,9 +2,13 @@
 # Copyright (c) Max-Planck-Institut für Eisenforschung GmbH - Computational Materials Design (CM) Department
 # Distributed under the terms of "New BSD License", see the LICENSE file.
 
+import os
+from typing import List, Optional, Union
+
 import pandas
+from jinja2 import Template
 
-from pysqa.wrapper.generic import SchedulerCommands
+from pysqa.wrapper.abstract import SchedulerCommands
 
 __author__ = "Jan Janssen"
 __copyright__ = (
@@ -18,6 +22,25 @@
 __date__ = "Feb 9, 2019"
 
 
+template = """\
+#!/bin/bash
+#BSUB -q queue
+#BSUB -J {{job_name}}
+#BSUB -o time.out
+#BSUB -n {{cores}}
+#BSUB -cwd {{working_directory}}
+#BSUB -e error.out
+{%- if run_time_max %}
+#BSUB -W {{run_time_max}}
+{%- endif %}
+{%- if memory_max %}
+#BSUB -M {{memory_max}}
+{%- endif %}
+
+{{command}}
+"""
+
+
 class LsfCommands(SchedulerCommands):
     @property
     def submit_job_command(self) -> list[str]:
@@ -63,3 +86,43 @@ def convert_queue_status(queue_status_output: str) -> pandas.DataFrame:
         df.loc[df.status == "RUN", "status"] = "running"
         df.loc[df.status == "PEND", "status"] = "pending"
         return df
+
+    def render_submission_template(
+        self,
+        command: str,
+        job_name: str = "pysqa",
+        working_directory: str = os.path.abspath("."),
+        cores: int = 1,
+        memory_max: Optional[int] = None,
+        run_time_max: Optional[int] = None,
+        dependency_list: Optional[List[int]] = None,
+        submission_template: Union[str, Template] = template,
+        **kwargs,
+    ) -> str:
+        """
+        Generate the job submission template.
+
+        Args:
+            command (str, optional): The command to be executed.
+            job_name (str, optional): The job name. Defaults to "pysqa".
+            working_directory (str, optional): The working directory. Defaults to ".".
+            cores (int, optional): The number of cores. Defaults to 1.
+            memory_max (int, optional): The maximum memory. Defaults to None.
+            run_time_max (int, optional): The maximum run time. Defaults to None.
+            dependency_list (list[int], optional): The list of dependency job IDs. Defaults to None.
+            submission_template (str): Submission script template pysqa.wrapper.flux.template
+
+        Returns:
+            str: The rendered job submission template.
+        """
+        return super().render_submission_template(
+            command=command,
+            job_name=job_name,
+            working_directory=working_directory,
+            cores=cores,
+            memory_max=memory_max,
+            run_time_max=run_time_max,
+            dependency_list=dependency_list,
+            submission_template=submission_template,
+            **kwargs,
+        )

pysqa/wrapper/moab.py

@@ -2,7 +2,12 @@
 # Copyright (c) Max-Planck-Institut für Eisenforschung GmbH - Computational Materials Design (CM) Department
 # Distributed under the terms of "New BSD License", see the LICENSE file.
 
-from pysqa.wrapper.generic import SchedulerCommands
+import os
+from typing import List, Optional, Union
+
+from jinja2 import Template
+
+from pysqa.wrapper.abstract import SchedulerCommands
 
 __author__ = "Jan Janssen"
 __copyright__ = (
@@ -16,6 +21,20 @@
 __date__ = "Feb 9, 2019"
 
 
+template = """\
+#!/bin/bash
+#MSUB -N {{job_name}}
+{%- if memory_max %}
+#MSUB -l pmem={{ memory_max| int }}gb
+{%- endif %}
+{%- if run_time_max %}
+#$ -l walltime={{run_time_max}}
+{%- endif %}
+
+{{command}}
+"""
+
+
 class MoabCommands(SchedulerCommands):
     @property
     def submit_job_command(self) -> list[str]:
@@ -46,3 +65,43 @@ def get_queue_status_command(self) -> list[str]:
             list[str]: The command to get the queue status.
         """
         return ["mdiag", "-x"]
+
+    def render_submission_template(
+        self,
+        command: str,
+        job_name: str = "pysqa",
+        working_directory: str = os.path.abspath("."),
+        cores: int = 1,
+        memory_max: Optional[int] = None,
+        run_time_max: Optional[int] = None,
+        dependency_list: Optional[List[int]] = None,
+        submission_template: Union[str, Template] = template,
+        **kwargs,
+    ) -> str:
+        """
+        Generate the job submission template.
+
+        Args:
+            command (str, optional): The command to be executed.
+            job_name (str, optional): The job name. Defaults to "pysqa".
+            working_directory (str, optional): The working directory. Defaults to ".".
+            cores (int, optional): The number of cores. Defaults to 1.
+            memory_max (int, optional): The maximum memory. Defaults to None.
+            run_time_max (int, optional): The maximum run time. Defaults to None.
+            dependency_list (list[int], optional): The list of dependency job IDs. Defaults to None.
+            submission_template (str): Submission script template pysqa.wrapper.flux.template
+
+        Returns:
+            str: The rendered job submission template.
+        """
+        return super().render_submission_template(
+            command=command,
+            job_name=job_name,
+            working_directory=working_directory,
+            cores=cores,
+            memory_max=memory_max,
+            run_time_max=run_time_max,
+            dependency_list=dependency_list,
+            submission_template=submission_template,
+            **kwargs,
+        )