diff --git a/pysqa/ext/modular.py b/pysqa/ext/modular.py index ef9ec83f..158797ba 100644 --- a/pysqa/ext/modular.py +++ b/pysqa/ext/modular.py @@ -63,13 +63,14 @@ def submit_job( cores=cores, memory_max=memory_max, run_time_max=run_time_max, + dependency_list=dependency_list, command=command, **kwargs, ) cluster_module = self._queue_to_cluster_dict[queue] commands = self._switch_cluster_command( cluster_module=cluster_module - ) + self._list_command_to_be_executed(dependency_list, queue_script_path) + ) + self._list_command_to_be_executed(queue_script_path=queue_script_path) out = self._execute_command( commands=commands, working_directory=working_directory, diff --git a/pysqa/utils/basic.py b/pysqa/utils/basic.py index a88c4263..04a58747 100644 --- a/pysqa/utils/basic.py +++ b/pysqa/utils/basic.py @@ -5,7 +5,7 @@ import importlib import os import re -from typing import Optional +from typing import Optional, List import pandas from jinja2 import Template @@ -178,7 +178,7 @@ def submit_job( ) out = self._execute_command( commands=self._list_command_to_be_executed( - dependency_list, queue_script_path + queue_script_path=queue_script_path ), working_directory=working_directory, split_output=False, @@ -188,14 +188,8 @@ def submit_job( else: return None - def _list_command_to_be_executed( - self, dependency_list: list[str], queue_script_path: str - ) -> list: - return ( - self._commands.submit_job_command - + self._commands.dependencies(dependency_list) - + [queue_script_path] - ) + def _list_command_to_be_executed(self, queue_script_path: str) -> list: + return self._commands.submit_job_command + [queue_script_path] def enable_reservation(self, process_id: int): """ @@ -354,6 +348,7 @@ def _write_queue_script( cores: Optional[int] = None, memory_max: Optional[int] = None, run_time_max: Optional[int] = None, + dependency_list: Optional[List[int]] = None, command: Optional[str] = None, **kwargs, ): @@ -380,6 +375,7 @@ def _write_queue_script( cores=cores, memory_max=memory_max, run_time_max=run_time_max, + dependency_list=dependency_list, command=command, **kwargs, ) @@ -398,6 +394,7 @@ def _job_submission_template( cores: Optional[int] = None, memory_max: Optional[int] = None, run_time_max: Optional[int] = None, + dependency_list: Optional[List[int]] = None, command: Optional[str] = None, **kwargs, ) -> str: @@ -410,6 +407,7 @@ def _job_submission_template( cores (int/None): memory_max (int/None): run_time_max (int/None): + dependency_list (list/None): command (str/None): Returns: @@ -441,6 +439,7 @@ def _job_submission_template( memory_max=memory_max, run_time_max=run_time_max, command=command, + dependency_list=dependency_list, **kwargs, ) diff --git a/tests/config/flux/flux.sh b/tests/config/flux/flux.sh index 928e3ba1..7f4bf39f 100644 --- a/tests/config/flux/flux.sh +++ b/tests/config/flux/flux.sh @@ -7,5 +7,10 @@ {%- if run_time_max %} # flux: -t {{ [1, run_time_max // 60]|max }} {%- endif %} +{%- if dependency %} +{%- for jobid in dependency %} +# flux: --dependency=afterok:{{jobid}} +{%- endfor %} +{%- endif %} {{command}} \ No newline at end of file diff --git a/tests/config/slurm/slurm.sh b/tests/config/slurm/slurm.sh index e5bcdc04..1cb6188c 100644 --- a/tests/config/slurm/slurm.sh +++ b/tests/config/slurm/slurm.sh @@ -7,6 +7,9 @@ {%- if run_time_max %} #SBATCH --time={{ [1, run_time_max // 60]|max }} {%- endif %} +{%- if dependency %} +#SBATCH --dependency=afterok:{{ dependency | join(',') }} +{%- endif %} {%- if memory_max %} #SBATCH --mem={{memory_max}}G {%- endif %}