-
Notifications
You must be signed in to change notification settings - Fork 0
/
slurm.mk
136 lines (115 loc) · 4.9 KB
/
slurm.mk
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
# -*-makefile-*-
# enable e-mail notification by setting EMAIL
WHOAMI = $(shell whoami)
ifeq ("$(WHOAMI)","tiedeman")
EMAIL = [email protected]
endif
##---------------------------------------------
## submit jobs
##---------------------------------------------
## submit job to gpu queue
##
## default resources for GPU jobs
## (most of them relate to CPU resources like MEM, CORES, ...)
## typically we model single node jobs, which can still have multiple GPUs!
GPUJOB_HPC_QUEUE ?= ${HPC_GPUQUEUE}
GPUJOB_HPC_MEM ?= 32g
GPUJOB_HPC_NODES ?= 1
GPUJOB_HPC_CORES ?= 1
GPUJOB_HPC_THREADS ?= ${GPUJOB_HPC_CORES}
GPUJOB_HPC_JOBS ?= ${GPUJOB_HPC_THREADS}
GPUJOB_HPC_TIME ?= ${HPC_TIME}
## exclude broken nodes:
## list comma separated nodes to be excluded
# BROKEN_NODES = g6301
ifdef WORK_DIR
WORKDIR ?= ${WORK_DIR}
else
WORKDIR ?= work
endif
%.submit:
mkdir -p ${WORKDIR}
mkdir -p ${dir ${TMPWORKDIR}/$@}
echo '#!/bin/bash -l' > ${TMPWORKDIR}/$@
echo '#SBATCH -J "${@:.submit=}"' >> ${TMPWORKDIR}/$@
echo '#SBATCH -o ${@:.submit=}.out.%j' >> ${TMPWORKDIR}/$@
echo '#SBATCH -e ${@:.submit=}.err.%j' >> ${TMPWORKDIR}/$@
ifdef EMAIL
echo '#SBATCH --mail-type=END' >> ${TMPWORKDIR}/$@
echo '#SBATCH --mail-user=${EMAIL}' >> ${TMPWORKDIR}/$@
endif
echo '#SBATCH --mem=${GPUJOB_HPC_MEM}' >> ${TMPWORKDIR}/$@
echo '#SBATCH -n ${GPUJOB_HPC_CORES}' >> ${TMPWORKDIR}/$@
echo '#SBATCH -N ${GPUJOB_HPC_NODES}' >> ${TMPWORKDIR}/$@
echo '#SBATCH -t ${GPUJOB_HPC_TIME}:00' >> ${TMPWORKDIR}/$@
echo '#SBATCH -p ${GPUJOB_HPC_QUEUE}' >> ${TMPWORKDIR}/$@
echo '#SBATCH ${HPC_GPU_ALLOCATION}' >> ${TMPWORKDIR}/$@
ifdef BROKEN_NODES
echo '#SBATCH --exclude=${BROKEN_NODES}' >> ${TMPWORKDIR}/$@
endif
echo '${HPC_EXTRA}' >> ${TMPWORKDIR}/$@
echo '${HPC_EXTRA1}' >> ${TMPWORKDIR}/$@
echo '${HPC_EXTRA2}' >> ${TMPWORKDIR}/$@
echo '${HPC_EXTRA3}' >> ${TMPWORKDIR}/$@
echo '${HPC_GPU_EXTRA1}' >> ${TMPWORKDIR}/$@
echo '${HPC_GPU_EXTRA2}' >> ${TMPWORKDIR}/$@
echo '${HPC_GPU_EXTRA3}' >> ${TMPWORKDIR}/$@
echo '${LOAD_GPU_ENV}' >> ${TMPWORKDIR}/$@
echo 'cd $${SLURM_SUBMIT_DIR:-.}' >> ${TMPWORKDIR}/$@
echo 'pwd' >> ${TMPWORKDIR}/$@
echo 'echo "Starting at `date`"' >> ${TMPWORKDIR}/$@
echo '${MAKE} -j ${GPUJOB_HPC_JOBS} ${MAKEARGS} ${@:.submit=}' >> ${TMPWORKDIR}/$@
echo 'echo "Finishing at `date`"' >> ${TMPWORKDIR}/$@
echo 'seff $$SLURM_JOBID' >> ${TMPWORKDIR}/$@
sbatch ${SBATCH_ARGS} ${TMPWORKDIR}/$@
mkdir -p ${WORKDIR}
-mv ${TMPWORKDIR}/$@ ${WORKDIR}/$@
# echo "echo ''" >> ${TMPWORKDIR}/$@
# echo "Makefile variables:" >> ${TMPWORKDIR}/$@
# echo "${MAKE} print-makefile-variables" >> ${TMPWORKDIR}/$@
# echo "echo ''" >> ${TMPWORKDIR}/$@
## submit job to cpu queue
## copy resources to CPUjob-specific variables
CPUJOB_HPC_QUEUE ?= ${HPC_QUEUE}
CPUJOB_HPC_MEM ?= ${HPC_MEM}
CPUJOB_HPC_NODES ?= ${HPC_NODES}
CPUJOB_HPC_TIME ?= ${HPC_TIME}
CPUJOB_HPC_CORES ?= ${HPC_CORES}
CPUJOB_HPC_THREADS ?= ${CPUJOB_HPC_CORES}
CPUJOB_HPC_JOBS ?= ${CPUJOB_HPC_THREADS}
%.submitcpu:
mkdir -p ${WORKDIR}
mkdir -p ${dir ${TMPWORKDIR}/$@}
echo '#!/bin/bash -l' > ${TMPWORKDIR}/$@
echo '#SBATCH -J "${@:.submitcpu=}"' >>${TMPWORKDIR}/$@
echo '#SBATCH -o ${@:.submitcpu=}.out.%j' >> ${TMPWORKDIR}/$@
echo '#SBATCH -e ${@:.submitcpu=}.err.%j' >> ${TMPWORKDIR}/$@
ifdef EMAIL
echo '#SBATCH --mail-type=END' >> ${TMPWORKDIR}/$@
echo '#SBATCH --mail-user=${EMAIL}' >> ${TMPWORKDIR}/$@
endif
echo '#SBATCH --mem=${CPUJOB_HPC_MEM}' >> ${TMPWORKDIR}/$@
echo '#SBATCH -n ${CPUJOB_HPC_CORES}' >> ${TMPWORKDIR}/$@
echo '#SBATCH -N ${CPUJOB_HPC_NODES}' >> ${TMPWORKDIR}/$@
echo '#SBATCH -p ${CPUJOB_HPC_QUEUE}' >> ${TMPWORKDIR}/$@
echo '#SBATCH -t ${CPUJOB_HPC_TIME}:00' >> ${TMPWORKDIR}/$@
ifdef BROKEN_NODES
echo '#SBATCH --exclude=${BROKEN_NODES}' >> ${TMPWORKDIR}/$@
endif
echo '${HPC_EXTRA}' >> ${TMPWORKDIR}/$@
echo '${HPC_EXTRA1}' >> ${TMPWORKDIR}/$@
echo '${HPC_EXTRA2}' >> ${TMPWORKDIR}/$@
echo '${HPC_EXTRA3}' >> ${TMPWORKDIR}/$@
echo '${HPC_CPU_EXTRA1}' >> ${TMPWORKDIR}/$@
echo '${HPC_CPU_EXTRA2}' >> ${TMPWORKDIR}/$@
echo '${HPC_CPU_EXTRA3}' >> ${TMPWORKDIR}/$@
echo '${LOAD_CPU_ENV}' >> ${TMPWORKDIR}/$@
echo 'cd $${SLURM_SUBMIT_DIR:-.}' >> ${TMPWORKDIR}/$@
echo 'pwd' >> ${TMPWORKDIR}/$@
echo 'echo "Starting at `date`"' >> ${TMPWORKDIR}/$@
echo '${MAKE} -j ${CPUJOB_HPC_JOBS} ${MAKEARGS} ${@:.submitcpu=}' >> ${TMPWORKDIR}/$@
echo 'echo "Finishing at `date`"' >> ${TMPWORKDIR}/$@
echo 'seff $$SLURM_JOBID' >> ${TMPWORKDIR}/$@
sbatch ${SBATCH_ARGS} ${TMPWORKDIR}/$@
mkdir -p ${WORKDIR}
-mv ${TMPWORKDIR}/$@ ${WORKDIR}/$@