diff --git a/.gitignore b/.gitignore
index 60febdeb4..59c551707 100644
--- a/.gitignore
+++ b/.gitignore
@@ -37,3 +37,4 @@ dpgen.egg-info
.eggs
.coverage
dbconfig.json
+.vscode/*
\ No newline at end of file
diff --git a/.travis.yml b/.travis.yml
index b0963bb18..bad74118a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,6 +14,6 @@ install:
- pip install .
# command to run tests
script:
- - coverage run --source=./dpgen -m unittest && coverage report
+ - coverage run --source=./dpgen -m unittest -v && coverage report
after_success:
- codecov
diff --git a/README.md b/README.md
index d9fed8fe8..044ef0cf0 100644
--- a/README.md
+++ b/README.md
@@ -21,9 +21,14 @@
## About DP-GEN
[![GitHub release](https://img.shields.io/github/release/deepmodeling/dpgen.svg?maxAge=86400)](https://github.com/deepmodeling/dpgen/releases/)
-[![arxiv:1910.12690](http://img.shields.io/badge/arXiv-1910.12690-B31B1B.svg?maxAge=86400)](https://arxiv.org/abs/1910.12690)
+[![doi:10.1016/j.cpc.2020.107206](https://zenodo.org/badge/DOI/10.1016/j.cpc.2020.107206.svg)](https://doi.org/10.1016/j.cpc.2020.107206)
+
+DP-GEN (Deep Generator) is a software written in Python, delicately designed to generate a deep learning based model of interatomic potential energy and force field. DP-GEN is depedent on DeepMD-kit (https://github.com/deepmodeling/deepmd-kit/blob/master/README.md). With highly scalable interface with common softwares for molecular simulation, DP-GEN is capable to automatically prepare scripts and maintain job queues on HPC machines (High Performance Cluster) and analyze results.
+
+If you use this software in any publication, please cite:
+
+Yuzhi Zhang, Haidi Wang, Weijie Chen, Jinzhe Zeng, Linfeng Zhang, Han Wang, and Weinan E, DP-GEN: A concurrent learning platform for the generation of reliable deep learning based potential energy models, Computer Physics Communications, 2020, 107206.
-DP-GEN (Deep Generator) is a software written in Python, delicately designed to generate a deep learning based model of interatomic potential energy and force field. DP-GEN is depedent on DeepMD-kit (https://github.com/deepmodeling/deepmd-kit/blob/master/README.md). With highly scalable interface with common softwares for molecular simulation, DP-GEN is capable to automatically prepare scripts and maintain job queues on HPC machines (High Performance Cluster) and analyze results
### Highlighted features
+ **Accurate and efficient**: DP-GEN is capable to sample more than tens of million structures and select only a few for first principles calculation. DP-GEN will finally obtain a uniformly accurate model.
+ **User-friendly and automatic**: Users may install and run DP-GEN easily. Once succusefully running, DP-GEN can dispatch and handle all jobs on HPCs, and thus there's no need for any personal effort.
@@ -56,7 +61,6 @@ Options for TASK:
* `test`: Auto-test for Deep Potential.
* `db`: Collecting data from DP-GEN.
-
## Download and Install
One can download the source code of dpgen by
```bash
@@ -264,6 +268,52 @@ Following is an example for `PARAM`, which generates data from a typical structu
}
```
+Another example is `from_poscar` method. Here you need to specify the POSCAR file.
+
+```
+{
+ "stages": [
+ 1,
+ 2
+ ],
+ "cell_type": "fcc",
+ "from_poscar": true,
+ "from_poscar_path": "POSCAR",
+ "super_cell": [
+ 1,
+ 1,
+ 1
+ ],
+ "layer_numb": 3,
+ "vacuum_max": 5,
+ "vacuum_resol": [0.5,2],
+ "mid_point": 2.0,
+ "millers": [
+ [
+ 1,
+ 0,
+ 0
+ ]
+ ],
+ "elements": [
+ "Al"
+ ],
+ "potcars": [
+ "./POTCAR"
+ ],
+ "relax_incar" : "INCAR_metal_rlx_low",
+ "scale": [
+ 1.0
+ ],
+ "skip_relax": true,
+ "pert_numb": 5,
+ "pert_box": 0.03,
+ "pert_atom": 0.01,
+ "coll_ndata": 5000,
+ "_comment": "that's all"
+}
+```
+
The following table gives explicit descriptions on keys in `PARAM`.
The bold notation of key (such as **Elements**) means that it's a necessary key.
@@ -275,7 +325,9 @@ The bold notation of key (such as **Elements**) means that it's a necessary key.
| **cell_type** | String | "hcp" | Specifying which typical structure to be generated. **Options** include fcc, hcp, bcc, sc, diamond.
| **latt** | Float | 4.479 | Lattice constant for single cell.
| **layer_numb** | Integer | 3 | Number of equavilent layers of slab.
+| **z__min** | Float | 9.0 | Thickness of slab without vacuum (Angstrom). If the `layer_numb` and `z_min` are all setted, the `z_min` value will be ignored.
| **vacuum_max** | Float | 9 | Maximal thickness of vacuum (Angstrom).
+| vacuum_min | Float | 3.0 | Minimal thickness of vacuum (Angstrom). Default value is 2 times atomic radius.
| **vacuum_resol** | List of float | [0.5, 1 ] | Interval of thichness of vacuum. If size of `vacuum_resol` is 1, the interval is fixed to its value. If size of `vacuum_resol` is 2, the interval is `vacuum_resol[0]` before `mid_point`, otherwise `vacuum_resol[1]` after `mid_point`.
| **millers** | List of list of Integer | [[1,0,0]] | Miller indices.
| relax_incar | String | "....../INCAR" | Path of INCAR for relaxation in VASP. **Necessary** if `stages` include 1.
@@ -466,6 +518,8 @@ The bold notation of key (such aas **type_map**) means that it's a necessary key
| **sys_batch_size** | List of integer | [8, 8] | Each number is the batch_size for training of corresponding system in `sys_configs`. If set to `auto`, batch size will be 32 divided by number of atoms. |
| *#Training*
| **numb_models** | Integer | 4 (recommend) | Number of models to be trained in `00.train`. |
+| training_iter0_model_path | list of string | ["/path/to/model0_ckpt/", ...] | The model used to init the first iter training. Number of element should be equal to `numb_models` |
+| training_init_model | bool | False | Iteration > 0, the model parameters will be initilized from the model trained at the previous iteration. Iteration == 0, the model parameters will be initialized from `training_iter0_model_path`. |
| **default_training_param** | Dict | {
...
"use_smooth": true,
"sel_a": [16, 4],
"rcut_smth": 0.5,
"rcut": 5,
"filter_neuron": [10, 20, 40],
...
} | Training parameters for `deepmd-kit` in `00.train`.
You can find instructions from here: (https://github.com/deepmodeling/deepmd-kit)..
We commonly let `stop_batch` = 200 * `decay_steps`. |
| *#Exploration*
| **model_devi_dt** | Float | 0.002 (recommend) | Timestep for MD |
@@ -475,6 +529,8 @@ The bold notation of key (such aas **type_map**) means that it's a necessary key
| **model_devi_e_trust_lo** | Float | 1e10 | Lower bound of energies for the selection. Recommend to set them a high number, since forces provide more precise information. Special cases such as energy minimization may need this. |
| **model_devi_e_trust_hi** | Float | 1e10 | Upper bound of energies for the selection. |
| **model_devi_clean_traj** | Boolean | true | Deciding whether to clean traj folders in MD since they are too large. |
+| **model_devi_nopbc** | Boolean | False | Assume open boundary condition in MD simulations. |
+| model_devi_activation_func | List of String | ["tanh", "tanh", "tanh", "tanh"] | Set activation functions for models, length of the list should be the same as `numb_models` |
| **model_devi_jobs** | [
{
"sys_idx": [0],
"temps":
[100],
"press":
[1],
"trj_freq":
10,
"nsteps":
1000,
"ensembles":
"nvt"
},
...
] | List of dict | Settings for exploration in `01.model_devi`. Each dict in the list corresponds to one iteration. The index of `model_devi_jobs` exactly accord with index of iterations |
| **model_devi_jobs["sys_idx"]** | List of integer | [0] | Systems to be selected as the initial structure of MD and be explored. The index corresponds exactly to the `sys_configs`. |
| **model_devi_jobs["temps"]** | List of integer | [50, 300] | Temperature (**K**) in MD
@@ -489,10 +545,13 @@ The bold notation of key (such aas **type_map**) means that it's a necessary key
| **fp_style** | string | "vasp" | Software for First Principles. **Options** include “vasp”, “pwscf”, “siesta” and “gaussian” up to now. |
| **fp_task_max** | Integer | 20 | Maximum of structures to be calculated in `02.fp` of each iteration. |
| **fp_task_min** | Integer | 5 | Minimum of structures to calculate in `02.fp` of each iteration. |
+| fp_accurate_threshold | Float | 0.9999 | If the accurate ratio is larger than this number, no fp calculation will be performed, i.e. fp_task_max = 0. |
+| fp_accurate_soft_threshold | Float | 0.9999 | If the accurate ratio is between this number and `fp_accurate_threshold`, the fp_task_max linearly decays to zero. |
| *fp_style == VASP*
| **fp_pp_path** | String | "/sharedext4/.../ch4/" | Directory of psuedo-potential file to be used for 02.fp exists. |
| **fp_pp_files** | List of string | ["POTCAR"] | Psuedo-potential file to be used for 02.fp. Note that the order of elements should correspond to the order in `type_map`. |
-|**fp_incar** | String | "/sharedext4/../ch4/INCAR" | Input file for VASP. INCAR must specify KSPACING.
+|**fp_incar** | String | "/sharedext4/../ch4/INCAR" | Input file for VASP. INCAR must specify KSPACING and KGAMMA.
+|**fp_aniso_kspacing** | List of integer | [1.0,1.0,1.0] | Set anisotropic kspacing. Usually useful for 1-D or 2-D materials. Only support VASP. If it is setting the KSPACING key in INCAR will be ignored.
|cvasp| Boolean | true | If `cvasp` is true, DP-GEN will use Custodian to help control VASP calculation.
| *fp_style == Gaussian*
| **use_clusters** | Boolean | false | If set to `true`, clusters will be taken instead of the whole system. This option does not work with DeePMD-kit 0.x.
@@ -632,7 +691,7 @@ param.json in a dictionary.
| conf_dir | path like string | "confs/Al/std-fcc" | the dir which contains vasp's POSCAR |
| key_id | string| "DZIwdXCXg1fiXXXXXX" |the API key of Material project|
| task_type | string | "vasp" | task type, one of deepmd vasp meam |
-| task | string | "equi" | task, one of equi, eos, elastic, vacancy, interstitial, surf or all |
+| task | string or list | "equi" | task, one or several tasks from { equi, eos, elastic, vacancy, interstitial, surf } or all stands for all tasks |
| vasp_params| dict | seeing below | params relating to vasp INCAR|
| lammps_params | dict| seeing below| params relating to lammps |
diff --git a/dpgen/__init__.py b/dpgen/__init__.py
index d77ec3898..b38875dee 100644
--- a/dpgen/__init__.py
+++ b/dpgen/__init__.py
@@ -45,3 +45,13 @@ def info():
except ImportError:
print('%10s %10s Not Found' % (modui, ''))
print()
+
+ # reference
+ print("""Reference
+------------
+Please cite:
+Yuzhi Zhang, Haidi Wang, Weijie Chen, Jinzhe Zeng, Linfeng Zhang, Han Wang, and Weinan E,
+DP-GEN: A concurrent learning platform for the generation of reliable deep learning
+based potential energy models, Computer Physics Communications, 2020, 107206.
+------------
+""")
diff --git a/dpgen/auto_test/cmpt_01_eos.py b/dpgen/auto_test/cmpt_01_eos.py
index f5670c422..385475059 100755
--- a/dpgen/auto_test/cmpt_01_eos.py
+++ b/dpgen/auto_test/cmpt_01_eos.py
@@ -12,7 +12,7 @@ def comput_lmp_eos(jdata,conf_dir, task_name) :
conf_path = os.path.abspath(conf_path)
conf_path = os.path.join(conf_path, task_name)
vol_paths = glob.glob(os.path.join(conf_path, 'vol-*'))
- vol_paths.sort()
+ vol_paths.sort(key=lambda k : float(k.split('-')[-1]))
result = os.path.join(conf_path,'result')
print('Vpa(A^3)\tEpA(eV)')
with open(result,'w') as fp:
@@ -38,7 +38,7 @@ def comput_vasp_eos(jdata, conf_dir) :
vasp_str='vasp-k%.2f' % kspacing
task_path = os.path.join(conf_path, vasp_str)
vol_paths = glob.glob(os.path.join(task_path, 'vol-*'))
- vol_paths.sort()
+ vol_paths.sort(key=lambda k : float(k.split('-')[-1]))
result = os.path.join(task_path,'result')
print('Vpa(A^3)\tEpA(eV)')
with open(result,'w') as fp:
diff --git a/dpgen/auto_test/gen_00_equi.py b/dpgen/auto_test/gen_00_equi.py
index 59c1ddfdb..a2b5b00c9 100755
--- a/dpgen/auto_test/gen_00_equi.py
+++ b/dpgen/auto_test/gen_00_equi.py
@@ -1,11 +1,19 @@
#!/usr/bin/env python3
-import os, re, argparse, filecmp, json, glob
+import os, re, argparse, filecmp, json, glob, shutil
import subprocess as sp
import numpy as np
import dpgen.auto_test.lib.vasp as vasp
import dpgen.auto_test.lib.lammps as lammps
+from dpgen import dlog
+from dpgen.generator.lib.vasp import incar_upper
+from dpgen import ROOT_PATH
+from pymatgen.io.vasp import Incar
+from dpgen.generator.lib.vasp import incar_upper
+
+cvasp_file=os.path.join(ROOT_PATH,'generator/lib/cvasp.py')
+
global_task_name = '00.equi'
'''
@@ -48,7 +56,14 @@ def make_vasp(jdata, conf_dir) :
relax_incar_path = jdata['relax_incar']
assert(os.path.exists(relax_incar_path))
relax_incar_path = os.path.abspath(relax_incar_path)
- fc = open(relax_incar_path).read()
+ incar = incar_upper(Incar.from_file(relax_incar_path))
+ isif = 3
+ if incar.get('ISIF') != isif:
+ dlog.info("%s:%s setting ISIF to %d" % (__file__, make_vasp.__name__, isif))
+ incar['ISIF'] = isif
+ fc = incar.get_string()
+ kspacing = incar['KSPACING']
+ kgamma = incar['KGAMMA']
vasp_path = os.path.join(equi_path, 'vasp-relax_incar' )
else :
fp_params = jdata['vasp_params']
@@ -63,14 +78,24 @@ def make_vasp(jdata, conf_dir) :
os.makedirs(vasp_path, exist_ok = True)
os.chdir(vasp_path)
- print(vasp_path)
+ # write incar
with open('INCAR', 'w') as fp :
fp.write(fc)
+
# gen poscar
if os.path.exists('POSCAR') :
os.remove('POSCAR')
os.symlink(os.path.relpath(to_poscar), 'POSCAR')
+
+ # gen kpoints
+ fc = vasp.make_kspacing_kpoints('POSCAR', kspacing, kgamma)
+ with open('KPOINTS', 'w') as fp: fp.write(fc)
+
+ #copy cvasp
+ if ('cvasp' in jdata) and (jdata['cvasp'] == True):
+ shutil.copyfile(cvasp_file, 'cvasp.py')
+
# gen potcar
with open('POTCAR', 'w') as outfile:
for fname in potcar_list:
@@ -84,6 +109,7 @@ def make_lammps (jdata, conf_dir,task_type) :
type_map = fp_params['type_map']
model_dir = os.path.abspath(model_dir)
model_name =fp_params['model_name']
+ deepmd_version = fp_params.get("deepmd_version", "0.12")
if not model_name and task_type =='deepmd':
models = glob.glob(os.path.join(model_dir, '*pb'))
model_name = [os.path.basename(ii) for ii in models]
@@ -91,8 +117,9 @@ def make_lammps (jdata, conf_dir,task_type) :
else:
models = [os.path.join(model_dir,ii) for ii in model_name]
- model_param = {'model_name' : fp_params['model_name'],
- 'param_type': fp_params['model_param_type']}
+ model_param = {'model_name' : model_name,
+ 'param_type': fp_params['model_param_type'],
+ 'deepmd_version' : deepmd_version}
ntypes = len(type_map)
conf_path = os.path.abspath(conf_dir)
@@ -121,7 +148,7 @@ def make_lammps (jdata, conf_dir,task_type) :
fc = lammps.make_lammps_equi(os.path.basename(conf_file),
ntypes,
lammps.inter_deepmd,
- model_name)
+ model_param)
elif task_type=='meam':
fc = lammps.make_lammps_equi(os.path.basename(conf_file),
ntypes,
diff --git a/dpgen/auto_test/gen_01_eos.py b/dpgen/auto_test/gen_01_eos.py
index c129a4df3..ba9fae386 100755
--- a/dpgen/auto_test/gen_01_eos.py
+++ b/dpgen/auto_test/gen_01_eos.py
@@ -1,11 +1,18 @@
#!/usr/bin/env python3
-import os, re, argparse, filecmp, json, glob
+import os, re, argparse, filecmp, json, glob, shutil
import subprocess as sp
import numpy as np
import dpgen.auto_test.lib.vasp as vasp
import dpgen.auto_test.lib.lammps as lammps
+
+from dpgen import dlog
+from dpgen.generator.lib.vasp import incar_upper
from pymatgen.core.structure import Structure
+from pymatgen.io.vasp import Incar
+from dpgen import ROOT_PATH
+
+cvasp_file=os.path.join(ROOT_PATH,'generator/lib/cvasp.py')
global_equi_name = '00.equi'
global_task_name = '01.eos'
@@ -19,12 +26,26 @@ def make_vasp(jdata, conf_dir) :
vol_start = jdata['vol_start']
vol_end = jdata['vol_end']
vol_step = jdata['vol_step']
-
+ eos_relax_cell_shape = jdata.get('eos_relax_cell_shape', True)
conf_path = os.path.abspath(conf_dir)
+ conf_poscar = os.path.join(conf_path, 'POSCAR')
+
+ if 'relax_incar' in jdata.keys():
+ vasp_str='vasp-relax_incar'
+ else:
+ kspacing = jdata['vasp_params']['kspacing']
+ vasp_str='vasp-k%.2f' % kspacing
+
+ # get equi poscar
+ equi_path = re.sub('confs', global_equi_name, conf_path)
+ equi_path = os.path.join(equi_path, vasp_str)
+ equi_contcar = os.path.join(equi_path, 'CONTCAR')
task_path = re.sub('confs', global_task_name, conf_path)
- os.makedirs(task_path, exist_ok = True)
+ task_path = os.path.join(task_path, vasp_str)
+ os.makedirs(task_path, exist_ok = True)
+ # link poscar
cwd = os.getcwd()
- from_poscar = os.path.join(conf_path, 'POSCAR')
+ from_poscar = os.path.join(equi_contcar)
to_poscar = os.path.join(task_path, 'POSCAR')
if os.path.exists(to_poscar) :
assert(filecmp.cmp(from_poscar, to_poscar))
@@ -56,8 +77,17 @@ def make_vasp(jdata, conf_dir) :
relax_incar_path = jdata['relax_incar']
assert(os.path.exists(relax_incar_path))
relax_incar_path = os.path.abspath(relax_incar_path)
- fc = open(relax_incar_path).read()
- vasp_path = os.path.join(task_path, 'vasp-relax_incar' )
+ incar = incar_upper(Incar.from_file(relax_incar_path))
+ if eos_relax_cell_shape:
+ isif = 4
+ else:
+ isif = 2
+ if incar.get('ISIF') != isif:
+ dlog.info("%s:%s setting ISIF to %d" % (__file__, make_vasp.__name__, isif))
+ incar['ISIF'] = isif
+ fc = incar.get_string()
+ kspacing = incar['KSPACING']
+ kgamma = incar['KGAMMA']
else :
fp_params = jdata['vasp_params']
ecut = fp_params['ecut']
@@ -67,11 +97,8 @@ def make_vasp(jdata, conf_dir) :
kspacing = fp_params['kspacing']
kgamma = fp_params['kgamma']
fc = vasp.make_vasp_relax_incar(ecut, ediff, is_alloy, True, False, npar, kpar, kspacing, kgamma)
- vasp_path = os.path.join(task_path, 'vasp-k%.2f' % kspacing)
- os.makedirs(vasp_path, exist_ok = True)
- os.chdir(vasp_path)
- print(vasp_path)
+ os.chdir(task_path)
with open('INCAR', 'w') as fp :
fp.write(fc)
@@ -82,30 +109,37 @@ def make_vasp(jdata, conf_dir) :
outfile.write(infile.read())
# loop over volumes
for vol in np.arange(vol_start, vol_end, vol_step) :
- vol_path = os.path.join(vasp_path, 'vol-%.2f' % vol)
+ vol_path = os.path.join(task_path, 'vol-%.2f' % vol)
os.makedirs(vol_path, exist_ok = True)
os.chdir(vol_path)
- print(vol_path)
for ii in ['INCAR', 'POTCAR', 'POSCAR.orig', 'POSCAR'] :
if os.path.exists(ii) :
os.remove(ii)
# link incar, potcar
- os.symlink(os.path.relpath(os.path.join(vasp_path, 'INCAR')), 'INCAR')
- os.symlink(os.path.relpath(os.path.join(vasp_path, 'POTCAR')), 'POTCAR')
+ os.symlink(os.path.relpath(os.path.join(task_path, 'INCAR')), 'INCAR')
+ os.symlink(os.path.relpath(os.path.join(task_path, 'POTCAR')), 'POTCAR')
# gen poscar
os.symlink(os.path.relpath(to_poscar), 'POSCAR.orig')
scale = (vol / vol_to_poscar) ** (1./3.)
# print(scale)
vasp.poscar_scale('POSCAR.orig', 'POSCAR', scale)
# print(vol_path, vasp.poscar_vol('POSCAR') / vasp.poscar_natoms('POSCAR'))
+ # gen kpoints
+ fc = vasp.make_kspacing_kpoints('POSCAR', kspacing, kgamma)
+ with open('KPOINTS', 'w') as fp: fp.write(fc)
+ #copy cvasp
+ if ('cvasp' in jdata) and (jdata['cvasp'] == True):
+ shutil.copyfile(cvasp_file, os.path.join(vol_path,'cvasp.py'))
os.chdir(cwd)
+
def make_lammps (jdata, conf_dir,task_type) :
fp_params = jdata['lammps_params']
model_dir = fp_params['model_dir']
type_map = fp_params['type_map']
model_dir = os.path.abspath(model_dir)
model_name =fp_params['model_name']
+ deepmd_version = fp_params.get("deepmd_version", "0.12")
if not model_name and task_type =='deepmd':
models = glob.glob(os.path.join(model_dir, '*pb'))
model_name = [os.path.basename(ii) for ii in models]
@@ -113,9 +147,9 @@ def make_lammps (jdata, conf_dir,task_type) :
else:
models = [os.path.join(model_dir,ii) for ii in model_name]
- model_param = {'model_name' : fp_params['model_name'],
- 'param_type': fp_params['model_param_type']}
-
+ model_param = {'model_name' : model_name,
+ 'param_type': fp_params['model_param_type'],
+ 'deepmd_version' : deepmd_version}
ntypes = len(type_map)
vol_start = jdata['vol_start']
@@ -194,7 +228,7 @@ def make_lammps (jdata, conf_dir,task_type) :
# make lammps input
scale = (vol / vpa) ** (1./3.)
if task_type=='deepmd':
- fc = lammps.make_lammps_press_relax('conf.lmp', ntypes, scale,lammps.inter_deepmd, model_name)
+ fc = lammps.make_lammps_press_relax('conf.lmp', ntypes, scale,lammps.inter_deepmd, model_param)
elif task_type =='meam':
fc = lammps.make_lammps_press_relax('conf.lmp', ntypes, scale,lammps.inter_meam, model_param)
with open(os.path.join(vol_path, 'lammps.in'), 'w') as fp :
@@ -207,15 +241,16 @@ def make_lammps_fixv (jdata, conf_dir,task_type) :
type_map = fp_params['type_map']
model_dir = os.path.abspath(model_dir)
model_name =fp_params['model_name']
+ deepmd_version = fp_params.get("deepmd_version", "0.12")
if not model_name and task_type =='deepmd':
models = glob.glob(os.path.join(model_dir, '*pb'))
model_name = [os.path.basename(ii) for ii in models]
else:
models = [os.path.join(model_dir,ii) for ii in model_name]
- model_param = {'model_name' : fp_params['model_name'],
- 'param_type': fp_params['model_param_type']}
-
+ model_param = {'model_name' : model_name,
+ 'param_type': fp_params['model_param_type'],
+ 'deepmd_version' : deepmd_version}
ntypes = len(type_map)
@@ -246,7 +281,7 @@ def make_lammps_fixv (jdata, conf_dir,task_type) :
fc = lammps.make_lammps_equi('conf.lmp',
ntypes,
lammps.inter_deepmd,
- model_name,
+ model_param,
change_box = False)
elif task_type=='meam':
fc = lammps.make_lammps_equi('conf.lmp',
diff --git a/dpgen/auto_test/gen_02_elastic.py b/dpgen/auto_test/gen_02_elastic.py
index fa9ca9053..13b20a715 100755
--- a/dpgen/auto_test/gen_02_elastic.py
+++ b/dpgen/auto_test/gen_02_elastic.py
@@ -1,29 +1,38 @@
#!/usr/bin/env python3
-import os, re, argparse, filecmp, json, glob
+import os, re, argparse, filecmp, json, glob, shutil
import subprocess as sp
import numpy as np
import dpgen.auto_test.lib.vasp as vasp
import dpgen.auto_test.lib.lammps as lammps
+
+from dpgen import dlog
+from dpgen.generator.lib.vasp import incar_upper
from pymatgen.core.structure import Structure
from pymatgen.analysis.elasticity.strain import Deformation, DeformedStructureSet, Strain
+from pymatgen.io.vasp import Incar
+from dpgen import ROOT_PATH
+
+cvasp_file=os.path.join(ROOT_PATH,'generator/lib/cvasp.py')
global_equi_name = '00.equi'
global_task_name = '02.elastic'
-def make_vasp(jdata, conf_dir, norm_def = 2e-3, shear_def = 5e-3) :
- norm_def = jdata['norm_deform']
- shear_def = jdata['shear_deform']
+def make_vasp(jdata, conf_dir) :
+ default_norm_def = 2e-3
+ default_shear_def = 5e-3
+ norm_def = jdata.get('norm_deform', default_norm_def)
+ shear_def = jdata.get('shear_deform', default_shear_def)
conf_path = os.path.abspath(conf_dir)
conf_poscar = os.path.join(conf_path, 'POSCAR')
- # get equi poscar
if 'relax_incar' in jdata.keys():
vasp_str='vasp-relax_incar'
else:
kspacing = jdata['vasp_params']['kspacing']
vasp_str='vasp-k%.2f' % kspacing
+ # get equi poscar
equi_path = re.sub('confs', global_equi_name, conf_path)
equi_path = os.path.join(equi_path, vasp_str)
equi_contcar = os.path.join(equi_path, 'CONTCAR')
@@ -55,10 +64,14 @@ def make_vasp(jdata, conf_dir, norm_def = 2e-3, shear_def = 5e-3) :
if 'relax_incar' in jdata.keys():
relax_incar_path = jdata['relax_incar']
assert(os.path.exists(relax_incar_path))
- relax_incar_path = os.path.abspath(relax_incar_path)
- fc = open(relax_incar_path).read()
- kspacing =float(re.findall((r"KSPACING(.+?)\n"),fc)[0].replace('=',''))
- kgamma =('T' in re.findall((r"KGAMMA(.+?)\n"),fc)[0])
+ relax_incar_path = os.path.abspath(relax_incar_path)
+ incar = incar_upper(Incar.from_file(relax_incar_path))
+ if incar.get('ISIF') != 2:
+ dlog.info("%s:%s setting ISIF to 2" % (__file__, make_vasp.__name__))
+ incar['ISIF'] = 2
+ fc = incar.get_string()
+ kspacing = incar['KSPACING']
+ kgamma = incar['KGAMMA']
else :
fp_params = jdata['vasp_params']
ecut = fp_params['ecut']
@@ -107,7 +120,11 @@ def make_vasp(jdata, conf_dir, norm_def = 2e-3, shear_def = 5e-3) :
os.symlink(os.path.relpath(os.path.join(task_path, 'INCAR')), 'INCAR')
os.symlink(os.path.relpath(os.path.join(task_path, 'POTCAR')), 'POTCAR')
os.symlink(os.path.relpath(os.path.join(task_path, 'KPOINTS')), 'KPOINTS')
- cwd = os.getcwd()
+ #copy cvasp
+ if ('cvasp' in jdata) and (jdata['cvasp'] == True):
+ shutil.copyfile(cvasp_file, os.path.join(dfm_path,'cvasp.py'))
+ os.chdir(cwd)
+
def make_lammps(jdata, conf_dir,task_type) :
fp_params = jdata['lammps_params']
@@ -115,6 +132,7 @@ def make_lammps(jdata, conf_dir,task_type) :
type_map = fp_params['type_map']
model_dir = os.path.abspath(model_dir)
model_name =fp_params['model_name']
+ deepmd_version = fp_params.get("deepmd_version", "0.12")
if not model_name and task_type =='deepmd':
models = glob.glob(os.path.join(model_dir, '*pb'))
model_name = [os.path.basename(ii) for ii in models]
@@ -122,8 +140,9 @@ def make_lammps(jdata, conf_dir,task_type) :
else:
models = [os.path.join(model_dir,ii) for ii in model_name]
- model_param = {'model_name' : fp_params['model_name'],
- 'param_type': fp_params['model_param_type']}
+ model_param = {'model_name' : model_name,
+ 'param_type': fp_params['model_param_type'],
+ 'deepmd_version' : deepmd_version}
ntypes = len(type_map)
@@ -168,7 +187,7 @@ def make_lammps(jdata, conf_dir,task_type) :
fc = lammps.make_lammps_elastic('conf.lmp',
ntypes,
lammps.inter_deepmd,
- model_name)
+ model_param)
elif task_type=='meam':
fc = lammps.make_lammps_elastic('conf.lmp',
ntypes,
diff --git a/dpgen/auto_test/gen_03_vacancy.py b/dpgen/auto_test/gen_03_vacancy.py
index 93b061ea9..f50591b45 100755
--- a/dpgen/auto_test/gen_03_vacancy.py
+++ b/dpgen/auto_test/gen_03_vacancy.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
-import os, re, argparse, filecmp, json, glob
+import os, re, argparse, filecmp, json, glob, shutil
import subprocess as sp
import numpy as np
import dpgen.auto_test.lib.vasp as vasp
@@ -9,6 +9,12 @@
from pymatgen.analysis.defects.core import Vacancy
from pymatgen.analysis.defects.generators import VacancyGenerator
+from dpgen import ROOT_PATH
+from pymatgen.io.vasp import Incar
+from dpgen.generator.lib.vasp import incar_upper
+cvasp_file=os.path.join(ROOT_PATH,'generator/lib/cvasp.py')
+
+
global_equi_name = '00.equi'
global_task_name = '03.vacancy'
@@ -48,7 +54,10 @@ def make_vasp(jdata, conf_dir, supercell = [1,1,1]) :
relax_incar_path = jdata['relax_incar']
assert(os.path.exists(relax_incar_path))
relax_incar_path = os.path.abspath(relax_incar_path)
- fc = open(relax_incar_path).read()
+ incar = incar_upper(Incar.from_file(relax_incar_path))
+ fc = incar.get_string()
+ kspacing = incar['KSPACING']
+ kgamma = incar['KGAMMA']
else :
fp_params = jdata['vasp_params']
ecut = fp_params['ecut']
@@ -91,6 +100,14 @@ def make_vasp(jdata, conf_dir, supercell = [1,1,1]) :
os.symlink(os.path.relpath(os.path.join(task_path, 'POTCAR')), 'POTCAR')
# save supercell
np.savetxt('supercell.out', supercell, fmt='%d')
+
+ # write kp
+ fc = vasp.make_kspacing_kpoints('POSCAR', kspacing, kgamma)
+ with open('KPOINTS', 'w') as fp: fp.write(fc)
+
+ #copy cvasp
+ if ('cvasp' in jdata) and (jdata['cvasp'] == True):
+ shutil.copyfile(cvasp_file, os.path.join(struct_path,'cvasp.py'))
os.chdir(cwd)
def make_lammps(jdata, conf_dir, task_type, supercell) :
@@ -100,6 +117,7 @@ def make_lammps(jdata, conf_dir, task_type, supercell) :
type_map = fp_params['type_map']
model_dir = os.path.abspath(model_dir)
model_name =fp_params['model_name']
+ deepmd_version = fp_params.get("deepmd_version", "0.12")
if not model_name and task_type =='deepmd':
models = glob.glob(os.path.join(model_dir, '*pb'))
model_name = [os.path.basename(ii) for ii in models]
@@ -107,8 +125,9 @@ def make_lammps(jdata, conf_dir, task_type, supercell) :
else:
models = [os.path.join(model_dir,ii) for ii in model_name]
- model_param = {'model_name' : fp_params['model_name'],
- 'param_type': fp_params['model_param_type']}
+ model_param = {'model_name' : model_name,
+ 'param_type': fp_params['model_param_type'],
+ 'deepmd_version' : deepmd_version}
ntypes = len(type_map)
@@ -154,7 +173,7 @@ def make_lammps(jdata, conf_dir, task_type, supercell) :
ntypes,
1,
lammps.inter_deepmd,
- model_name)
+ model_param)
elif task_type =='meam':
fc = lammps.make_lammps_press_relax('conf.lmp',
ntypes,
diff --git a/dpgen/auto_test/gen_04_interstitial.py b/dpgen/auto_test/gen_04_interstitial.py
index 376be45c6..f1c10c4c3 100755
--- a/dpgen/auto_test/gen_04_interstitial.py
+++ b/dpgen/auto_test/gen_04_interstitial.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
-import os, re, argparse, filecmp, json, glob, warnings
+import os, re, argparse, filecmp, json, glob, warnings, shutil
import subprocess as sp
import numpy as np
import dpgen.auto_test.lib.vasp as vasp
@@ -9,6 +9,12 @@
from pymatgen.analysis.defects.core import Interstitial
from pymatgen.analysis.defects.generators import InterstitialGenerator
+from dpgen import ROOT_PATH
+from pymatgen.io.vasp import Incar
+from dpgen.generator.lib.vasp import incar_upper
+cvasp_file=os.path.join(ROOT_PATH,'generator/lib/cvasp.py')
+
+
global_equi_name = '00.equi'
global_task_name = '04.interstitial'
@@ -53,7 +59,11 @@ def _make_vasp(jdata, conf_dir, supercell, insert_ele) :
relax_incar_path = jdata['relax_incar']
assert(os.path.exists(relax_incar_path))
relax_incar_path = os.path.abspath(relax_incar_path)
- fc = open(relax_incar_path).read()
+ incar = incar_upper(Incar.from_file(relax_incar_path))
+ fc = incar.get_string()
+ kspacing = incar['KSPACING']
+ kgamma = incar['KGAMMA']
+
else :
fp_params = jdata['vasp_params']
ecut = fp_params['ecut']
@@ -100,6 +110,14 @@ def _make_vasp(jdata, conf_dir, supercell, insert_ele) :
os.symlink(os.path.relpath(os.path.join(task_path, 'INCAR')), 'INCAR')
# save supercell
np.savetxt('supercell.out', supercell, fmt='%d')
+
+ # write kp
+ fc = vasp.make_kspacing_kpoints('POSCAR', kspacing, kgamma)
+ with open('KPOINTS', 'w') as fp: fp.write(fc)
+
+ #copy cvasp
+ if ('cvasp' in jdata) and (jdata['cvasp'] == True):
+ shutil.copyfile(cvasp_file, os.path.join(struct_path,'cvasp.py'))
os.chdir(cwd)
@@ -114,6 +132,7 @@ def _make_reprod_traj(jdata, conf_dir, supercell, insert_ele, task_type) :
type_map = fp_params['type_map']
model_dir = os.path.abspath(model_dir)
model_name =fp_params['model_name']
+ deepmd_version = fp_params.get("deepmd_version", "0.12")
if not model_name and task_type=='deepmd':
models = glob.glob(os.path.join(model_dir, '*pb'))
model_name = [os.path.basename(ii) for ii in models]
@@ -121,8 +140,9 @@ def _make_reprod_traj(jdata, conf_dir, supercell, insert_ele, task_type) :
else:
models = [os.path.join(model_dir,ii) for ii in model_name]
- model_param = {'model_name' : fp_params['model_name'],
- 'param_type': fp_params['model_param_type']}
+ model_param = {'model_name' : model_name,
+ 'param_type': fp_params['model_param_type'],
+ 'deepmd_version' : deepmd_version}
ntypes = len(type_map)
@@ -152,7 +172,7 @@ def _make_reprod_traj(jdata, conf_dir, supercell, insert_ele, task_type) :
fc = lammps.make_lammps_eval('conf.lmp',
ntypes,
lammps.inter_deepmd,
- model_name)
+ model_param)
elif task_type =='meam':
fc = lammps.make_lammps_eval('conf.lmp',
ntypes,
@@ -238,6 +258,7 @@ def _make_lammps(jdata, conf_dir, supercell, insert_ele, task_type) :
type_map = fp_params['type_map']
model_dir = os.path.abspath(model_dir)
model_name =fp_params['model_name']
+ deepmd_version = fp_params.get("deepmd_version", "0.12")
if not model_name and task_type=='deepmd':
models = glob.glob(os.path.join(model_dir, '*pb'))
model_name = [os.path.basename(ii) for ii in models]
@@ -245,8 +266,9 @@ def _make_lammps(jdata, conf_dir, supercell, insert_ele, task_type) :
else:
models = [os.path.join(model_dir,ii) for ii in model_name]
- model_param = {'model_name' : fp_params['model_name'],
- 'param_type': fp_params['model_param_type']}
+ model_param = {'model_name' : model_name,
+ 'param_type': fp_params['model_param_type'],
+ 'deepmd_version' : deepmd_version}
ntypes = len(type_map)
@@ -291,7 +313,7 @@ def _make_lammps(jdata, conf_dir, supercell, insert_ele, task_type) :
ntypes,
1,
lammps.inter_deepmd,
- model_name)
+ model_param)
elif task_type =='meam':
fc = lammps.make_lammps_press_relax('conf.lmp',
ntypes,
diff --git a/dpgen/auto_test/gen_05_surf.py b/dpgen/auto_test/gen_05_surf.py
index 56a275630..ee6cd662d 100755
--- a/dpgen/auto_test/gen_05_surf.py
+++ b/dpgen/auto_test/gen_05_surf.py
@@ -1,12 +1,18 @@
#!/usr/bin/env python3
-import os, re, argparse, filecmp, json, glob
+import os, re, argparse, filecmp, json, glob, shutil
import subprocess as sp
import numpy as np
import dpgen.auto_test.lib.vasp as vasp
import dpgen.auto_test.lib.lammps as lammps
from pymatgen.core.surface import generate_all_slabs, Structure
+from dpgen import ROOT_PATH
+from pymatgen.io.vasp import Incar
+from dpgen.generator.lib.vasp import incar_upper
+cvasp_file=os.path.join(ROOT_PATH,'generator/lib/cvasp.py')
+
+
global_equi_name = '00.equi'
global_task_name = '05.surf'
@@ -59,7 +65,10 @@ def make_vasp(jdata, conf_dir, max_miller = 2, relax_box = False, static = False
scf_incar_path = jdata['scf_incar']
assert(os.path.exists(scf_incar_path))
scf_incar_path = os.path.abspath(scf_incar_path)
- fc = open(scf_incar_path).read()
+ incar = incar_upper(Incar.from_file(scf_incar_path))
+ fc = incar.get_string()
+ kspacing = incar['KSPACING']
+ kgamma = incar['KGAMMA']
else :
fp_params = jdata['vasp_params']
ecut = fp_params['ecut']
@@ -74,7 +83,10 @@ def make_vasp(jdata, conf_dir, max_miller = 2, relax_box = False, static = False
relax_incar_path = jdata['relax_incar']
assert(os.path.exists(relax_incar_path))
relax_incar_path = os.path.abspath(relax_incar_path)
- fc = open(relax_incar_path).read()
+ incar = incar_upper(Incar.from_file(relax_incar_path))
+ fc = incar.get_string()
+ kspacing = incar['KSPACING']
+ kgamma = incar['KGAMMA']
else :
fp_params = jdata['vasp_params']
ecut = fp_params['ecut']
@@ -122,6 +134,14 @@ def make_vasp(jdata, conf_dir, max_miller = 2, relax_box = False, static = False
# link incar, potcar, kpoints
os.symlink(os.path.relpath(os.path.join(task_path, 'INCAR')), 'INCAR')
os.symlink(os.path.relpath(os.path.join(task_path, 'POTCAR')), 'POTCAR')
+
+ # write kp
+ fc = vasp.make_kspacing_kpoints('POSCAR', kspacing, kgamma)
+ with open('KPOINTS', 'w') as fp: fp.write(fc)
+
+ #copy cvasp
+ if ('cvasp' in jdata) and (jdata['cvasp'] == True):
+ shutil.copyfile(cvasp_file, os.path.join(struct_path,'cvasp.py'))
cwd = os.getcwd()
def make_lammps(jdata, conf_dir, max_miller = 2, static = False, relax_box = False, task_type = 'wrong-task') :
@@ -131,6 +151,7 @@ def make_lammps(jdata, conf_dir, max_miller = 2, static = False, relax_box = Fal
type_map = fp_params['type_map']
model_dir = os.path.abspath(model_dir)
model_name =fp_params['model_name']
+ deepmd_version = fp_params.get("deepmd_version", "0.12")
if not model_name and task_type=='deepmd':
models = glob.glob(os.path.join(model_dir, '*pb'))
model_name = [os.path.basename(ii) for ii in models]
@@ -138,8 +159,9 @@ def make_lammps(jdata, conf_dir, max_miller = 2, static = False, relax_box = Fal
else:
models = [os.path.join(model_dir,ii) for ii in model_name]
- model_param = {'model_name' : fp_params['model_name'],
- 'param_type': fp_params['model_param_type']}
+ model_param = {'model_name' : model_name,
+ 'param_type': fp_params['model_param_type'],
+ 'deepmd_version' : deepmd_version}
ntypes = len(type_map)
@@ -184,12 +206,12 @@ def make_lammps(jdata, conf_dir, max_miller = 2, static = False, relax_box = Fal
fc = lammps.make_lammps_eval('conf.lmp',
ntypes,
lammps.inter_deepmd,
- model_name)
+ model_param)
else :
fc = lammps.make_lammps_equi('conf.lmp',
ntypes,
lammps.inter_deepmd,
- model_name,
+ model_param,
change_box = relax_box)
elif task_type =='meam':
if static :
diff --git a/dpgen/auto_test/gen_06_phonon.py b/dpgen/auto_test/gen_06_phonon.py
index 2b1e2af14..09e9ef9b3 100644
--- a/dpgen/auto_test/gen_06_phonon.py
+++ b/dpgen/auto_test/gen_06_phonon.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
-import os, re, argparse, filecmp, json, glob
+import os, re, argparse, filecmp, json, glob, shutil
import subprocess as sp
import numpy as np
import dpgen.auto_test.lib.vasp as vasp
@@ -8,6 +8,10 @@
from phonopy.structure.atoms import PhonopyAtoms
import yaml
+from dpgen import ROOT_PATH
+from pymatgen.io.vasp import Incar
+from dpgen.generator.lib.vasp import incar_upper
+cvasp_file=os.path.join(ROOT_PATH,'generator/lib/cvasp.py')
global_equi_name = '00.equi'
@@ -117,9 +121,10 @@ def make_vasp(jdata, conf_dir) :
user_incar_path = jdata['user_incar']
assert(os.path.exists(user_incar_path))
user_incar_path = os.path.abspath(user_incar_path)
- fc = open(user_incar_path).read()
- kspacing =float(re.findall((r"KSPACING(.+?)\n"),fc)[0].replace('=',''))
- kgamma =('T' in re.findall((r"KGAMMA(.+?)\n"),fc)[0])
+ incar = incar_upper(Incar.from_file(user_incar_path))
+ fc = incar.get_string()
+ kspacing = incar['KSPACING']
+ kgamma = incar['KGAMMA']
else :
fp_params = jdata['vasp_params']
ecut = fp_params['ecut']
@@ -145,9 +150,17 @@ def make_vasp(jdata, conf_dir) :
with open(fname) as infile:
outfile.write(infile.read())
# gen kpoints
- fc = vasp.make_kspacing_kpoints(task_poscar, kspacing, kgamma)
- with open(os.path.join(task_path,'KPOINTS'), 'w') as fp:
- fp.write(fc)
+# fc = vasp.make_kspacing_kpoints(task_poscar, kspacing, kgamma)
+# with open(os.path.join(task_path,'KPOINTS'), 'w') as fp:
+# fp.write(fc)
+
+ # write kp
+ fc = vasp.make_kspacing_kpoints('POSCAR', kspacing, kgamma)
+ with open('KPOINTS', 'w') as fp: fp.write(fc)
+ #copy cvasp
+ if ('cvasp' in jdata) and (jdata['cvasp'] == True):
+ shutil.copyfile(cvasp_file, os.path.join(task_path,'cvasp.py'))
+
# gen band.conf
os.chdir(task_path)
with open('band.conf','w') as fp:
@@ -169,14 +182,16 @@ def make_lammps(jdata, conf_dir,task_type) :
type_map = fp_params['type_map']
model_dir = os.path.abspath(model_dir)
model_name =fp_params['model_name']
+ deepmd_version = fp_params.get("deepmd_version", "0.12")
if not model_name :
models = glob.glob(os.path.join(model_dir, '*pb'))
model_name = [os.path.basename(ii) for ii in models]
else:
models = [os.path.join(model_dir,ii) for ii in model_name]
- model_param = {'model_name' : fp_params['model_name'],
- 'param_type': fp_params['model_param_type']}
+ model_param = {'model_name' : model_name,
+ 'param_type': fp_params['model_param_type'],
+ 'deepmd_version' : deepmd_version}
supercell_matrix=jdata['supercell_matrix']
band_path=jdata['band']
@@ -214,7 +229,7 @@ def make_lammps(jdata, conf_dir,task_type) :
fc = lammps.make_lammps_phonon('conf.lmp',
unitcell.masses,
lammps.inter_deepmd,
- model_name)
+ model_param)
if task_type=='meam':
fc = lammps.make_lammps_phonon('conf.lmp',
unitcell.masses,
diff --git a/dpgen/auto_test/gen_07_SScurve.py b/dpgen/auto_test/gen_07_SScurve.py
index c97f3c733..557aaccf3 100644
--- a/dpgen/auto_test/gen_07_SScurve.py
+++ b/dpgen/auto_test/gen_07_SScurve.py
@@ -104,15 +104,16 @@ def make_lammps(jdata, conf_dir,task_type) :
type_map = fp_params['type_map']
model_dir = os.path.abspath(model_dir)
model_name =fp_params['model_name']
+ deepmd_version = fp_params.get("deepmd_version", "0.12")
if not model_name :
models = glob.glob(os.path.join(model_dir, '*pb'))
model_name = [os.path.basename(ii) for ii in models]
else:
models = [os.path.join(model_dir,ii) for ii in model_name]
- model_param = {'model_name' : fp_params['model_name'],
- 'param_type': fp_params['model_param_type']}
-
+ model_param = {'model_name' : model_name,
+ 'param_type': fp_params['model_param_type'],
+ 'deepmd_version' : deepmd_version}
ntypes = len(type_map)
strain_start=jdata['strain_start']
strain_end=jdata['strain_end']
@@ -153,7 +154,7 @@ def make_lammps(jdata, conf_dir,task_type) :
fc = lammps.make_lammps_elastic('conf.lmp',
ntypes,
lammps.inter_deepmd,
- model_name)
+ model_param)
elif task_type =='meam':
fc = lammps.make_lammps_elastic('conf.lmp',
ntypes,
diff --git a/dpgen/auto_test/lib/lammps.py b/dpgen/auto_test/lib/lammps.py
index b6a21f44a..aafc35e21 100644
--- a/dpgen/auto_test/lib/lammps.py
+++ b/dpgen/auto_test/lib/lammps.py
@@ -4,6 +4,7 @@
import dpdata
import subprocess as sp
import dpgen.auto_test.lib.util as util
+from distutils.version import LooseVersion
def cvt_lammps_conf(fin, fout, ofmt = 'lammps/data'):
"""
@@ -89,16 +90,26 @@ def _get_conf_natom(conf) :
return int(ii.split()[0])
raise RuntimeError("cannot find line indicate atom types in ", conf)
-def inter_deepmd(models) :
- ret = ""
- line = "pair_style deepmd "
- if len(models) > 1 :
- for ii in models :
- line += ii + ' '
- line += ' 10 model_devi.out\npair_coeff\n'
- else :
- line += models[0] + '\npair_coeff\n'
- ret += line
+def inter_deepmd(param) :
+ models = param["model_name"]
+ deepmd_version = param["deepmd_version"]
+ ret = "pair_style deepmd "
+ model_list = ""
+ for ii in models:
+ model_list += ii + " "
+ if LooseVersion(deepmd_version) < LooseVersion('1'):
+ ## DeePMD-kit version == 0.x
+ if len(models) > 1 :
+ ret += '%s 10 model_devi.out\n' % model_list
+ else :
+ ret += models[0] + '\n'
+ else:
+ ## DeePMD-kit version >= 1
+ if len(models) > 1:
+ ret += "%s out_freq 10 out_file model_devi.out\n" % model_list
+ else:
+ ret += models[0] + '\n'
+ ret += "pair_coeff\n"
return ret
def inter_meam(param) :
diff --git a/dpgen/auto_test/lib/util.py b/dpgen/auto_test/lib/util.py
index bbb8d438b..06fd27208 100644
--- a/dpgen/auto_test/lib/util.py
+++ b/dpgen/auto_test/lib/util.py
@@ -1,7 +1,7 @@
import numpy as np
import requests
import os,re
-from dpgen.remote.RemoteJob import SSHSession
+from dpgen import dlog
from dpgen.auto_test.lib import vasp
from dpgen.auto_test.lib import lammps
from dpgen.auto_test.lib.utils import cmd_append_log
@@ -60,6 +60,7 @@ def make_work_path(jdata,task,reprod_opt,static,user):
if 'relax_incar' in jdata.keys():
task_type=task_type+'-reprod-relax_incar'
else:
+ kspacing = jdata['vasp_params']['kspacing']
task_type=task_type+'-reprod-k%.2f'% (kspacing)
work_path=os.path.join(task_path, task_type)
@@ -73,7 +74,6 @@ def get_machine_info(mdata,task_type):
group_size = mdata['fp_group_size']
resources = mdata['fp_resources']
machine=mdata['fp_machine']
- machine_type = mdata['fp_machine']['machine_type']
command = vasp_exec
command = cmd_append_log(command, "log")
elif task_type in lammps_task_type:
@@ -81,11 +81,9 @@ def get_machine_info(mdata,task_type):
group_size = mdata['model_devi_group_size']
resources = mdata['model_devi_resources']
machine=mdata['model_devi_machine']
- machine_type = mdata['model_devi_machine']['machine_type']
command = lmp_exec + " -i lammps.in"
command = cmd_append_log(command, "model_devi.log")
- ssh_sess = SSHSession(machine)
- return machine, machine_type,ssh_sess,resources, command, group_size
+ return machine, resources, command, group_size
def collect_task(all_task,task_type):
diff --git a/dpgen/auto_test/lib/vasp.py b/dpgen/auto_test/lib/vasp.py
index f2610de82..2e234b3c1 100644
--- a/dpgen/auto_test/lib/vasp.py
+++ b/dpgen/auto_test/lib/vasp.py
@@ -1,9 +1,11 @@
#!/usr/bin/python3
-
+import os
import warnings
import numpy as np
import dpgen.auto_test.lib.lammps as lammps
import dpgen.auto_test.lib.util as util
+from dpgen.generator.lib.vasp import incar_upper
+from pymatgen.io.vasp import Incar,Kpoints,Potcar
class OutcarItemError(Exception):
pass
@@ -102,6 +104,8 @@ def reciprocal_box(box) :
return rbox
def make_kspacing_kpoints(poscar, kspacing, kgamma) :
+ if type(kspacing) is not list:
+ kspacing = [kspacing, kspacing, kspacing]
with open(poscar, 'r') as fp:
lines = fp.read().split('\n')
scale = float(lines[1])
@@ -111,7 +115,7 @@ def make_kspacing_kpoints(poscar, kspacing, kgamma) :
box = np.array(box)
box *= scale
rbox = reciprocal_box(box)
- kpoints = [(np.ceil(2 * np.pi * np.linalg.norm(ii) / kspacing).astype(int)) for ii in rbox]
+ kpoints = [max(1,(np.ceil(2 * np.pi * np.linalg.norm(ii) / ks).astype(int))) for ii,ks in zip(rbox,kspacing)]
ret = make_vasp_kpoints(kpoints, kgamma)
return ret
@@ -247,7 +251,7 @@ def make_vasp_static_incar (ecut, ediff,
ret += 'PREC=A\n'
ret += 'ENCUT=%d\n' % ecut
ret += '# ISYM=0\n'
- ret += 'ALGO=fast\n'
+ ret += 'ALGO=normal\n'
ret += 'EDIFF=%e\n' % ediff
ret += 'EDIFFG=-0.01\n'
ret += 'LREAL=A\n'
@@ -288,7 +292,7 @@ def make_vasp_relax_incar (ecut, ediff,
ret += 'PREC=A\n'
ret += 'ENCUT=%d\n' % ecut
ret += '# ISYM=0\n'
- ret += 'ALGO=fast\n'
+ ret += 'ALGO=normal\n'
ret += 'EDIFF=%e\n' % ediff
ret += 'EDIFFG=-0.01\n'
ret += 'LREAL=A\n'
@@ -329,7 +333,7 @@ def make_vasp_phonon_incar (ecut, ediff,
ret += 'PREC=A\n'
ret += 'ENCUT=%d\n' % ecut
ret += '# ISYM=0\n'
- ret += 'ALGO=fast\n'
+ ret += 'ALGO=normal\n'
ret += 'EDIFF=%e\n' % ediff
ret += 'EDIFFG=-0.01\n'
ret += 'LREAL=A\n'
@@ -456,3 +460,38 @@ def make_vasp_kpoints (kpoints, kgamma = False) :
ret = _make_vasp_kp_mp(kpoints)
return ret
+
+def make_vasp_kpoints_from_incar(work_dir,jdata):
+ cwd=os.getcwd()
+ fp_aniso_kspacing = jdata.get('fp_aniso_kspacing')
+ os.chdir(work_dir)
+ # get kspacing and kgamma from incar
+ assert(os.path.exists('INCAR'))
+ with open('INCAR') as fp:
+ incar = fp.read()
+ standard_incar = incar_upper(Incar.from_string(incar))
+ if fp_aniso_kspacing is None:
+ try:
+ kspacing = standard_incar['KSPACING']
+ except KeyError:
+ raise RuntimeError ("KSPACING must be given in INCAR")
+ else :
+ kspacing = fp_aniso_kspacing
+ try:
+ gamma = standard_incar['KGAMMA']
+ if isinstance(gamma,bool):
+ pass
+ else:
+ if gamma[0].upper()=="T":
+ gamma=True
+ else:
+ gamma=False
+ except KeyError:
+ raise RuntimeError ("KGAMMA must be given in INCAR")
+ # check poscar
+ assert(os.path.exists('POSCAR'))
+ # make kpoints
+ ret=make_kspacing_kpoints('POSCAR', kspacing, gamma)
+ kp=Kpoints.from_string(ret)
+ kp.write_file("KPOINTS")
+ os.chdir(cwd)
diff --git a/dpgen/auto_test/run.py b/dpgen/auto_test/run.py
index a44206909..9c5380893 100644
--- a/dpgen/auto_test/run.py
+++ b/dpgen/auto_test/run.py
@@ -35,15 +35,13 @@
from dpgen.auto_test.lib.utils import log_iter
from dpgen.auto_test.lib.pwscf import make_pwscf_input
from dpgen.auto_test.lib.siesta import make_siesta_input
-from dpgen.remote.RemoteJob import SSHSession, JobStatus, SlurmJob, PBSJob, CloudMachineJob
-from dpgen.remote.decide_machine import decide_fp_machine, decide_model_devi_machine
-from dpgen.remote.group_jobs import *
from dpgen.auto_test import gen_00_equi,cmpt_00_equi
from dpgen.auto_test import gen_01_eos,cmpt_01_eos
from dpgen.auto_test import gen_02_elastic,cmpt_02_elastic
from dpgen.auto_test import gen_03_vacancy,cmpt_03_vacancy
from dpgen.auto_test import gen_04_interstitial,cmpt_04_interstitial
from dpgen.auto_test import gen_05_surf,cmpt_05_surf
+from dpgen.remote.decide_machine import decide_fp_machine, decide_model_devi_machine
#from dpgen.auto_test import gen_06_phonon,cmpt_06_phonon
from dpgen.auto_test import gen_confs
import requests
@@ -74,8 +72,11 @@ def run_equi(task_type,jdata,mdata):
if task_type=="vasp":
mdata=decide_fp_machine(mdata)
- forward_files = ['INCAR', 'POTCAR']
- backward_files = ['OUTCAR', 'autotest.out' , 'CONTCAR','OSZICAR']
+ forward_files = ['INCAR', 'POTCAR', 'KPOINTS']
+ if ('cvasp' in jdata) and (jdata['cvasp'] == True):
+ mdata['fp_resources']['cvasp'] = True
+ forward_files.append('cvasp.py')
+ backward_files = ['OUTCAR', task_type+'.out' , 'CONTCAR','OSZICAR']
common_files=['POSCAR']
#lammps
@@ -83,7 +84,7 @@ def run_equi(task_type,jdata,mdata):
mdata = decide_model_devi_machine(mdata)
forward_files = ['conf.lmp', 'lammps.in']
- backward_files = ['dump.relax','log.lammps', 'autotest.out']
+ backward_files = ['dump.relax','log.lammps', task_type+'.out']
fp_params = jdata['lammps_params']
model_dir = fp_params['model_dir']
@@ -104,8 +105,12 @@ def run_equi(task_type,jdata,mdata):
run_tasks = util.collect_task(all_task,task_type)
if len(run_tasks)==0: return
- machine,machine_type,ssh_sess,resources,command,group_size=util.get_machine_info(mdata,task_type)
- disp = make_dispatcher(machine)
+ machine,resources,command,group_size=util.get_machine_info(mdata,task_type)
+ disp = make_dispatcher(machine, resources, work_path, run_tasks, group_size)
+ #debug#
+ #print(' '.join(common_files))
+ #print(' '.join(forward_files))
+ #print(' '.join(backward_files))
disp.run_jobs(resources,
command,
work_path,
@@ -114,8 +119,8 @@ def run_equi(task_type,jdata,mdata):
common_files,
forward_files,
backward_files,
- outlog='autotest.out',
- errlog='autotest.err')
+ outlog=task_type+'.out',
+ errlog=task_type+'.err')
def cmpt_equi(task_type,jdata,mdata):
@@ -163,9 +168,12 @@ def run_eos(task_type,jdata,mdata):
if task_type=="vasp":
mdata=decide_fp_machine(mdata)
- forward_files = ['INCAR', 'POSCAR','POTCAR']
- backward_files = ['OUTCAR', 'autotest.out' , 'OSZICAR']
+ forward_files = ['INCAR', 'POSCAR','POTCAR','KPOINTS']
+ backward_files = ['OUTCAR', task_type+'.out' , 'OSZICAR']
common_files=['INCAR','POTCAR']
+ if ('cvasp' in jdata) and (jdata['cvasp'] == True):
+ mdata['fp_resources']['cvasp'] = True
+ forward_files.append('cvasp.py')
#lammps
elif task_type in lammps_task_type:
@@ -181,7 +189,7 @@ def run_eos(task_type,jdata,mdata):
else:
models = [os.path.join(model_dir,ii) for ii in model_name]
forward_files = ['conf.lmp', 'lammps.in']+model_name
- backward_files = ['log.lammps', 'autotest.out']
+ backward_files = ['log.lammps', task_type+'.out']
common_files=['lammps.in']+model_name
if len(model_name)>1 and task_type == 'deepmd':
@@ -192,8 +200,8 @@ def run_eos(task_type,jdata,mdata):
run_tasks = util.collect_task(all_task,task_type)
if len(run_tasks)==0: return
- machine,machine_type,ssh_sess,resources,command,group_size=util.get_machine_info(mdata,task_type)
- disp = make_dispatcher(machine)
+ machine,resources,command,group_size=util.get_machine_info(mdata,task_type)
+ disp = make_dispatcher(machine, resources, work_path, run_tasks, group_size)
disp.run_jobs(resources,
command,
work_path,
@@ -202,8 +210,8 @@ def run_eos(task_type,jdata,mdata):
common_files,
forward_files,
backward_files,
- outlog='autotest.out',
- errlog='autotest.err')
+ outlog=task_type+'.out',
+ errlog=task_type+'.err')
def cmpt_eos(task_type,jdata,mdata):
conf_dir=jdata['conf_dir']
@@ -240,8 +248,11 @@ def run_elastic(task_type,jdata,mdata):
mdata=decide_fp_machine(mdata)
forward_files = ['INCAR', 'POSCAR','POTCAR','KPOINTS']
- backward_files = ['OUTCAR', 'autotest.out' , 'CONTCAR','OSZICAR']
+ backward_files = ['OUTCAR', task_type+'.out' , 'CONTCAR','OSZICAR']
common_files=['INCAR','POTCAR','KPOINTS']
+ if ('cvasp' in jdata) and (jdata['cvasp'] == True):
+ mdata['fp_resources']['cvasp'] = True
+ forward_files.append('cvasp.py')
#lammps
elif task_type in lammps_task_type:
@@ -257,7 +268,7 @@ def run_elastic(task_type,jdata,mdata):
else:
models = [os.path.join(model_dir,ii) for ii in model_name]
forward_files = ['conf.lmp', 'lammps.in','strain.out']+model_name
- backward_files = ['log.lammps', 'autotest.out']
+ backward_files = ['log.lammps', task_type+'.out']
common_files=['lammps.in']+model_name
if len(model_name)>1 and task_type == 'deepmd':
@@ -268,8 +279,8 @@ def run_elastic(task_type,jdata,mdata):
run_tasks = util.collect_task(all_task,task_type)
if len(run_tasks)==0: return
- machine,machine_type,ssh_sess,resources,command,group_size=util.get_machine_info(mdata,task_type)
- disp = make_dispatcher(machine)
+ machine,resources,command,group_size=util.get_machine_info(mdata,task_type)
+ disp = make_dispatcher(machine, resources, work_path, run_tasks, group_size)
disp.run_jobs(resources,
command,
work_path,
@@ -278,8 +289,8 @@ def run_elastic(task_type,jdata,mdata):
common_files,
forward_files,
backward_files,
- outlog='autotest.out',
- errlog='autotest.err')
+ outlog=task_type+'.out',
+ errlog=task_type+'.err')
def cmpt_elastic(task_type,jdata,mdata):
conf_dir=jdata['conf_dir']
@@ -313,9 +324,12 @@ def run_vacancy(task_type,jdata,mdata):
if task_type == "vasp":
mdata=decide_fp_machine(mdata)
- forward_files = ['INCAR', 'POSCAR','POTCAR']
- backward_files = ['OUTCAR', 'autotest.out' , 'OSZICAR']
+ forward_files = ['INCAR', 'POSCAR','POTCAR','KPOINTS']
+ backward_files = ['OUTCAR', task_type+'.out' , 'OSZICAR']
common_files=['INCAR','POTCAR']
+ if ('cvasp' in jdata) and (jdata['cvasp'] == True):
+ mdata['fp_resources']['cvasp'] = True
+ forward_files.append('cvasp.py')
#lammps
elif task_type in lammps_task_type:
@@ -332,7 +346,7 @@ def run_vacancy(task_type,jdata,mdata):
models = [os.path.join(model_dir,ii) for ii in model_name]
common_files = model_name
forward_files = ['conf.lmp', 'lammps.in']+model_name
- backward_files = ['log.lammps','autotest.out']
+ backward_files = ['log.lammps',task_type+'.out']
common_files=['lammps.in']+model_name
if len(model_name)>1 and task_type == 'deepmd':
@@ -343,8 +357,8 @@ def run_vacancy(task_type,jdata,mdata):
run_tasks = util.collect_task(all_task,task_type)
if len(run_tasks)==0: return
- machine,machine_type,ssh_sess,resources,command,group_size=util.get_machine_info(mdata,task_type)
- disp = make_dispatcher(machine)
+ machine,resources,command,group_size=util.get_machine_info(mdata,task_type)
+ disp = make_dispatcher(machine, resources, work_path, run_tasks, group_size)
disp.run_jobs(resources,
command,
work_path,
@@ -353,8 +367,8 @@ def run_vacancy(task_type,jdata,mdata):
common_files,
forward_files,
backward_files,
- outlog='autotest.out',
- errlog='autotest.err')
+ outlog=task_type+'.out',
+ errlog=task_type+'.err')
def cmpt_vacancy(task_type,jdata,mdata):
conf_dir=jdata['conf_dir']
@@ -397,9 +411,12 @@ def run_interstitial(task_type,jdata,mdata):
if task_type == "vasp":
mdata=decide_fp_machine(mdata)
- forward_files = ['INCAR', 'POSCAR','POTCAR']
- backward_files = ['OUTCAR', 'autotest.out' , 'XDATCAR','OSZICAR']
+ forward_files = ['INCAR', 'POSCAR','POTCAR',"KPOINTS"]
+ backward_files = ['OUTCAR', task_type+'.out' , 'XDATCAR','OSZICAR']
common_files=['INCAR']
+ if ('cvasp' in jdata) and (jdata['cvasp'] == True):
+ mdata['fp_resources']['cvasp'] = True
+ forward_files.append('cvasp.py')
#lammps
elif task_type in lammps_task_type:
@@ -431,7 +448,7 @@ def run_interstitial(task_type,jdata,mdata):
else:
models = [os.path.join(model_dir,ii) for ii in model_name]
forward_files = ['conf.lmp', 'lammps.in']+model_name
- backward_files = ['log.lammps', 'autotest.out']
+ backward_files = ['log.lammps', task_type+'.out']
common_files=['lammps.in']+model_name
if len(model_name)>1 and task_type == 'deepmd':
@@ -440,15 +457,14 @@ def run_interstitial(task_type,jdata,mdata):
else:
raise RuntimeError ("unknow task %s, something wrong" % task_type)
- machine,machine_type,ssh_sess,resources,command,group_size=util.get_machine_info(mdata,task_type)
- disp = make_dispatcher(machine)
+ machine,resources,command,group_size=util.get_machine_info(mdata,task_type)
if reprod_opt:
for ii in work_path:
run_tasks=[]
for jj in run_tasks_:
if ii in jj:
run_tasks.append(os.path.basename(jj))
-
+ disp = make_dispatcher(machine, resources, work_path, run_tasks, group_size)
disp.run_jobs(resources,
command,
ii,
@@ -457,11 +473,12 @@ def run_interstitial(task_type,jdata,mdata):
common_files,
forward_files,
backward_files,
- outlog='autotest.out',
- errlog='autotest.err')
+ outlog=task_type+'.out',
+ errlog=task_type+'.err')
else:
run_tasks = util.collect_task(all_task,task_type)
if len(run_tasks)==0: return
+ disp = make_dispatcher(machine, resources, work_path, run_tasks, group_size)
disp.run_jobs(resources,
command,
work_path,
@@ -470,8 +487,8 @@ def run_interstitial(task_type,jdata,mdata):
common_files,
forward_files,
backward_files,
- outlog='autotest.out',
- errlog='autotest.err')
+ outlog=task_type+'.out',
+ errlog=task_type+'.err')
def cmpt_interstitial(task_type,jdata,mdata):
conf_dir=jdata['conf_dir']
@@ -519,9 +536,12 @@ def run_surf(task_type,jdata,mdata):
if task_type == "vasp":
mdata=decide_fp_machine(mdata)
- forward_files = ['INCAR', 'POSCAR','POTCAR']
- backward_files = ['OUTCAR', 'autotest.out' , 'OSZICAR']
+ forward_files = ['INCAR', 'POSCAR','POTCAR','KPOINTS']
+ backward_files = ['OUTCAR', task_type+'.out' , 'OSZICAR']
common_files=['INCAR','POTCAR']
+ if ('cvasp' in jdata) and (jdata['cvasp'] == True):
+ mdata['fp_resources']['cvasp'] = True
+ forward_files.append('cvasp.py')
#lammps
elif task_type in lammps_task_type:
@@ -537,7 +557,7 @@ def run_surf(task_type,jdata,mdata):
else:
models = [os.path.join(model_dir,ii) for ii in model_name]
forward_files = ['conf.lmp', 'lammps.in']+model_name
- backward_files = ['log.lammps','autotest.out']
+ backward_files = ['log.lammps',task_type+'.out']
common_files=['lammps.in']+model_name
if len(model_name)>1 and task_type == 'deepmd':
@@ -548,8 +568,8 @@ def run_surf(task_type,jdata,mdata):
run_tasks = util.collect_task(all_task,task_type)
if len(run_tasks)==0: return
- machine,machine_type,ssh_sess,resources,command,group_size=util.get_machine_info(mdata,task_type)
- disp = make_dispatcher(machine)
+ machine,resources,command,group_size=util.get_machine_info(mdata,task_type)
+ disp = make_dispatcher(machine, resources, work_path, run_tasks, group_size)
disp.run_jobs(resources,
command,
work_path,
@@ -558,8 +578,8 @@ def run_surf(task_type,jdata,mdata):
common_files,
forward_files,
backward_files,
- outlog='autotest.out',
- errlog='autotest.err')
+ outlog=task_type+'.out',
+ errlog=task_type+'.err')
def cmpt_surf(task_type,jdata,mdata):
conf_dir=jdata['conf_dir']
@@ -601,14 +621,17 @@ def run_phonon(task_type,jdata,mdata):
#vasp
if task_type == "vasp":
mdata=decide_fp_machine(mdata)
- machine,machine_type,ssh_sess,resources,command,group_size=util.get_machine_info(mdata,task_type)
+ machine,resources,command,group_size=util.get_machine_info(mdata,task_type)
run_tasks = util.collect_task(all_task,task_type)
- forward_files = ['INCAR', 'POTCAR','KPOINTS']
- backward_files = ['OUTCAR', 'autotest.out' , 'OSZICAR','vasprun.xml']
+ forward_files = ['INCAR', 'POTCAR','KPOINTS','KPOINTS']
+ backward_files = ['OUTCAR', task_type+'.out' , 'OSZICAR','vasprun.xml']
common_files=['POSCAR']
+ if ('cvasp' in jdata) and (jdata['cvasp'] == True):
+ mdata['fp_resources']['cvasp'] = True
+ forward_files.append('cvasp.py')
- disp = make_dispatcher(machine)
+ disp = make_dispatcher(machine, resources, work_path, run_tasks, group_size)
disp.run_jobs(resources,
command,
work_path,
@@ -617,13 +640,13 @@ def run_phonon(task_type,jdata,mdata):
common_files,
forward_files,
backward_files,
- outlog='autotest.out',
- errlog='autotest.err')
+ outlog=task_type+'.out',
+ errlog=task_type+'.err')
#lammps
elif task_type in lammps_task_type:
None
else:
- raise RuntimeError ("unknow task %s, something wrong" % task_type)
+ raise RuntimeError ("unknown task %s, something wrong" % task_type)
def cmpt_phonon(task_type,jdata,mdata):
conf_dir=jdata['conf_dir']
@@ -652,10 +675,29 @@ def run_task (json_file, machine_file) :
ii = jdata['task_type']
jj=jdata['task']
- task_list=['equi','eos','elastic','vacancy','interstitial','surf','phonon','all']
+ task_list=['equi','eos','elastic','vacancy','interstitial','surf','phonon']
+
+ if isinstance(jj,str):
+ if jj=='all':
+ all_task_list=task_list.copy()
+ else:
+ try:
+ assert jj in task_list
+ except AssertionError:
+ raise RuntimeError ("unknow task %s, something wrong" % jj)
+ all_task_list=[jj]
+ elif isinstance(jj,list):
+ try:
+ assert set(jj).issubset(set(task_list))
+ except AssertionError:
+ raise RuntimeError ("unknow task %s, some tasks may not supported" % ' '.join(jj))
+ all_task_list=jj.copy()
+ else:
+ raise RuntimeError ('unknow format for task, it must be a string or list')
+
task_type_list=['vasp']+lammps_task_type
- if jj not in task_list :
- raise RuntimeError ("unknow task %s, something wrong" % jj)
+ #if jj not in task_list :
+ # raise RuntimeError ("unknow task %s, something wrong" % jj)
if ii not in task_type_list :
raise RuntimeError ("unknow task type %s, something wrong" % ii)
@@ -673,51 +715,52 @@ def run_task (json_file, machine_file) :
run_equi (ii, jdata, mdata)
log_iter ("cmpt_equi", ii,"equi")
cmpt_equi (ii, jdata, mdata)
- if jj == "eos" or jj=="all":
- log_iter ("gen_eos", ii, "eos")
- gen_eos (ii, jdata, mdata)
- log_iter ("run_eos", ii, "eos")
- run_eos (ii, jdata, mdata)
- log_iter ("cmpt_eos", ii, "eos")
- cmpt_eos (ii, jdata, mdata)
- if jj=="elastic" or jj=="all":
- log_iter ("gen_elastic", ii, "elastic")
- gen_elastic (ii, jdata, mdata)
- log_iter ("run_elastic", ii, "elastic")
- run_elastic (ii, jdata, mdata)
- log_iter ("cmpt_elastic", ii, "elastic")
- cmpt_elastic (ii, jdata, mdata)
- if jj=="vacancy" or jj=="all":
- log_iter ("gen_vacancy", ii, "vacancy")
- gen_vacancy (ii, jdata, mdata)
- log_iter ("run_vacancy", ii, "vacancy")
- run_vacancy (ii, jdata, mdata)
- log_iter ("cmpt_vacancy", ii, "vacancy")
- cmpt_vacancy (ii, jdata, mdata)
- if jj=="interstitial" or jj=="all":
- log_iter ("gen_interstitial", ii, "interstitial")
- gen_interstitial (ii, jdata, mdata)
- log_iter ("run_interstitial", ii, "interstitial")
- run_interstitial (ii, jdata, mdata)
- log_iter ("cmpt_interstitial", ii, "interstitial")
- cmpt_interstitial (ii, jdata, mdata)
- if jj=="surf" or jj=="all":
- log_iter ("gen_surf", ii, "surf")
- gen_surf (ii, jdata, mdata)
- log_iter ("run_surf", ii, "surf")
- run_surf (ii, jdata, mdata)
- log_iter ("cmpt_surf", ii, "surf")
- cmpt_surf (ii, jdata, mdata)
- '''
- if jj=="phonon":
- log_iter ("gen_phonon", ii, "phonon")
- gen_phonon (ii, jdata, mdata)
- log_iter ("run_phonon", ii, "phonon")
- run_phonon (ii, jdata, mdata)
- log_iter ("cmpt_phonon", ii, "phonon")
- cmpt_phonon (ii, jdata, mdata)
- '''
- record_iter (record, confs, ii, jj)
+ for jj in all_task_list:
+ if jj == "eos":
+ log_iter ("gen_eos", ii, "eos")
+ gen_eos (ii, jdata, mdata)
+ log_iter ("run_eos", ii, "eos")
+ run_eos (ii, jdata, mdata)
+ log_iter ("cmpt_eos", ii, "eos")
+ cmpt_eos (ii, jdata, mdata)
+ if jj=="elastic":
+ log_iter ("gen_elastic", ii, "elastic")
+ gen_elastic (ii, jdata, mdata)
+ log_iter ("run_elastic", ii, "elastic")
+ run_elastic (ii, jdata, mdata)
+ log_iter ("cmpt_elastic", ii, "elastic")
+ cmpt_elastic (ii, jdata, mdata)
+ if jj=="vacancy":
+ log_iter ("gen_vacancy", ii, "vacancy")
+ gen_vacancy (ii, jdata, mdata)
+ log_iter ("run_vacancy", ii, "vacancy")
+ run_vacancy (ii, jdata, mdata)
+ log_iter ("cmpt_vacancy", ii, "vacancy")
+ cmpt_vacancy (ii, jdata, mdata)
+ if jj=="interstitial":
+ log_iter ("gen_interstitial", ii, "interstitial")
+ gen_interstitial (ii, jdata, mdata)
+ log_iter ("run_interstitial", ii, "interstitial")
+ run_interstitial (ii, jdata, mdata)
+ log_iter ("cmpt_interstitial", ii, "interstitial")
+ cmpt_interstitial (ii, jdata, mdata)
+ if jj=="surf":
+ log_iter ("gen_surf", ii, "surf")
+ gen_surf (ii, jdata, mdata)
+ log_iter ("run_surf", ii, "surf")
+ run_surf (ii, jdata, mdata)
+ log_iter ("cmpt_surf", ii, "surf")
+ cmpt_surf (ii, jdata, mdata)
+ '''
+ if jj=="phonon":
+ log_iter ("gen_phonon", ii, "phonon")
+ gen_phonon (ii, jdata, mdata)
+ log_iter ("run_phonon", ii, "phonon")
+ run_phonon (ii, jdata, mdata)
+ log_iter ("cmpt_phonon", ii, "phonon")
+ cmpt_phonon (ii, jdata, mdata)
+ '''
+ record_iter (record, confs, ii, jj)
def gen_test(args):
logging.info ("start auto-testing")
diff --git a/dpgen/collect/__init__.py b/dpgen/collect/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/dpgen/collect/collect.py b/dpgen/collect/collect.py
new file mode 100644
index 000000000..a8d5b4060
--- /dev/null
+++ b/dpgen/collect/collect.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+
+import os,sys,json,glob,argparse,dpdata
+import numpy as np
+from dpgen.generator.run import data_system_fmt
+
+def collect_data(target_folder, param_file, output,
+ verbose = True,
+ shuffle = True,
+ merge = True) :
+ target_folder = os.path.abspath(target_folder)
+ output = os.path.abspath(output)
+ # goto input
+ cwd = os.getcwd()
+ os.chdir(target_folder)
+ jdata = json.load(open(param_file))
+ sys_configs_prefix = jdata.get('sys_configs_prefix', '')
+ sys_configs = jdata.get('sys_configs', [])
+ if verbose :
+ max_str_len = max([len(str(ii)) for ii in sys_configs])
+ max_form_len = 16
+ ptr_fmt = '%%%ds %%%ds natoms %%6d nframes %%6d' % (max_str_len+5, max_form_len)
+ # init systems
+ init_data = []
+ init_data_prefix = jdata.get('init_data_prefix', '')
+ init_data_sys = jdata.get('init_data_sys', [])
+ for ii in init_data_sys:
+ init_data.append(dpdata.LabeledSystem(os.path.join(init_data_prefix, ii), fmt='deepmd/npy'))
+ # collect systems from iter dirs
+ coll_data = {}
+ numb_sys = len(sys_configs)
+ model_devi_jobs = jdata.get('model_devi_jobs', {})
+ numb_jobs = len(model_devi_jobs)
+ iters = ['iter.%06d' % ii for ii in range(numb_jobs)]
+ # loop over iters to collect data
+ for ii in range(len(iters)) :
+ iter_data = glob.glob(os.path.join(iters[ii], '02.fp', 'data.[0-9]*[0-9]'))
+ iter_data.sort()
+ for jj in iter_data :
+ sys = dpdata.LabeledSystem(jj, fmt = 'deepmd/npy')
+ if merge:
+ sys_str = sys.formula
+ else:
+ sys_str = (os.path.basename(jj).split('.')[-1])
+ if sys_str in coll_data.keys():
+ coll_data[sys_str].append(sys)
+ else:
+ coll_data[sys_str] = sys
+ # print information
+ if verbose:
+ for ii in range(len(init_data)):
+ print(ptr_fmt % (str(init_data_sys[ii]),
+ init_data[ii].formula,
+ init_data[ii].get_natoms(),
+ init_data[ii].get_nframes() ))
+ keys = list(coll_data.keys())
+ keys.sort()
+ for ii in keys:
+ if merge:
+ sys_str = ii
+ else :
+ sys_str = str(sys_configs[int(ii)])
+ print(ptr_fmt % (sys_str,
+ coll_data[ii].formula,
+ coll_data[ii].get_natoms(),
+ coll_data[ii].get_nframes() ))
+ # shuffle system data
+ if shuffle:
+ for kk in coll_data.keys():
+ coll_data[kk].shuffle()
+ # create output dir
+ os.chdir(cwd)
+ os.makedirs(output, exist_ok = True)
+ # dump init data
+ for idx,ii in enumerate(init_data):
+ out_dir = 'init.' + (data_system_fmt % idx)
+ ii.to('deepmd/npy', os.path.join(output, out_dir))
+ # dump iter data
+ for kk in coll_data.keys():
+ out_dir = 'sys.%s' % kk
+ nframes = coll_data[kk].get_nframes()
+ coll_data[kk].to('deepmd/npy', os.path.join(output, out_dir), set_size = nframes)
+
+def gen_collect(args):
+ collect_data(args.JOB_DIR, args.parameter, args.OUTPUT,
+ verbose = args.verbose,
+ shuffle = args.shuffle,
+ merge = args.merge)
+
+def _main() :
+ parser = argparse.ArgumentParser(description='Collect data from DP-GEN iterations')
+ parser.add_argument("JOB_DIR", type=str,
+ help="the directory of the DP-GEN job")
+ parser.add_argument("OUTPUT", type=str,
+ help="the output directory of data")
+ parser.add_argument('-p',"--parameter", type=str, default = 'param.json',
+ help="the json file provides DP-GEN paramters, should be located in JOB_DIR")
+ parser.add_argument('-v',"--verbose", action = 'store_true',
+ help="print number of data in each system")
+ parser.add_argument('-m',"--merge", action = 'store_true',
+ help="merge the systems with the same chemical formula")
+ parser.add_argument('-s',"--shuffle", action = 'store_true',
+ help="shuffle the data systems")
+ args = parser.parse_args()
+ gen_collect(args)
+
+if __name__ == '__main__':
+ _main()
+
+
diff --git a/dpgen/data/gen.py b/dpgen/data/gen.py
index 73bb8e25f..ce1573a61 100644
--- a/dpgen/data/gen.py
+++ b/dpgen/data/gen.py
@@ -20,25 +20,33 @@
import dpgen.data.tools.bcc as bcc
import dpgen.data.tools.diamond as diamond
import dpgen.data.tools.sc as sc
+from dpgen.generator.lib.vasp import incar_upper
from pymatgen import Structure
+from pymatgen.io.vasp import Incar
from dpgen.remote.decide_machine import decide_fp_machine
from dpgen import ROOT_PATH
from dpgen.dispatcher.Dispatcher import Dispatcher, make_dispatcher
-def create_path (path) :
+def create_path (path,back=False) :
if path[-1] != "/":
path += '/'
if os.path.isdir(path) :
- dirname = os.path.dirname(path)
- counter = 0
- while True :
- bk_dirname = dirname + ".bk%03d" % counter
- if not os.path.isdir(bk_dirname) :
- shutil.move (dirname, bk_dirname)
- break
- counter += 1
+ if back:
+ dirname = os.path.dirname(path)
+ counter = 0
+ while True :
+ bk_dirname = dirname + ".bk%03d" % counter
+ if not os.path.isdir(bk_dirname) :
+ shutil.move (dirname, bk_dirname)
+ break
+ counter += 1
+ os.makedirs (path)
+ return path
+ else:
+ return path
+
os.makedirs (path)
return path
@@ -238,7 +246,10 @@ def make_super_cell_poscar(jdata) :
cwd = os.getcwd()
to_file = os.path.abspath(to_file)
os.chdir(path_work)
- os.symlink(os.path.relpath(to_file), 'POSCAR')
+ try:
+ os.symlink(os.path.relpath(to_file), 'POSCAR')
+ except FileExistsError:
+ pass
os.chdir(cwd)
def make_combines (dim, natoms) :
@@ -317,9 +328,15 @@ def make_vasp_relax (jdata, mdata) :
for ss in sys_list:
os.chdir(ss)
ln_src = os.path.relpath(os.path.join(work_dir,'INCAR'))
- os.symlink(ln_src, 'INCAR')
+ try:
+ os.symlink(ln_src, 'INCAR')
+ except FileExistsError:
+ pass
ln_src = os.path.relpath(os.path.join(work_dir,'POTCAR'))
- os.symlink(ln_src, 'POTCAR')
+ try:
+ os.symlink(ln_src, 'POTCAR')
+ except FileExistsError:
+ pass
os.chdir(work_dir)
os.chdir(cwd)
@@ -452,8 +469,15 @@ def make_vasp_md(jdata) :
shutil.copy2 (init_pos, 'POSCAR')
file_incar = os.path.join(path_md, 'INCAR')
file_potcar = os.path.join(path_md, 'POTCAR')
- os.symlink(os.path.relpath(file_incar), 'INCAR')
- os.symlink(os.path.relpath(file_potcar), 'POTCAR')
+ try:
+ os.symlink(os.path.relpath(file_incar), 'INCAR')
+ except FileExistsError:
+ pass
+ try:
+ os.symlink(os.path.relpath(file_potcar), 'POTCAR')
+ except FileExistsError:
+ pass
+
os.chdir(cwd)
def coll_vasp_md(jdata) :
@@ -532,7 +556,7 @@ def _vasp_check_fin (ii) :
return False
return True
-def run_vasp_relax(jdata, mdata, dispatcher):
+def run_vasp_relax(jdata, mdata):
fp_command = mdata['fp_command']
fp_group_size = mdata['fp_group_size']
fp_resources = mdata['fp_resources']
@@ -557,7 +581,7 @@ def run_vasp_relax(jdata, mdata, dispatcher):
# if not _vasp_check_fin(ii):
# relax_run_tasks.append(ii)
run_tasks = [os.path.basename(ii) for ii in relax_run_tasks]
-
+ dispatcher = make_dispatcher(mdata['fp_machine'], mdata['fp_resources'], work_dir, run_tasks, fp_group_size)
#dlog.info(run_tasks)
dispatcher.run_jobs(fp_resources,
[fp_command],
@@ -568,7 +592,7 @@ def run_vasp_relax(jdata, mdata, dispatcher):
forward_files,
backward_files)
-def run_vasp_md(jdata, mdata, dispatcher):
+def run_vasp_md(jdata, mdata):
fp_command = mdata['fp_command']
fp_group_size = mdata['fp_group_size']
fp_resources = mdata['fp_resources']
@@ -603,7 +627,7 @@ def run_vasp_md(jdata, mdata, dispatcher):
run_tasks = [ii.replace(work_dir+"/", "") for ii in md_run_tasks]
#dlog.info("md_work_dir", work_dir)
#dlog.info("run_tasks",run_tasks)
-
+ dispatcher = make_dispatcher(mdata['fp_machine'], mdata['fp_resources'], work_dir, run_tasks, fp_group_size)
dispatcher.run_jobs(fp_resources,
[fp_command],
work_dir,
@@ -631,7 +655,7 @@ def gen_init_bulk(args) :
if args.MACHINE is not None:
# Selecting a proper machine
mdata = decide_fp_machine(mdata)
- disp = make_dispatcher(mdata["fp_machine"])
+ #disp = make_dispatcher(mdata["fp_machine"])
# Decide work path
out_dir = out_dir_name(jdata)
@@ -646,15 +670,11 @@ def gen_init_bulk(args) :
try:
md_incar = jdata['md_incar']
if os.path.isfile(md_incar):
- with open(md_incar , "r") as fr:
- md_incar_lines = fr.readlines()
+ standard_incar = incar_upper(Incar.from_file(md_incar))
nsw_flag = False
- for incar_line in md_incar_lines:
- line = incar_line.split()
- if "NSW" in line:
+ if "NSW" in standard_incar:
nsw_flag = True
- nsw_steps = int(incar_line.split()[-1])
- break
+ nsw_steps = standard_incar['NSW']
#dlog.info("nsw_steps is", nsw_steps)
#dlog.info("md_nstep_jdata is", md_nstep_jdata)
if nsw_flag:
@@ -664,7 +684,7 @@ def gen_init_bulk(args) :
dlog.info("MD steps in md_incar is %d"%(nsw_steps))
dlog.info("DP-GEN will use settings in md_incar!")
jdata['md_nstep'] = nsw_steps
- except:
+ except KeyError:
pass
## correct element name
temp_elements = []
@@ -688,7 +708,7 @@ def gen_init_bulk(args) :
place_element(jdata)
if args.MACHINE is not None:
make_vasp_relax(jdata, mdata)
- run_vasp_relax(jdata, mdata, disp)
+ run_vasp_relax(jdata, mdata)
else:
make_vasp_relax(jdata, {"fp_resources":{}})
elif stage == 2 :
@@ -699,7 +719,7 @@ def gen_init_bulk(args) :
dlog.info("Current stage is 3, run a short md")
make_vasp_md(jdata)
if args.MACHINE is not None:
- run_vasp_md(jdata, mdata, disp)
+ run_vasp_md(jdata, mdata)
elif stage == 4 :
dlog.info("Current stage is 4, collect data")
coll_vasp_md(jdata)
diff --git a/dpgen/data/surf.py b/dpgen/data/surf.py
index 78c6f918c..45afc9a91 100644
--- a/dpgen/data/surf.py
+++ b/dpgen/data/surf.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
+import time
import os,json,shutil,re,glob,argparse
import numpy as np
import subprocess as sp
@@ -9,17 +10,18 @@
import dpgen.data.tools.sc as sc
import dpgen.data.tools.bcc as bcc
from dpgen import dlog
-import time
from dpgen import ROOT_PATH
from dpgen.remote.decide_machine import decide_fp_machine
-from pymatgen.core.surface import SlabGenerator,generate_all_slabs, Structure
-from pymatgen.io.vasp import Poscar
from dpgen.dispatcher.Dispatcher import Dispatcher, make_dispatcher
-#-----ASE-------
+#-----PMG---------
+from pymatgen.io.vasp import Poscar
+from pymatgen import Structure,Element
from pymatgen.io.ase import AseAtomsAdaptor
+#-----ASE-------
from ase.io import read
from ase.build import general_surface
+
def create_path (path) :
path += '/'
if os.path.isdir(path) :
@@ -59,6 +61,8 @@ def replace (file_name, pattern, subst) :
global_dirname_03 = '01.scale_pert'
global_dirname_04 = '02.md'
+max_layer_numb = 50
+
def out_dir_name(jdata) :
super_cell = jdata['super_cell']
@@ -174,7 +178,7 @@ def poscar_scale (poscar_in, poscar_out, scale) :
with open(poscar_out, 'w') as fout:
fout.write("".join(lines))
-def poscar_elong (poscar_in, poscar_out, elong) :
+def poscar_elong (poscar_in, poscar_out, elong, shift_center=True) :
with open(poscar_in, 'r') as fin :
lines = list(fin)
if lines[7][0].upper() != 'C' :
@@ -185,8 +189,18 @@ def poscar_elong (poscar_in, poscar_out, elong) :
elong_ratio = elong / boxzl
boxz = boxz * (1. + elong_ratio)
lines[4] = '%.16e %.16e %.16e\n' % (boxz[0],boxz[1],boxz[2])
- with open(poscar_out, 'w') as fout:
- fout.write("".join(lines))
+ if shift_center:
+ poscar_str="".join(lines)
+ st=Structure.from_str(poscar_str,fmt='poscar')
+ cart_coords=st.cart_coords
+ z_mean=cart_coords[:,2].mean()
+ z_shift=st.lattice.c/2-z_mean
+ cart_coords[:,2]=cart_coords[:,2]+z_shift
+ nst=Structure(st.lattice,st.species,coords=cart_coords,coords_are_cartesian=True)
+ nst.to('poscar',poscar_out)
+ else:
+ with open(poscar_out, 'w') as fout:
+ fout.write("".join(lines))
def make_unit_cell (jdata) :
@@ -213,6 +227,12 @@ def make_super_cell_pymatgen (jdata) :
make_unit_cell(jdata)
out_dir = jdata['out_dir']
path_uc = os.path.join(out_dir, global_dirname_02)
+
+ elements=[Element(ii) for ii in jdata['elements']]
+ if 'vacuum_min' in jdata:
+ vacuum_min=jdata['vacuum_min']
+ else:
+ vacuum_min=max([float(ii.atomic_radius) for ii in elements])
from_poscar= jdata.get('from_poscar',False)
@@ -232,8 +252,14 @@ def make_super_cell_pymatgen (jdata) :
all_millers = jdata['millers']
path_sc = os.path.join(out_dir, global_dirname_02)
- #z_min = jdata['z_min']
- layer_numb = jdata['layer_numb']
+
+ user_layer_numb = None # set default value
+ z_min = None
+ if 'layer_numb' in jdata:
+ user_layer_numb = jdata['layer_numb']
+ else:
+ z_min = jdata['z_min']
+
super_cell = jdata['super_cell']
cwd = os.getcwd()
@@ -247,7 +273,16 @@ def make_super_cell_pymatgen (jdata) :
path_cur_surf = create_path('surf-'+miller_str)
os.chdir(path_cur_surf)
#slabgen = SlabGenerator(ss, miller, z_min, 1e-3)
- slab=general_surface.surface(ss,indices=miller,vacuum=1e-3,layers=layer_numb)
+ if user_layer_numb:
+ slab=general_surface.surface(ss,indices=miller,vacuum=vacuum_min,layers=user_layer_numb)
+ else:
+ # build slab according to z_min value
+ for layer_numb in range( 1,max_layer_numb+1):
+ slab=general_surface.surface(ss,indices=miller,vacuum=vacuum_min,layers=layer_numb)
+ if slab.cell.lengths()[-1] >= z_min:
+ break
+ if layer_numb == max_layer_numb:
+ raise RuntimeError("can't build the required slab")
#all_slabs = slabgen.get_slabs()
dlog.info(os.getcwd())
#dlog.info("Miller %s: The slab has %s termination, use the first one" %(str(miller), len(all_slabs)))
diff --git a/dpgen/dispatcher/ALI.py b/dpgen/dispatcher/ALI.py
index 5ceb5ad8d..f6e2c12ce 100644
--- a/dpgen/dispatcher/ALI.py
+++ b/dpgen/dispatcher/ALI.py
@@ -4,57 +4,387 @@
from aliyunsdkcore.acs_exception.exceptions import ServerException
from aliyunsdkecs.request.v20140526.RunInstancesRequest import RunInstancesRequest
from aliyunsdkecs.request.v20140526.DeleteInstancesRequest import DeleteInstancesRequest
-import time, json, os, glob
-from dpgen.dispatcher.Dispatcher import Dispatcher, _split_tasks
+from aliyunsdkecs.request.v20140526.DescribeAutoProvisioningGroupInstancesRequest import DescribeAutoProvisioningGroupInstancesRequest
+from aliyunsdkecs.request.v20140526.CreateAutoProvisioningGroupRequest import CreateAutoProvisioningGroupRequest
+from aliyunsdkecs.request.v20140526.DeleteAutoProvisioningGroupRequest import DeleteAutoProvisioningGroupRequest
+from aliyunsdkecs.request.v20140526.ModifyAutoProvisioningGroupRequest import ModifyAutoProvisioningGroupRequest
+from aliyunsdkecs.request.v20140526.DeleteLaunchTemplateRequest import DeleteLaunchTemplateRequest
+from aliyunsdkvpc.request.v20160428.DescribeVpcsRequest import DescribeVpcsRequest
+from aliyunsdkecs.request.v20140526.DescribeLaunchTemplatesRequest import DescribeLaunchTemplatesRequest
+from aliyunsdkecs.request.v20140526.CreateLaunchTemplateRequest import CreateLaunchTemplateRequest
+from aliyunsdkecs.request.v20140526.DescribeImagesRequest import DescribeImagesRequest
+from aliyunsdkecs.request.v20140526.DescribeSecurityGroupsRequest import DescribeSecurityGroupsRequest
+import time, json, os, glob, string, random
+from dpgen.dispatcher.Dispatcher import Dispatcher, _split_tasks, JobRecord
+from dpgen.dispatcher.SSHContext import SSHSession
from os.path import join
+from dpgen import dlog
+from hashlib import sha1
+
+def manual_delete(stage):
+ with open('machine-ali.json') as fp1:
+ mdata = json.load(fp1)
+ adata = mdata[stage][0]['machine']['ali_auth']
+ mdata_resources = mdata[stage][0]['resources']
+ mdata_machine = mdata[stage][0]['machine']
+ ali = ALI(adata, mdata_resources, mdata_machine, 0)
+ with open('apg_id.json', 'r') as fp2:
+ apg_id = json.load(fp2)
+ ali.apg_id = apg_id['apg_id']
+ ali.delete_apg()
+
+def manual_create(stage, machine_number):
+ with open('machine-ali.json') as fp:
+ mdata = json.load(fp)
+ adata = mdata[stage][0]['machine']['ali_auth']
+ mdata_resources = mdata[stage][0]['resources']
+ mdata_machine = mdata[stage][0]['machine']
+ ali = ALI(adata, mdata_resources, mdata_machine, machine_number)
+ ali.create_ess()
+ print(ali.ip_list)
class ALI():
- def __init__(self, adata, mdata_resources, mdata_machine, nchunks, work_path):
- self.ip_list = None
- self.instance_list = None
- self.dispatchers = None
+ def __init__(self, adata, mdata_resources, mdata_machine, nchunks):
+ self.ip_list = ["unalloc" for i in range(nchunks)]
+ self.instance_list = ["unalloc" for i in range(nchunks)]
+ self.dispatchers = [[None, "unalloc"] for i in range(nchunks)]
+ self.job_handlers = ["unalloc" for i in range(nchunks)]
+ self.task_chunks = None
self.adata = adata
+ self.apg_id = None
+ self.template_id = None
+ self.vsw_id = None
+ self.regionID = adata["regionID"]
+ self.client = AcsClient(adata["AccessKey_ID"], adata["AccessKey_Secret"], self.regionID)
self.mdata_resources = mdata_resources
self.mdata_machine = mdata_machine
self.nchunks = nchunks
- self.work_path = work_path
-
- def init(self):
- if self.check_restart():
+
+ def init(self, work_path, tasks, group_size):
+ if self.check_restart(work_path, tasks, group_size):
pass
else:
- self.create_machine()
- self.dispatchers = self.make_dispatchers()
+ self.create_ess()
+ self.make_dispatchers()
+
+ def create_ess(self):
+ img_id = self.get_image_id(self.adata["img_name"])
+ sg_id, vpc_id = self.get_sg_vpc_id()
+ self.template_id = self.create_template(img_id, sg_id, vpc_id)
+ self.vsw_id = self.get_vsw_id(vpc_id)
+ self.apg_id = self.create_apg()
+ dlog.info("begin to create ess, please wait two minutes")
+ time.sleep(120)
+ new_server_list = self.describe_apg_instances()
+ new_ip_list = self.get_ip(new_server_list)
+ for ii in range(len(new_server_list)):
+ profile = self.mdata_machine.copy()
+ profile['hostname'] = new_ip_list[ii]
+ profile['instance_id'] = new_server_list[ii]
+ if self.check_server(profile):
+ self.instance_list[ii] = new_server_list[ii]
+ self.ip_list[ii] = new_ip_list[ii]
+
+ def delete_apg(self):
+ request = DeleteAutoProvisioningGroupRequest()
+ request.set_accept_format('json')
+ request.set_AutoProvisioningGroupId(self.apg_id)
+ request.set_TerminateInstances(True)
+ try:
+ response = self.client.do_action_with_exception(request)
+ except ServerException as e:
+ dlog.info(e)
+ except ClientException as e:
+ dlog.info(e)
+
+ def create_apg(self):
+ request = CreateAutoProvisioningGroupRequest()
+ request.set_accept_format('json')
+ request.set_TotalTargetCapacity(str(self.nchunks))
+ request.set_LaunchTemplateId(self.template_id)
+ request.set_AutoProvisioningGroupName(self.adata["instance_name"] + ''.join(random.choice(string.ascii_uppercase) for _ in range(20)))
+ request.set_AutoProvisioningGroupType("maintain")
+ request.set_SpotAllocationStrategy("lowest-price")
+ request.set_SpotInstanceInterruptionBehavior("terminate")
+ request.set_SpotInstancePoolsToUseCount(1)
+ request.set_ExcessCapacityTerminationPolicy("termination")
+ request.set_TerminateInstances(True)
+ request.set_PayAsYouGoTargetCapacity("0")
+ request.set_SpotTargetCapacity(str(self.nchunks))
+ config = self.generate_config()
+ request.set_LaunchTemplateConfigs(config)
+ try:
+ response = self.client.do_action_with_exception(request)
+ response = json.loads(response)
+ with open('apg_id.json', 'w') as fp:
+ json.dump({'apg_id': response["AutoProvisioningGroupId"]}, fp, indent=4)
+ return response["AutoProvisioningGroupId"]
+ except ServerException as e:
+ dlog.info(e)
+ except ClientException as e:
+ dlog.info(e)
+
+ def update_server_list(self):
+ instance_list = self.describe_apg_instances()
+ return list(set(instance_list) - set(self.instance_list))
- def check_restart(self):
- dispatchers = []
+ def describe_apg_instances(self):
+ request = DescribeAutoProvisioningGroupInstancesRequest()
+ request.set_accept_format('json')
+ request.set_AutoProvisioningGroupId(self.apg_id)
+ request.set_PageSize(100)
+ iteration = self.nchunks // 100
instance_list = []
- if len(glob.glob(os.path.join(self.work_path, 'jr.*.json'))) == self.nchunks:
+ for i in range(iteration + 1):
+ request.set_PageNumber(i+1)
+ response = self.client.do_action_with_exception(request)
+ response = json.loads(response)
+ for ins in response["Instances"]["Instance"]:
+ instance_list.append(ins["InstanceId"])
+ return instance_list
+
+ def generate_config(self):
+ machine_config = self.adata["machine_type_price"]
+ config = []
+ for conf in machine_config:
+ for vsw in self.vsw_id:
+ tmp = {
+ "InstanceType": conf["machine_type"],
+ "MaxPrice": str(conf["price_limit"] * conf["numb"]),
+ "VSwitchId": vsw,
+ "WeightedCapacity": "1",
+ "Priority": str(conf["priority"])
+ }
+ config.append(tmp)
+ return config
+
+ def create_template(self, image_id, sg_id, vpc_id):
+ request = CreateLaunchTemplateRequest()
+ request.set_accept_format('json')
+ request.set_LaunchTemplateName(''.join(random.choice(string.ascii_uppercase) for _ in range(20)))
+ request.set_ImageId(image_id)
+ request.set_ImageOwnerAlias("self")
+ request.set_PasswordInherit(True)
+ request.set_InstanceType("ecs.c6.large")
+ request.set_InstanceName(self.adata["instance_name"])
+ request.set_SecurityGroupId(sg_id)
+ request.set_VpcId(vpc_id)
+ request.set_SystemDiskCategory("cloud_efficiency")
+ request.set_SystemDiskSize(40)
+ request.set_IoOptimized("optimized")
+ request.set_InstanceChargeType("PostPaid")
+ request.set_NetworkType("vpc")
+ request.set_SpotStrategy("SpotWithPriceLimit")
+ request.set_SpotPriceLimit(100)
+ try:
+ response = self.client.do_action_with_exception(request)
+ response = json.loads(response)
+ return response["LaunchTemplateId"]
+ except ServerException as e:
+ dlog.info(e)
+ except ClientException as e:
+ dlog.info(e)
+
+ def delete_template(self):
+ request = DeleteLaunchTemplateRequest()
+ request.set_accept_format('json')
+ request.set_LaunchTemplateId(self.template_id)
+ response = self.client.do_action_with_exception(request)
+
+ def get_image_id(self, img_name):
+ request = DescribeImagesRequest()
+ request.set_accept_format('json')
+ request.set_ImageOwnerAlias("self")
+ response = self.client.do_action_with_exception(request)
+ response = json.loads(response)
+ for img in response["Images"]["Image"]:
+ if img["ImageName"] == img_name:
+ return img["ImageId"]
+
+ def get_sg_vpc_id(self):
+ request = DescribeSecurityGroupsRequest()
+ request.set_accept_format('json')
+ response = self.client.do_action_with_exception(request)
+ response = json.loads(response)
+ for sg in response["SecurityGroups"]["SecurityGroup"]:
+ if sg["SecurityGroupName"] == "sg":
+ return sg["SecurityGroupId"], sg["VpcId"]
+
+ def get_vsw_id(self, vpc_id):
+ request = DescribeVpcsRequest()
+ request.set_accept_format('json')
+ request.set_VpcId(vpc_id)
+ response = self.client.do_action_with_exception(request)
+ response = json.loads(response)
+ for vpc in response["Vpcs"]["Vpc"]:
+ if vpc["VpcId"] == vpc_id:
+ return vpc["VSwitchIds"]["VSwitchId"]
+
+ def change_apg_capasity(self, capasity):
+ request = ModifyAutoProvisioningGroupRequest()
+ request.set_accept_format('json')
+ request.set_AutoProvisioningGroupId(self.apg_id)
+ request.set_TotalTargetCapacity(str(capasity))
+ request.set_SpotTargetCapacity(str(capasity))
+ request.set_PayAsYouGoTargetCapacity("0")
+ response = self.client.do_action_with_exception(request)
+
+ def check_spot_callback(self, instance_id):
+ request = DescribeInstancesRequest()
+ request.set_accept_format('json')
+ request.set_InstanceIds([instance_id])
+ status = False
+ try:
+ response = self.client.do_action_with_exception(request)
+ response = json.loads(response)
+ if len(response["Instances"]["Instance"]) == 1 and "Recycling" in response["Instances"]["Instance"][0]["OperationLocks"]["LockReason"]:
+ status = True
+ if instance_id not in self.describe_apg_instances():
+ status = True
+ except:
+ pass
+ return status
+
+ def check_restart(self, work_path, tasks, group_size):
+ if os.path.exists('apg_id.json'):
+ with open('apg_id.json') as fp:
+ apg = json.load(fp)
+ self.apg_id = apg["apg_id"]
+ self.task_chunks = _split_tasks(tasks, group_size)
+ task_chunks_str = ['+'.join(ii) for ii in self.task_chunks]
+ task_hashes = [sha1(ii.encode('utf-8')).hexdigest() for ii in task_chunks_str]
+ nchunks = len(self.task_chunks)
+ for ii in range(nchunks):
+ fn = 'jr.%.06d.json' % ii
+ if not os.path.exists(os.path.join(os.path.abspath(work_path), fn)):
+ pass
+ else:
+ job_record = JobRecord(work_path, self.task_chunks[ii], fname = fn)
+ cur_chunk = self.task_chunks[ii]
+ cur_hash = task_hashes[ii]
+ if not job_record.check_finished(cur_hash):
+ with open(os.path.join(work_path, fn)) as fp:
+ jr = json.load(fp)
+ ip = jr[cur_hash]['context'][3]
+ instance_id = jr[cur_hash]['context'][4]
+ profile = self.mdata_machine.copy()
+ profile['hostname'] = ip
+ profile['instance_id'] = instance_id
+ if self.check_server(profile):
+ disp = Dispatcher(profile, context_type='ssh', batch_type='shell', job_record='jr.%.06d.json' % ii)
+ self.dispatchers[ii] = [disp, "working"]
+ self.ip_list[ii] = ip
+ self.instance_list[ii] = instance_id
+ # spot callback
+ else:
+ os.remove(os.path.join(work_path, fn))
+ else:
+ self.ip_list[ii] = "finished"
+ self.instance_list[ii] = "finished"
+ self.dispatchers[ii] = [None, "finished"]
+ return True
+ else:
+ self.task_chunks = _split_tasks(tasks, group_size)
+ return False
+
+ def get_ip(self, instance_list):
+ request = DescribeInstancesRequest()
+ request.set_accept_format('json')
+ ip_list = []
+ if len(instance_list) <= 10:
+ for i in range(len(instance_list)):
+ request.set_InstanceIds([instance_list[i]])
+ response = self.client.do_action_with_exception(request)
+ response = json.loads(response)
+ ip_list.append(response["Instances"]["Instance"][0]["VpcAttributes"]["PrivateIpAddress"]['IpAddress'][0])
+ else:
+ iteration = len(instance_list) // 10
+ for i in range(iteration):
+ for j in range(10):
+ request.set_InstanceIds([instance_list[i*10+j]])
+ response = self.client.do_action_with_exception(request)
+ response = json.loads(response)
+ ip_list.append(response["Instances"]["Instance"][0]["VpcAttributes"]["PrivateIpAddress"]['IpAddress'][0])
+ if len(instance_list) - iteration * 10 != 0:
+ for j in range(len(instance_list) - iteration * 10):
+ request.set_InstanceIds([instance_list[iteration*10+j]])
+ response = self.client.do_action_with_exception(request)
+ response = json.loads(response)
+ ip_list.append(response["Instances"]["Instance"][0]["VpcAttributes"]["PrivateIpAddress"]['IpAddress'][0])
+ return ip_list
+
+ def get_finished_job_num(self):
+ finished_num = 0
+ for ii in range(len(self.dispatchers)):
+ if self.dispatchers[ii][1] == "finished":
+ finished_num += 1
+ return finished_num
+
+ def resubmission(self, machine_exception_num):
+ if self.adata["img_name"] == "kit":
+ new_ip_list = []
+ try:
+ new_server_list = self.update_server_list()
+ new_ip_list = self.get_ip(new_server_list)
+ except:
+ pass
for ii in range(self.nchunks):
- with open(os.path.join(self.work_path, 'jr.%.06d.json' % ii)) as fp:
- job_record = json.load(fp)
- key = list(job_record.keys())[0]
- ip, instance_id = job_record[key]['context'][-2], job_record[key]['context'][-1]
- instance_list.append(instance_id)
+ if len(new_ip_list) == 0:
+ break
+ if self.dispatchers[ii][1] == "exception":
+ self.ip_list[ii] = new_ip_list.pop()
+ self.instance_list[ii] = new_server_list.pop()
profile = self.mdata_machine.copy()
- profile['hostname'] = ip
- profile['instance_id'] = instance_id
- disp = Dispatcher(profile, context_type='ssh', batch_type='shell', job_record='jr.%.06d.json' % ii)
- max_check = 10
- cnt = 0
- while not disp.session._check_alive():
- cnt += 1
- if cnt == max_check:
- break
- if cnt != max_check:
- dispatchers.append(disp)
- restart = False
- if len(dispatchers) == self.nchunks:
- restart = True
- self.dispatchers = dispatchers
- self.instance_list = instance_list
- return restart
+ profile["hostname"] = self.ip_list[ii]
+ profile["instance_id"] = self.instance_list[ii]
+ self.dispatchers[ii] = [Dispatcher(profile, context_type='ssh', batch_type='shell', job_record="jr.%.06d.json" % ii), "working"]
+ dlog.info(self.ip_list[ii])
+ job_handler = self.dispatchers[ii][0].submit_jobs(resources,
+ command,
+ work_path,
+ self.task_chunks[ii],
+ group_size,
+ forward_common_files,
+ forward_task_files,
+ backward_task_files,
+ forward_task_deference,
+ outlog,
+ errlog)
+ self.job_handlers[ii] = job_handler
+ elif self.adata["img_name"] == "vasp":
+ if machine_exception_num / self.nchunks > 0.05:
+ self.change_apg_capasity(self.nchunks - self.get_finished_job_num() + machine_exception_num)
+ time.sleep(120)
+ new_ip_list = []
+ try:
+ new_server_list = self.update_server_list()
+ new_ip_list = self.get_ip(new_server_list)
+ except:
+ pass
+ if len(new_ip_list) == machine_exception_num:
+ dlog.info("new submission of callback machine")
+ for ii in range(self.nchunks):
+ self.ip_list[ii] = new_ip_list.pop()
+ self.instance_list[ii] = new_server_list.pop()
+ profile = self.mdata_machine.copy()
+ profile["hostname"] = self.ip_list[ii]
+ profile["instance_id"] = self.instance_list[ii]
+ self.dispatchers[ii] = [Dispatcher(profile, context_type='ssh', batch_type='shell', job_record="jr.%.06d.json" % ii), "working"]
+ dlog.info(self.ip_list[ii])
+ job_handler = self.dispatchers[ii][0].submit_jobs(resources,
+ command,
+ work_path,
+ self.task_chunks[ii],
+ group_size,
+ forward_common_files,
+ forward_task_files,
+ backward_task_files,
+ forward_task_deference,
+ outlog,
+ errlog)
+ self.job_handlers[ii] = job_handler
+ #@profile(precision=6)
def run_jobs(self,
resources,
command,
@@ -65,82 +395,144 @@ def run_jobs(self,
forward_task_files,
backward_task_files,
forward_task_deference = True,
+ mark_failure = False,
outlog = 'log',
errlog = 'err'):
- task_chunks = _split_tasks(tasks, group_size)
- job_handlers = []
for ii in range(self.nchunks):
- job_handler = self.dispatchers[ii].submit_jobs(resources,
- command,
- work_path,
- task_chunks[ii],
- group_size,
- forward_common_files,
- forward_task_files,
- backward_task_files,
- forward_task_deference,
- outlog,
- errlog)
- job_handlers.append(job_handler)
+ if self.dispatchers[ii][1] == "working":
+ dlog.info(self.ip_list[ii])
+ job_handler = self.dispatchers[ii][0].submit_jobs(resources,
+ command,
+ work_path,
+ self.task_chunks[ii],
+ group_size,
+ forward_common_files,
+ forward_task_files,
+ backward_task_files,
+ forward_task_deference,
+ outlog,
+ errlog)
+ self.job_handlers[ii] = job_handler
+ machine_exception_num = 0
while True:
- cnt = 0
+ new_ip_list = []
+ try:
+ new_server_list = self.update_server_list()
+ new_ip_list = self.get_ip(new_server_list)
+ except:
+ pass
+ if machine_exception_num > 0:
+ self.resubmission(machine_exception_num)
for ii in range(self.nchunks):
- if self.dispatchers[ii].all_finished(job_handlers[ii]):
- cnt += 1
- if cnt == self.nchunks:
+ if self.dispatchers[ii][1] == "working":
+ if self.check_spot_callback(self.instance_list[ii]):
+ machine_exception_num += 1
+ dlog.info("machine %s callback" % self.instance_list[ii])
+ os.remove(self.job_handlers[ii]["job_record"].fname)
+ self.job_handlers[ii] = "exception"
+ self.ip_list[ii] = "exception"
+ self.instance_list[ii] = "exception"
+ self.dispatchers[ii][1] = "exception"
+ if self.adata["img_name"] == "vasp":
+ self.change_apg_capasity(self.nchunks - self.get_finished_job_num() - machine_exception_num)
+ continue
+ if self.check_server(self.dispatchers[ii][0].remote_profile):
+ if self.dispatchers[ii][0].all_finished(self.job_handlers[ii], mark_failure, False):
+ self.delete(ii)
+ self.dispatchers[ii][1] = "finished"
+ self.ip_list[ii] = "finished"
+ self.instance_list[ii] = "finished"
+ self.change_apg_capasity(self.nchunks - self.get_finished_job_num())
+ else:
+ dlog.info("ssh exception accured in %s" % self.ip_list[ii])
+ elif self.dispatchers[ii][1] == "finished":
+ continue
+ elif self.dispatchers[ii][1] == "unalloc":
+ if new_ip_list:
+ profile = self.mdata_machine.copy()
+ profile["hostname"] = new_ip_list[0]
+ profile["instance_id"] = new_server_list[0]
+ if self.check_server(profile):
+ self.ip_list[ii] = new_ip_list.pop(0)
+ self.instance_list[ii] = new_server_list.pop(0)
+ self.dispatchers[ii] = [Dispatcher(profile, context_type='ssh', batch_type='shell', job_record='jr.%.06d.json' % ii), "working"]
+ dlog.info(self.ip_list[ii])
+ job_handler = self.dispatchers[ii][0].submit_jobs(resources,
+ command,
+ work_path,
+ self.task_chunks[ii],
+ group_size,
+ forward_common_files,
+ forward_task_files,
+ backward_task_files,
+ forward_task_deference,
+ outlog,
+ errlog)
+ self.job_handlers[ii] = job_handler
+ if self.check_dispatcher_finished():
+ os.remove('apg_id.json')
+ self.delete_template()
+ self.delete_apg()
break
else:
time.sleep(10)
- self.delete_machine()
+
+# status = ["unalloc", "working", "finished", "exception"]
+ def check_server(self, profile):
+ try:
+ session = SSHSession(profile)
+ session.close()
+ return True
+ except:
+ return False
+
+ def check_dispatcher_finished(self):
+ count = 0
+ flag = True
+ for ii in range(len(self.dispatchers)):
+ if self.dispatchers[ii][1] == "unalloc" or self.dispatchers[ii][1] == "working": flag = False
+ if self.dispatchers[ii][1] == "exception": count += 1
+ if self.adata["img_name"] == "vasp" and count / self.nchunks > 0.05: flag = False
+ elif self.adata["img_name"] == "kit" and count > 0: flag = False
+ return flag
+
+ def delete(self, ii):
+ request = DeleteInstancesRequest()
+ request.set_accept_format('json')
+ request.set_InstanceIds([self.instance_list[ii]])
+ request.set_Force(True)
+ response = self.client.do_action_with_exception(request)
def make_dispatchers(self):
- dispatchers = []
for ii in range(self.nchunks):
- profile = self.mdata_machine.copy()
- profile['hostname'] = self.ip_list[ii]
- profile['instance_id'] = self.instance_list[ii]
- disp = Dispatcher(profile, context_type='ssh', batch_type='shell', job_record='jr.%.06d.json' % ii)
- dispatchers.append(disp)
- return dispatchers
-
- def create_machine(self):
- AccessKey_ID = self.adata["AccessKey_ID"]
- AccessKey_Secret = self.adata["AccessKey_Secret"]
- strategy = self.adata["pay_strategy"]
- pwd = self.adata["password"]
- regionID = self.mdata_machine['regionID']
- template_name = '%s_%s_%s' % (self.mdata_resources['partition'], self.mdata_resources['numb_gpu'], strategy)
- instance_name = self.adata["instance_name"]
- client = AcsClient(AccessKey_ID,AccessKey_Secret, regionID)
- request = RunInstancesRequest()
- request.set_accept_format('json')
- request.set_UniqueSuffix(True)
- request.set_Password(pwd)
- request.set_InstanceName(instance_name)
- request.set_Amount(self.nchunks)
- request.set_LaunchTemplateName(template_name)
- response = client.do_action_with_exception(request)
- response = json.loads(response)
- self.instance_list = response["InstanceIdSets"]["InstanceIdSet"]
- time.sleep(50)
- request = DescribeInstancesRequest()
- request.set_accept_format('json')
- request.set_InstanceIds(self.instance_list)
- response = client.do_action_with_exception(request)
- response = json.loads(response)
- ip = []
- for i in range(len(response["Instances"]["Instance"])):
- ip.append(response["Instances"]["Instance"][i]["PublicIpAddress"]['IpAddress'][0])
- self.ip_list = ip
-
+ if self.ip_list[ii] != "unalloc":
+ profile = self.mdata_machine.copy()
+ profile['hostname'] = self.ip_list[ii]
+ profile['instance_id'] = self.instance_list[ii]
+ if self.check_server(profile):
+ self.dispatchers[ii] = [Dispatcher(profile, context_type='ssh', batch_type='shell', job_record='jr.%.06d.json' % ii), "working"]
+ else:
+ continue
+ #self.dispatchers[ii] = [Dispatcher(profile, context_type='ssh', batch_type='shell', job_record='jr.%.06d.json' % ii), "working"]
+
def delete_machine(self):
- AccessKey_ID = self.adata["AccessKey_ID"]
- AccessKey_Secret = self.adata["AccessKey_Secret"]
- regionID = self.mdata_machine['regionID']
- client = AcsClient(AccessKey_ID,AccessKey_Secret, regionID)
request = DeleteInstancesRequest()
request.set_accept_format('json')
- request.set_InstanceIds(self.instance_list)
- request.set_Force(True)
- response = client.do_action_with_exception(request)
-
+ if len(self.instance_list) <= 100:
+ request.set_InstanceIds(self.instance_list)
+ request.set_Force(True)
+ response = self.client.do_action_with_exception(request)
+ else:
+ iteration = len(self.instance_list) // 100
+ for i in range(iteration):
+ request.set_InstanceIds(self.instance_list[i*100:(i+1)*100])
+ request.set_Force(True)
+ response = self.client.do_action_with_exception(request)
+ if len(self.instance_list) - iteration * 100 != 0:
+ request.set_InstanceIds(self.instance_list[iteration*100:])
+ request.set_Force(True)
+ response = self.client.do_action_with_exception(request)
+ self.instance_list = []
+ self.ip_list = []
+ os.remove('machine_record.json')
+ dlog.debug("Successfully free the machine!")
diff --git a/dpgen/dispatcher/Batch.py b/dpgen/dispatcher/Batch.py
index 0b6d16b71..02b09e68e 100644
--- a/dpgen/dispatcher/Batch.py
+++ b/dpgen/dispatcher/Batch.py
@@ -7,13 +7,16 @@
class Batch(object) :
def __init__ (self,
context,
- uuid_names = False) :
+ uuid_names = True) :
self.context = context
+ self.uuid_names = uuid_names
if uuid_names:
+ self.upload_tag_name = '%s_tag_upload' % self.context.job_uuid
self.finish_tag_name = '%s_tag_finished' % self.context.job_uuid
self.sub_script_name = '%s.sub' % self.context.job_uuid
self.job_id_name = '%s_job_id' % self.context.job_uuid
else:
+ self.upload_tag_name = 'tag_upload'
self.finish_tag_name = 'tag_finished'
self.sub_script_name = 'run.sub'
self.job_id_name = 'job_id'
@@ -99,7 +102,6 @@ def submit(self,
args = None,
res = None,
restart = False,
- sleep = 0,
outlog = 'log',
errlog = 'err'):
if restart:
@@ -119,7 +121,11 @@ def submit(self,
else:
dlog.debug('new task')
self.do_submit(job_dirs, cmd, args, res, outlog=outlog, errlog=errlog)
- time.sleep(sleep) # For preventing the crash of the tasks while submitting
+ if res is None:
+ sleep = 0
+ else:
+ sleep = res.get('submit_wait_time', 0)
+ time.sleep(sleep) # For preventing the crash of the tasks while submitting
def check_finish_tag(self) :
return self.context.check_file_exists(self.finish_tag_name)
@@ -133,19 +139,17 @@ def _sub_script_inner(self,
outlog = 'log',
errlog = 'err') :
ret = ""
- try:
- allow_failure = res['allow_failure']
- except:
- allow_failure = False
+ allow_failure = res.get('allow_failure', False)
for ii,jj in zip(job_dirs, args) :
ret += 'cd %s\n' % ii
- ret += 'test $? -ne 0 && exit\n\n'
+ ret += 'test $? -ne 0 && exit 1\n\n'
if self.manual_cuda_devices <= 0:
ret += 'if [ ! -f tag_%d_finished ] ;then\n' % idx
ret += ' %s 1>> %s 2>> %s \n' % (self.sub_script_cmd(cmd, jj, res), outlog, errlog)
if res['allow_failure'] is False:
- ret += ' if test $? -ne 0; then exit; else touch tag_%d_finished; fi \n' % idx
+ ret += ' if test $? -ne 0; then exit 1; else touch tag_%d_finished; fi \n' % idx
else :
+ ret += ' if test $? -ne 0; then touch tag_failure_%d; fi \n' % idx
ret += ' touch tag_%d_finished \n' % idx
ret += 'fi\n\n'
else :
@@ -154,7 +158,7 @@ def _sub_script_inner(self,
ret += 'CUDA_VISIBLE_DEVICES=%d %s &\n\n' % ((self.cmd_cnt % self.manual_cuda_devices), tmp_cmd)
self.cmd_cnt += 1
ret += 'cd %s\n' % self.context.remote_root
- ret += 'test $? -ne 0 && exit\n'
+ ret += 'test $? -ne 0 && exit 1\n'
if self.manual_cuda_devices > 0 and self.cmd_cnt % (self.manual_cuda_devices * self.manual_cuda_multiplicity) == 0:
ret += '\nwait\n\n'
ret += '\nwait\n\n'
diff --git a/dpgen/dispatcher/Dispatcher.py b/dpgen/dispatcher/Dispatcher.py
index 915d7e201..444f3431e 100644
--- a/dpgen/dispatcher/Dispatcher.py
+++ b/dpgen/dispatcher/Dispatcher.py
@@ -1,5 +1,4 @@
import os,sys,time,random,json,glob
-
from dpgen.dispatcher.LocalContext import LocalSession
from dpgen.dispatcher.LocalContext import LocalContext
from dpgen.dispatcher.LazyLocalContext import LazyLocalContext
@@ -36,10 +35,11 @@ def __init__ (self,
batch_type = 'slurm',
job_record = 'jr.json'):
self.remote_profile = remote_profile
+
if context_type == 'local':
self.session = LocalSession(remote_profile)
self.context = LocalContext
- self.uuid_names = False
+ self.uuid_names = True
elif context_type == 'lazy-local':
self.session = None
self.context = LazyLocalContext
@@ -47,7 +47,7 @@ def __init__ (self,
elif context_type == 'ssh':
self.session = SSHSession(remote_profile)
self.context = SSHContext
- self.uuid_names = False
+ self.uuid_names = True
else :
raise RuntimeError('unknown context')
if batch_type == 'slurm':
@@ -74,6 +74,7 @@ def run_jobs(self,
forward_task_files,
backward_task_files,
forward_task_deference = True,
+ mark_failure = False,
outlog = 'log',
errlog = 'err') :
job_handler = self.submit_jobs(resources,
@@ -87,8 +88,8 @@ def run_jobs(self,
forward_task_deference,
outlog,
errlog)
- while not self.all_finished(job_handler) :
- time.sleep(10)
+ while not self.all_finished(job_handler, mark_failure) :
+ time.sleep(60)
# delete path map file when job finish
# _pmap.delete()
@@ -134,13 +135,14 @@ def submit_jobs(self,
batch = self.batch(context, uuid_names = self.uuid_names)
rjob = {'context':context, 'batch':batch}
# upload files
- if not rjob['context'].check_file_exists('tag_upload'):
+ if not rjob['context'].check_file_exists(rjob['batch'].upload_tag_name):
rjob['context'].upload('.',
forward_common_files)
rjob['context'].upload(cur_chunk,
forward_task_files,
dereference = forward_task_deference)
- rjob['context'].write_file('tag_upload', '')
+
+ rjob['context'].write_file(rjob['batch'].upload_tag_name, '')
dlog.debug('uploaded files for %s' % task_chunks_str[ii])
# submit new or recover old submission
if not submitted:
@@ -155,10 +157,9 @@ def submit_jobs(self,
job_list.append(rjob)
ip = None
instance_id = None
- if "type" in self.remote_profile:
- if self.remote_profile['type'] == 'ALI':
- ip = self.remote_profile['hostname']
- instance_id = self.remote_profile['instance_id']
+ if 'ali_auth' in self.remote_profile:
+ ip = self.remote_profile['hostname']
+ instance_id = self.remote_profile['instance_id']
job_record.record_remote_context(cur_hash,
context.local_root,
context.remote_root,
@@ -184,13 +185,16 @@ def submit_jobs(self,
def all_finished(self,
- job_handler):
+ job_handler,
+ mark_failure,
+ clean=True):
task_chunks = job_handler['task_chunks']
task_chunks_str = ['+'.join(ii) for ii in task_chunks]
task_hashes = [sha1(ii.encode('utf-8')).hexdigest() for ii in task_chunks_str]
job_list = job_handler['job_list']
job_record = job_handler['job_record']
command = job_handler['command']
+ tag_failure_list = ['tag_failure_%d' % ii for ii in range(len(command))]
resources = job_handler['resources']
outlog = job_handler['outlog']
errlog = job_handler['errlog']
@@ -214,8 +218,13 @@ def all_finished(self,
rjob['batch'].submit(task_chunks[idx], command, res = resources, outlog=outlog, errlog=errlog,restart=True)
elif status == JobStatus.finished :
dlog.info('job %s finished' % job_uuid)
- rjob['context'].download(task_chunks[idx], backward_task_files)
- rjob['context'].clean()
+ if mark_failure:
+ rjob['context'].download(task_chunks[idx], tag_failure_list, check_exists = True, mark_failure = False)
+ rjob['context'].download(task_chunks[idx], backward_task_files, check_exists = True)
+ else:
+ rjob['context'].download(task_chunks[idx], backward_task_files)
+ if clean:
+ rjob['context'].clean()
job_record.record_finish(cur_hash)
job_record.dump()
job_record.dump()
@@ -295,23 +304,28 @@ def _new_record(self):
}
-def make_dispatcher(mdata):
- try:
- hostname = mdata['hostname']
- context_type = 'ssh'
- except:
- context_type = 'local'
- try:
- batch_type = mdata['batch']
- except:
- dlog.info('cannot find key "batch" in machine file, try to use deprecated key "machine_type"')
- batch_type = mdata['machine_type']
- try:
- lazy_local = mdata['lazy_local']
- except:
- lazy_local = False
- if lazy_local and context_type == 'local':
- dlog.info('Dispatcher switches to the lazy local mode')
- context_type = 'lazy-local'
- disp = Dispatcher(mdata, context_type=context_type, batch_type=batch_type)
- return disp
+def make_dispatcher(mdata, mdata_resource=None, work_path=None, run_tasks=None, group_size=None):
+ if 'ali_auth' in mdata:
+ from dpgen.dispatcher.ALI import ALI
+ nchunks = len(_split_tasks(run_tasks, group_size))
+ dispatcher = ALI(mdata['ali_auth'], mdata_resource, mdata, nchunks)
+ dispatcher.init(work_path, run_tasks, group_size)
+ return dispatcher
+ else:
+ hostname = mdata.get('hostname', None)
+ #use_uuid = mdata.get('use_uuid', False)
+ if hostname:
+ context_type = 'ssh'
+ else:
+ context_type = 'local'
+ try:
+ batch_type = mdata['batch']
+ except:
+ dlog.info('cannot find key "batch" in machine file, try to use deprecated key "machine_type"')
+ batch_type = mdata['machine_type']
+ lazy_local = (mdata.get('lazy-local', False)) or (mdata.get('lazy_local', False))
+ if lazy_local and context_type == 'local':
+ dlog.info('Dispatcher switches to the lazy local mode')
+ context_type = 'lazy-local'
+ disp = Dispatcher(mdata, context_type=context_type, batch_type=batch_type)
+ return disp
diff --git a/dpgen/dispatcher/DispatcherList.py b/dpgen/dispatcher/DispatcherList.py
new file mode 100644
index 000000000..a62bd2107
--- /dev/null
+++ b/dpgen/dispatcher/DispatcherList.py
@@ -0,0 +1,153 @@
+from dpgen.dispatcher.Dispatcher import Dispatcher, _split_tasks, JobRecord
+
+class Entity():
+ def __init__(self, ip, instance_id, job_handler=None):
+ self.ip = ip
+ self.instance_id = instance_id
+ self.job_handler = job_handler
+
+class DispatcherList():
+ def __init__(self, mdata_machine, mdata_resources, nchunks, cloud_resources=None):
+ self.mdata_machine = mdata_machine
+ self.mdata_resources = mdata_resources
+ self.nchunks = nchunks
+ self.cloud_resources = cloud_resources
+ self.dispatcher_list = list({"dispatcher": None,
+ "dispatcher_status": "unallocated",
+ "entity": None} for ii in range(nchunks))
+ # Base
+ def init(self):
+ for ii in range(self.nchunks):
+ self.create(ii)
+
+ # Base
+ def run_jobs(self,
+ resources,
+ command,
+ work_path,
+ tasks,
+ group_size,
+ forward_common_files,
+ forward_task_files,
+ backward_task_files,
+ forward_task_deference = True,
+ mark_failure = False,
+ outlog = 'log',
+ errlog = 'err'):
+ task_chunks = _split_tasks(tasks, group_size)
+ while True:
+ if self.check_all_dispatchers_finished():
+ break
+ ratio_failure = self.mdata_resources["ratio_failure"]
+ self.exception_handling(ratio_failure)
+ for ii in range(self.nchunks):
+ dispatcher_status = self.check_dispatcher_status(ii)
+ if dispatcher_status == "unsubmitted":
+ self.dispatcher_list[ii]["job_handler"] = self.dispatcher_list[ii]["dispatcher"].submit_jobs(resources,
+ command,
+ work_path,
+ task_chunks[ii],
+ group_size,
+ forward_common_files,
+ forward_task_files,
+ backward_task_files,
+ forward_task_deference,
+ outlog,
+ errlog)
+ self.dispatcher_list[ii]["dispatcher_status"] == "running"
+ elif dispatcher_status == "finished":
+ # to do
+ # no jobs in queue, delete current machine
+ # else allocate current machine to unalloc dispatcher
+ pass
+ elif dispatcher_status == "running":
+ pass
+ elif dispatcher_status == "unallocated":
+ # to do: if we can create machine, then make dispatcher
+ # else pass
+ pass
+ elif dispatcher_status == "terminated":
+ pass
+
+ # Derivate
+ def create(self, ii):
+ '''if jr.json existed and job not finished, use jr.json to rebuild dispatcher
+ else create one machine, then make_dispatcher, change status from unallocated to unsubmitted'''
+ if not os.path.exists(os.path.join(os.path.abspath(work_path), "jr.%.06d.json" % ii)):
+ # to do: create machine, make dispatcher, change status from unallocated to unsubmitted
+ pass
+ else:
+ task_chunks = _split_tasks(tasks, group_size)
+ task_chunks_str = ['+'.join(ii) for ii in self.task_chunks]
+ task_hashes = [sha1(ii.encode('utf-8')).hexdigest() for ii in task_chunks_str]
+ job_record = JobRecord(work_path, task_chunks[ii], fname = "jr.%.06d.json" % ii)
+ cur_chunk = self.task_chunks[ii]
+ cur_hash = task_hashes[ii]
+ if not job_record.check_finished(cur_hash):
+ with open(os.path.join(work_path, fn)) as fp:
+ jr = json.load(fp)
+ self.dispatcher_list[ii]["entity"] = Entity(jr[cur_hash]['context'][3], jr[cur_hash]['context'][4])
+ self.make_dispatcher(ii)
+ # Derivate
+ def delete(self):
+ '''delete one machine'''
+ pass
+
+ # Base
+ def check_all_dispatchers_finished(self, ratio_failure=0):
+ exception_num = 0
+ for ii in range(self.nchunks):
+ if self.dispatcher_list[ii]["dispatcher_status"] == "terminated":
+ exception_num += 1
+ if exception_num / self.nchunks > ratio_failure: return False
+ else: return True
+
+ # Base
+ def exception_handling(self, ratio_failure):
+ terminated_num = 0
+ for ii in range(self.nchunks):
+ if self.dispatcher_list[ii]["dispatcher_status"] == "terminated":
+ terminated_num += 1
+ if terminated_num / self.nchunks > ratio_failure:
+ self.resubmit()
+
+ # Derivate
+ def resubmit(self):
+ '''create machines
+ make dispatcher
+ change status from terminated to unsubmitted'''
+ pass
+
+ # Base
+ def make_dispatcher(self, ii):
+ '''use entity to distinguish machine, for example if ip isn't None, means we can make_dispatcher
+ change status from unallocated to unsubmitted'''
+ entity = self.dispatcher_list[ii]["entity"]
+ if entity.ip:
+ profile = self.mdata_machine.copy()
+ profile['hostname'] = entity.ip
+ profile['instance_id'] = entity.instance_id
+ self.dispatcher_list[ii]["dispatcher"] = Dispatcher(profile, context_type='ssh', batch_type='shell', job_record='jr.%.06d.json' % ii)
+ self.dispatcher_list[ii]["dispatcher_status"] = "unsubmitted"
+
+ # Base
+ def check_dispatcher_status(self, ii):
+ '''catch running dispatcher exception
+ if no exception occured, check finished'''
+ if self.dispatcher_list[ii]["dispatcher_status"] == "running":
+ if not self.catch_dispatcher_exception():
+ if self.dispatcher_list[ii]["dispatcher"].all_finished():
+ self.dispatcher_list[ii]["dispatcher_status"] = "finished"
+ else:
+ self.dispatcher_list[ii]["dispatcher_status"] = "terminated"
+ return self.dispatcher_list[ii]["dispatcher_status"]
+
+ # Derivate
+ def catch_dispatcher_exception(self):
+ pass
+
+
+
+
+
+
diff --git a/dpgen/dispatcher/LSF.py b/dpgen/dispatcher/LSF.py
index 62b46e99b..7c5d05854 100644
--- a/dpgen/dispatcher/LSF.py
+++ b/dpgen/dispatcher/LSF.py
@@ -34,9 +34,9 @@ def check_status(self):
status_word = status_line.split()[2]
# ref: https://www.ibm.com/support/knowledgecenter/en/SSETD4_9.1.2/lsf_command_ref/bjobs.1.html
- if status_word in ["PEND", "WAIT"] :
+ if status_word in ["PEND", "WAIT", "PSUSP"] :
return JobStatus.waiting
- elif status_word in ["RUN"] :
+ elif status_word in ["RUN", "USUSP"] :
return JobStatus.running
elif status_word in ["DONE","EXIT"] :
if self.check_finish_tag() :
@@ -102,14 +102,19 @@ def sub_script_head(self, res):
ret = ''
ret += "#!/bin/bash -l\n#BSUB -e %J.err\n#BSUB -o %J.out\n"
if res['numb_gpu'] == 0:
- ret += '#BSUB -R span[ptile=%d]\n#BSUB -n %d\n' % (
- res['node_cpu'], res['numb_node'] * res['task_per_node'])
+ ret += '#BSUB -n %d\n#BSUB -R span[ptile=%d]\n' % (
+ res['numb_node'] * res['task_per_node'], res['node_cpu'])
else:
if res['node_cpu']:
ret += '#BSUB -R span[ptile=%d]\n' % res['node_cpu']
- # It is selected only for the situation that GPU is related to CPU node.
- ret += '#BSUB -R "select[ngpus >0] rusage[ngpus_excl_p=1]"\n#BSUB -n %d\n' % (
- res['numb_gpu'])
+ if res.get('new_lsf_gpu', False):
+ # supportted in LSF >= 10.1.0 SP6
+ # ref: https://www.ibm.com/support/knowledgecenter/en/SSWRJV_10.1.0/lsf_resource_sharing/use_gpu_res_reqs.html
+ ret += '#BSUB -n %d\n#BSUB -gpu "num=%d:mode=shared:j_exclusive=yes"\n' % (
+ res['numb_gpu'], res['task_per_node'])
+ else:
+ ret += '#BSUB -n %d\n#BSUB -R "select[ngpus >0] rusage[ngpus_excl_p=%d]"\n' % (
+ res['numb_gpu'], res['task_per_node'])
if res['time_limit']:
ret += '#BSUB -W %s\n' % (res['time_limit'].split(':')[
0] + ':' + res['time_limit'].split(':')[1])
diff --git a/dpgen/dispatcher/LazyLocalContext.py b/dpgen/dispatcher/LazyLocalContext.py
index e42ad1ac8..0f79a8161 100644
--- a/dpgen/dispatcher/LazyLocalContext.py
+++ b/dpgen/dispatcher/LazyLocalContext.py
@@ -47,8 +47,22 @@ def upload(self,
def download(self,
job_dirs,
remote_down_files,
+ check_exists = False,
+ mark_failure = True,
back_error=False) :
- pass
+ for ii in job_dirs :
+ for jj in remote_down_files :
+ fname = os.path.join(self.local_root, ii, jj)
+ exists = os.path.exists(fname)
+ if not exists:
+ if check_exists:
+ if mark_failure:
+ with open(os.path.join(self.local_root, ii, 'tag_failure_download_%s' % jj), 'w') as fp: pass
+ else:
+ pass
+ else:
+ raise RuntimeError('do not find download file ' + fname)
+
def block_checkcall(self,
cmd) :
diff --git a/dpgen/dispatcher/LocalContext.py b/dpgen/dispatcher/LocalContext.py
index 4549b307a..373f112c7 100644
--- a/dpgen/dispatcher/LocalContext.py
+++ b/dpgen/dispatcher/LocalContext.py
@@ -88,6 +88,8 @@ def upload(self,
def download(self,
job_dirs,
remote_down_files,
+ check_exists = False,
+ mark_failure = True,
back_error=False) :
cwd = os.getcwd()
for ii in job_dirs :
@@ -103,7 +105,13 @@ def download(self,
lfile = os.path.join(local_job, jj)
if not os.path.realpath(rfile) == os.path.realpath(lfile) :
if (not os.path.exists(rfile)) and (not os.path.exists(lfile)):
- raise RuntimeError('do not find download file ' + rfile)
+ if check_exists :
+ if mark_failure:
+ with open(os.path.join(self.local_root, ii, 'tag_failure_download_%s' % jj), 'w') as fp: pass
+ else :
+ pass
+ else :
+ raise RuntimeError('do not find download file ' + rfile)
elif (not os.path.exists(rfile)) and (os.path.exists(lfile)) :
# already downloaded
pass
@@ -114,7 +122,7 @@ def download(self,
# both exists, replace!
dlog.info('find existing %s, replacing by %s' % (lfile, rfile))
if os.path.isdir(lfile):
- shutil.rmtree(lfile)
+ shutil.rmtree(lfile, ignore_errors=True)
elif os.path.isfile(lfile) or os.path.islink(lfile):
os.remove(lfile)
shutil.move(rfile, lfile)
@@ -152,7 +160,7 @@ def block_call(self, cmd) :
return code, None, stdout, stderr
def clean(self) :
- shutil.rmtree(self.remote_root)
+ shutil.rmtree(self.remote_root, ignore_errors=True)
def write_file(self, fname, write_str):
with open(os.path.join(self.remote_root, fname), 'w') as fp :
diff --git a/dpgen/dispatcher/SSHContext.py b/dpgen/dispatcher/SSHContext.py
index 185ade392..a6ac3c1d4 100644
--- a/dpgen/dispatcher/SSHContext.py
+++ b/dpgen/dispatcher/SSHContext.py
@@ -11,8 +11,8 @@ def __init__ (self, jdata) :
# with open(remote_profile) as fp :
# self.remote_profile = json.load(fp)
self.remote_host = self.remote_profile['hostname']
- self.remote_port = self.remote_profile['port']
self.remote_uname = self.remote_profile['username']
+ self.remote_port = self.remote_profile.get('port', 22)
self.remote_password = None
if 'password' in self.remote_profile :
self.remote_password = self.remote_profile['password']
@@ -86,14 +86,17 @@ def __init__ (self,
self.job_uuid = str(uuid.uuid4())
self.remote_root = os.path.join(ssh_session.get_session_root(), self.job_uuid)
self.ssh_session = ssh_session
- self.ssh = self.ssh_session.get_ssh_client()
self.ssh_session.ensure_alive()
try:
- sftp = self.ssh.open_sftp()
+ sftp = self.ssh_session.ssh.open_sftp()
sftp.mkdir(self.remote_root)
sftp.close()
except:
pass
+
+ @property
+ def ssh(self):
+ return self.ssh_session.get_ssh_client()
def close(self):
self.ssh_session.close()
@@ -118,6 +121,8 @@ def upload(self,
def download(self,
job_dirs,
remote_down_files,
+ check_exists = False,
+ mark_failure = True,
back_error=False) :
self.ssh_session.ensure_alive()
cwd = os.getcwd()
@@ -125,19 +130,37 @@ def download(self,
file_list = []
for ii in job_dirs :
for jj in remote_down_files :
- file_list.append(os.path.join(ii,jj))
+ file_name = os.path.join(ii,jj)
+ if check_exists:
+ if self.check_file_exists(file_name):
+ file_list.append(file_name)
+ elif mark_failure :
+ with open(os.path.join(self.local_root, ii, 'tag_failure_download_%s' % jj), 'w') as fp: pass
+ else:
+ pass
+ else:
+ file_list.append(file_name)
if back_error:
errors=glob(os.path.join(ii,'error*'))
file_list.extend(errors)
- self._get_files(file_list)
+ if len(file_list) > 0:
+ self._get_files(file_list)
os.chdir(cwd)
def block_checkcall(self,
- cmd) :
+ cmd,
+ retry=0) :
self.ssh_session.ensure_alive()
stdin, stdout, stderr = self.ssh.exec_command(('cd %s ;' % self.remote_root) + cmd)
exit_status = stdout.channel.recv_exit_status()
if exit_status != 0:
+ if retry<3:
+ # sleep 60 s
+ dlog.warning("Get error code %d in calling %s through ssh with job: %s . message: %s" %
+ (exit_status, cmd, self.job_uuid, stderr.read().decode('utf-8')))
+ dlog.warning("Sleep 60 s and retry the command...")
+ time.sleep(60)
+ return self.block_checkcall(cmd, retry=retry+1)
raise RuntimeError("Get error code %d in calling %s through ssh with job: %s . message: %s" %
(exit_status, cmd, self.job_uuid, stderr.read().decode('utf-8')))
return stdin, stdout, stderr
@@ -190,7 +213,8 @@ def call(self, cmd):
def check_finish(self, cmd_pipes):
return cmd_pipes['stdout'].channel.exit_status_ready()
-
+
+
def get_return(self, cmd_pipes):
if not self.check_finish(cmd_pipes):
return None, None, None
@@ -232,7 +256,10 @@ def _put_files(self,
from_f = os.path.join(self.local_root, of)
to_f = os.path.join(self.remote_root, of)
sftp = self.ssh.open_sftp()
- sftp.put(from_f, to_f)
+ try:
+ sftp.put(from_f, to_f)
+ except FileNotFoundError:
+ raise FileNotFoundError("from %s to %s Error!"%(from_f,to_f))
# remote extract
self.block_checkcall('tar xf %s' % of)
# clean up
diff --git a/dpgen/dispatcher/Shell.py b/dpgen/dispatcher/Shell.py
index 81bff523c..731ad52f5 100644
--- a/dpgen/dispatcher/Shell.py
+++ b/dpgen/dispatcher/Shell.py
@@ -1,6 +1,7 @@
import os,getpass,time
from dpgen.dispatcher.Batch import Batch
from dpgen.dispatcher.JobStatus import JobStatus
+import datetime
def _default_item(resources, key, value) :
if key not in resources :
@@ -10,14 +11,13 @@ def _default_item(resources, key, value) :
class Shell(Batch) :
def check_status(self) :
- if not hasattr(self, 'proc'):
- return JobStatus.unsubmitted
- if not self.context.check_finish(self.proc) :
- return JobStatus.running
- elif (self.context.get_return(self.proc))[0] == 0 :
+ if self.check_finish_tag():
return JobStatus.finished
- else :
+ elif self.check_running():
+ return JobStatus.running
+ else:
return JobStatus.terminated
+ ## warn: cannont distinguish terminated from unsubmitted.
def do_submit(self,
job_dirs,
@@ -32,6 +32,16 @@ def do_submit(self,
self.context.write_file(self.sub_script_name, script_str)
self.proc = self.context.call('cd %s && exec bash %s' % (self.context.remote_root, self.sub_script_name))
+ def check_running(self):
+ uuid_names = self.context.job_uuid
+ ## Check if the uuid.sub is running on remote machine
+ cnt = 0
+ ret, stdin, stdout, stderr = self.context.block_call("ps aux | grep %s"%uuid_names)
+ response_list = stdout.read().decode('utf-8').split("\n")
+ for response in response_list:
+ if uuid_names + ".sub" in response:
+ return True
+ return False
def default_resources(self, res_) :
if res_ == None :
@@ -91,13 +101,12 @@ def sub_script_cmd(self,
_cmd = cmd.split('1>')[0].strip()
if cvasp :
if res['with_mpi']:
- _cmd = 'python ../cvasp.py "mpirun -n %d %s %s" %s' % (res['task_per_node'], _cmd, arg, fp_max_errors)
+ _cmd = 'python cvasp.py "mpirun -n %d %s %s" %s' % (res['task_per_node'], _cmd, arg, fp_max_errors)
else :
- _cmd = 'python ../cvasp.py "%s %s" %s' % (_cmd, arg, fp_max_errors)
+ _cmd = 'python cvasp.py "%s %s" %s' % (_cmd, arg, fp_max_errors)
else :
if res['with_mpi']:
_cmd = 'mpirun -n %d %s %s' % (res['task_per_node'], _cmd, arg)
else :
_cmd = '%s %s' % (_cmd, arg)
return _cmd
-
diff --git a/dpgen/dispatcher/Slurm.py b/dpgen/dispatcher/Slurm.py
index 5ae5493cd..9d409dbcc 100644
--- a/dpgen/dispatcher/Slurm.py
+++ b/dpgen/dispatcher/Slurm.py
@@ -75,18 +75,18 @@ def sub_script_head(self, res):
ret = ''
ret += "#!/bin/bash -l\n"
ret += "#SBATCH -N %d\n" % res['numb_node']
- ret += "#SBATCH --ntasks-per-node %d\n" % res['task_per_node']
+ ret += "#SBATCH --ntasks-per-node=%d\n" % res['task_per_node']
if res['cpus_per_task'] > 0 :
- ret += "#SBATCH --cpus-per-task %d\n" % res['cpus_per_task']
+ ret += "#SBATCH --cpus-per-task=%d\n" % res['cpus_per_task']
ret += "#SBATCH -t %s\n" % res['time_limit']
if res['mem_limit'] > 0 :
- ret += "#SBATCH --mem %dG \n" % res['mem_limit']
+ ret += "#SBATCH --mem=%dG \n" % res['mem_limit']
if len(res['account']) > 0 :
- ret += "#SBATCH --account %s \n" % res['account']
+ ret += "#SBATCH --account=%s \n" % res['account']
if len(res['partition']) > 0 :
- ret += "#SBATCH --partition %s \n" % res['partition']
+ ret += "#SBATCH --partition=%s \n" % res['partition']
if len(res['qos']) > 0 :
- ret += "#SBATCH --qos %s \n" % res['qos']
+ ret += "#SBATCH --qos=%s \n" % res['qos']
if res['numb_gpu'] > 0 :
ret += "#SBATCH --gres=gpu:%d\n" % res['numb_gpu']
for ii in res['constraint_list'] :
@@ -99,7 +99,7 @@ def sub_script_head(self, res):
temp_exclude += ii
temp_exclude += ","
temp_exclude = temp_exclude[:-1]
- ret += '#SBATCH --exclude %s \n' % temp_exclude
+ ret += '#SBATCH --exclude=%s \n' % temp_exclude
ret += "\n"
for ii in res['module_unload_list'] :
ret += "module unload %s\n" % ii
@@ -133,9 +133,9 @@ def sub_script_cmd(self,
_cmd = cmd.split('1>')[0].strip()
if cvasp :
if res['with_mpi']:
- _cmd = 'python ../cvasp.py "srun %s %s" %s' % (_cmd, arg, fp_max_errors)
+ _cmd = 'python cvasp.py "srun %s %s" %s' % (_cmd, arg, fp_max_errors)
else :
- _cmd = 'python ../cvasp.py "%s %s" %s' % (_cmd, arg, fp_max_errors)
+ _cmd = 'python cvasp.py "%s %s" %s' % (_cmd, arg, fp_max_errors)
else :
if res['with_mpi']:
_cmd = 'srun %s %s' % (_cmd, arg)
@@ -149,9 +149,9 @@ def _get_job_id(self) :
else:
return ""
- def _check_status_inner(self, job_id):
+ def _check_status_inner(self, job_id, retry=0):
ret, stdin, stdout, stderr\
- = self.context.block_call ("squeue --job " + job_id)
+ = self.context.block_call ('squeue -o "%.18i %.2t" -j ' + job_id)
if (ret != 0) :
err_str = stderr.read().decode('utf-8')
if str("Invalid job id specified") in err_str :
@@ -160,10 +160,19 @@ def _check_status_inner(self, job_id):
else :
return JobStatus.terminated
else :
+ # retry 3 times
+ if retry < 3:
+ # rest 60s
+ time.sleep(60)
+ return self._check_status_inner(job_id, retry=retry+1)
raise RuntimeError\
("status command squeue fails to execute\nerror message:%s\nreturn code %d\n" % (err_str, ret))
status_line = stdout.read().decode('utf-8').split ('\n')[-2]
- status_word = status_line.split ()[-4]
+ status_word = status_line.split ()[-1]
+ if not (len(status_line.split()) == 2 and status_word.isupper()):
+ raise RuntimeError("Error in getting job status, " +
+ f"status_line = {status_line}, " +
+ f"parsed status_word = {status_word}")
if status_word in ["PD","CF","S"] :
return JobStatus.waiting
elif status_word in ["R"] :
@@ -185,7 +194,7 @@ def _check_sub_limit(self, task_max, **kwarg) :
username = getpass.getuser()
stdin, stdout, stderr = self.context.block_checkcall('squeue -u %s -h' % username)
nj = len(stdout.readlines())
- return nj < task_max
+ return nj >= task_max
def _make_squeue(self,mdata1, res):
ret = ''
diff --git a/dpgen/generator/lib/gaussian.py b/dpgen/generator/lib/gaussian.py
index 79a420e27..304622dfb 100644
--- a/dpgen/generator/lib/gaussian.py
+++ b/dpgen/generator/lib/gaussian.py
@@ -3,6 +3,7 @@
import uuid
import itertools
+import warnings
import numpy as np
import dpdata
from scipy.sparse import csr_matrix
@@ -113,30 +114,33 @@ def make_gaussian_input(sys_data, fp_params):
keywords = [keywords]
# assume default charge is zero and default spin multiplicity is 1
charge = fp_params.get('charge', 0)
- mult_auto = False
- frag = False
- if 'multiplicity' in fp_params:
- if type(fp_params['multiplicity']) == int:
- multiplicity = fp_params['multiplicity']
- elif fp_params['multiplicity'] == 'auto':
- mult_auto = True
- elif fp_params['multiplicity'] == 'frag':
- mult_auto = True
- frag = True
- else:
- raise RuntimeError('The keyword "multiplicity" is illegal.')
+ use_fragment_guesses = False
+ multiplicity = fp_params.get('multiplicity', 'auto')
+ if type(multiplicity) == int:
+ multiplicity = fp_params['multiplicity']
+ mult_auto = False
+ elif multiplicity == 'auto':
+ mult_auto = True
else:
- multiplicity = 1
+ raise RuntimeError('The keyword "multiplicity" is illegal.')
+
+ if fp_params.get("fragment_guesses", False):
+ # Initial guess generated from fragment guesses
+ # New feature of Gaussian 16
+ use_fragment_guesses = True
+ if not mult_auto:
+ warnings.warn("Automatically set multiplicity to auto!")
+ mult_auto = True
if mult_auto:
frag_numb, frag_index = _crd2frag(symbols, coordinates)
if frag_numb == 1:
- frag = False
+ use_fragment_guesses = False
mult_frags = []
for i in range(frag_numb):
idx = frag_index == i
mult_frags.append(detect_multiplicity(np.array(symbols)[idx]))
- if frag:
+ if use_fragment_guesses:
multiplicity = sum(mult_frags) - frag_numb + 1
chargekeywords_frag = "%d %d" % (charge, multiplicity) + \
''.join([' %d %d' % (charge, mult_frag)
@@ -148,7 +152,7 @@ def make_gaussian_input(sys_data, fp_params):
np.count_nonzero(multi_frags == 3) * 2
buff = []
# keywords, e.g., force b3lyp/6-31g**
- if frag:
+ if use_fragment_guesses:
keywords[0] = '{} guess=fragment={}'.format(
keywords[0], frag_numb)
@@ -161,10 +165,10 @@ def make_gaussian_input(sys_data, fp_params):
chargekeywords = '{} {}'.format(charge, multiplicity)
buff = [*chkkeywords, nprockeywords, '#{}'.format(
- keywords[0]), '', titlekeywords, '', (chargekeywords_frag if frag else chargekeywords)]
+ keywords[0]), '', titlekeywords, '', (chargekeywords_frag if use_fragment_guesses else chargekeywords)]
for ii, (symbol, coordinate) in enumerate(zip(symbols, coordinates)):
- if frag:
+ if use_fragment_guesses:
buff.append("%s(Fragment=%d) %f %f %f" %
(symbol, frag_index[ii] + 1, *coordinate))
else:
diff --git a/dpgen/generator/lib/lammps.py b/dpgen/generator/lib/lammps.py
index 367817a89..95dcb8568 100644
--- a/dpgen/generator/lib/lammps.py
+++ b/dpgen/generator/lib/lammps.py
@@ -31,6 +31,7 @@ def make_lammps_input(ensemble,
ele_temp_f = None,
ele_temp_a = None,
max_seed = 1000000,
+ nopbc = False,
deepmd_version = '0.1') :
if (ele_temp_f is not None or ele_temp_a is not None) and LooseVersion(deepmd_version) < LooseVersion('1'):
raise RuntimeError('the electron temperature is only supported by deepmd-kit >= 1.0.0, please upgrade your deepmd-kit')
@@ -49,7 +50,10 @@ def make_lammps_input(ensemble,
ret+= "variable TAU_P equal %f\n" % tau_p
ret+= "\n"
ret+= "units metal\n"
- ret+= "boundary p p p\n"
+ if nopbc:
+ ret+= "boundary f f f\n"
+ else:
+ ret+= "boundary p p p\n"
ret+= "atom_style atomic\n"
ret+= "\n"
ret+= "neighbor 1.0 bin\n"
@@ -104,6 +108,8 @@ def make_lammps_input(ensemble,
ret+= "\n"
if ensemble.split('-')[0] == 'npt' :
assert (pres is not None)
+ if nopbc:
+ raise RuntimeError('ensemble %s is conflicting with nopbc' % ensemble)
if ensemble == "npt" or ensemble == "npt-i" or ensemble == "npt-iso" :
ret+= "fix 1 all npt temp ${TEMP} ${TEMP} ${TAU_T} iso ${PRES} ${PRES} ${TAU_P}\n"
elif ensemble == 'npt-a' or ensemble == 'npt-aniso' :
@@ -116,6 +122,9 @@ def make_lammps_input(ensemble,
ret+= "fix 1 all nve\n"
else :
raise RuntimeError("unknown emsemble " + ensemble)
+ if nopbc:
+ ret+= "velocity all zero linear\n"
+ ret+= "fix fm all momentum 1 linear 1 1 1\n"
ret+= "\n"
ret+= "timestep %f\n" % dt
ret+= "run ${NSTEPS}\n"
diff --git a/dpgen/generator/lib/pwmat.py b/dpgen/generator/lib/pwmat.py
new file mode 100644
index 000000000..4011b82c7
--- /dev/null
+++ b/dpgen/generator/lib/pwmat.py
@@ -0,0 +1,177 @@
+#!/usr/bin/python3
+
+import os
+import numpy as np
+
+def _reciprocal_box(box) :
+ rbox = np.linalg.inv(box)
+ rbox = rbox.T
+ return rbox
+
+def _make_pwmat_kp_mp(kpoints) :
+ ret = ""
+ ret += "%d %d %d 0 0 0 " % (kpoints[0], kpoints[1], kpoints[2])
+ return ret
+
+def _make_kspacing_kpoints(config, kspacing) :
+ with open(config, 'r') as fp:
+ lines = fp.read().split('\n')
+ box = []
+ for idx, ii in enumerate(lines):
+ if 'lattice' in ii or 'Lattice' in ii or 'LATTICE' in ii:
+ for kk in range(idx+1,idx+1+3):
+ vector=[float(jj) for jj in lines[kk].split()[0:3]]
+ box.append(vector)
+ box = np.array(box)
+ rbox = _reciprocal_box(box)
+ kpoints = [(np.ceil(2 * np.pi * np.linalg.norm(ii) / kspacing).astype(int)) for ii in rbox]
+ ret = _make_pwmat_kp_mp(kpoints)
+ return ret
+
+
+def make_pwmat_input_dict (node1, node2, atom_config, ecut, e_error,
+ rho_error, icmix = None, smearing = None,
+ sigma = None,kspacing = 0.5, flag_symm = None) :
+ input_dict = {}
+ input_dict['node1'] = node1
+ input_dict['node2'] = node2
+ input_dict['in.atom'] = atom_config
+ input_dict['ecut'] = ecut
+ input_dict['e_error'] = e_error
+ input_dict['rho_error'] = rho_error
+ if icmix is not None:
+ if sigma is not None:
+ if smearing is not None:
+ SCF_ITER0_1 = "6 4 3 0.0000 " + str(sigma) + " " + str(smearing)
+ SCF_ITER0_2 = "94 4 3 " + str(icmix) + " " + str(sigma) + " " + str(smearing)
+ else:
+ SCF_ITER0_1 = "6 4 3 0.0000 " + str(simga) + " 2"
+ SCF_ITER0_2 = "94 4 3 " + str(icmix) + " " + str(simga) + " 2"
+
+ else:
+ if smearing is not None:
+ SCF_ITER0_1 = "6 4 3 0.0000 0.025 " + str(smearing)
+ SCF_ITER0_2 = "94 4 3 " + str(icmix) + " 0.025 " + str(smearing)
+ else:
+ SCF_ITER0_1 = "6 4 3 0.0000 0.025 2"
+ SCF_ITER0_2 = "94 4 3 " + str(icmix) + " 0.025 2"
+ else:
+ if sigma is not None:
+ if smearing is not None:
+ SCF_ITER0_1 = "6 4 3 0.0000 " + str(sigma) + " " + str(smearing)
+ SCF_ITER0_2 = "94 4 3 1.0000 " + str(sigma) + " " + str(smearing)
+ else:
+ SCF_ITER0_1 = "6 4 3 0.0000 " + str(sigma) + " 2"
+ SCF_ITER0_2 = "94 4 3 1.0000 " + str(sigma) + " 2"
+ else:
+ if smearing is not None:
+ SCF_ITER0_1 = "6 4 3 0.0000 0.025 " + str(smearing)
+ SCF_ITER0_2 = "94 4 3 1.0000 0.025 " + str(smearing)
+ else:
+ SCF_ITER0_1 = "6 4 3 0.0000 0.025 2"
+ SCF_ITER0_2 = "94 4 3 1.0000 0.025 2"
+ input_dict['scf_iter0_1'] = SCF_ITER0_1
+ input_dict['scf_iter0_2'] = SCF_ITER0_2
+ if flag_symm is not None :
+ MP_N123 = _make_kspacing_kpoints(atom_config, kspacing)
+ MP_N123 += str(flag_symm)
+ else:
+ MP_N123 = _make_kspacing_kpoints(atom_config, kspacing)
+ input_dict['mp_n123'] = MP_N123
+ input_dict['out.wg'] = 'F'
+ input_dict['out.rho'] = 'F'
+ input_dict['out.mlmd'] = 'T\n'
+ return input_dict
+
+def _update_input_dict(input_dict_, user_dict) :
+ if user_dict is None:
+ return input_dict_
+ input_dict = input_dict_
+ for ii in user_dict :
+ input_dict[ci] = user_dict[ii]
+ return input_dict
+
+def write_input_dict(input_dict) :
+ lines = []
+ for key in input_dict:
+ if (type(input_dict[key]) == bool):
+ if input_dict[key]:
+ rs = 'T'
+ else :
+ rs = 'F'
+ else :
+ rs = str(input_dict[key])
+ lines.append('%s=%s' % (key, rs))
+ return '\n'.join(lines)
+
+
+def _make_smearing(fp_params) :
+ icmix = None
+ smearing = None
+ sigma = None
+ if 'icmix' in fp_params :
+ icmix = fp_params['icmix']
+ if 'smearing' in fp_params :
+ smearing = fp_params['smearing']
+ if 'sigma' in fp_params :
+ sigma = fp_params['sigma']
+ if icmix == None:
+ if smearing == None:
+ if sigma == None:
+ return None, None, None
+ else:
+ return None, None, sigma
+ else:
+ if sigma == None:
+ return None, smearing, None
+ else:
+ return None, smearing, sigma
+ else:
+ if smearing == None:
+ if sigma == None:
+ return icmix, None, None
+ else:
+ return icmix, None, sigma
+ else:
+ if sigma == None:
+ return icmix, smearing, None
+ else:
+ return icmix, smearing, sigma
+def _make_flag_symm(fp_params) :
+ flag_symm = None
+ if 'flag_symm' in fp_params :
+ flag_symm = fp_params['flag_symm']
+ if flag_symm == 'NONE' :
+ flag_symm = None
+ elif str(flag_symm) not in [None, '0', '1', '2', '3'] :
+ raise RuntimeError ("unknow flag_symm type " + str(flag_symm))
+ return flag_symm
+
+def make_pwmat_input_user_dict(fp_params) :
+ node1 = fp_params['node1']
+ node2 = fp_params['node2']
+ atom_config = fp_params['in.atom']
+ ecut = fp_params['ecut']
+ e_error = fp_params['e_error']
+ rho_error = fp_params['rho_error']
+ kspacing = fp_params['kspacing']
+ if 'user_pwmat_params' in fp_params :
+ user_dict = fp_params['user_pwmat_params']
+ else :
+ user_dict = None
+ icmix, smearing, sigma = _make_smearing(fp_params)
+ flag_symm = _make_flag_symm(fp_params)
+ input_dict = make_pwmat_input_dict(node1, node2, atom_config, ecut, e_error,
+ rho_error, icmix = icmix, smearing = smearing,
+ sigma = sigma, kspacing = kspacing,
+ flag_symm = flag_symm
+ )
+ input_dict = _update_input_dict(input_dict, user_dict)
+ input = write_input_dict(input_dict)
+ return input
+
+def input_upper(dinput):
+ standard_input={}
+ for key,val in dinput.items():
+ standard_input[key.upper()]=val
+ return Input(standard_input)
diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py
index ceac94ca1..3b6c2a47d 100644
--- a/dpgen/generator/run.py
+++ b/dpgen/generator/run.py
@@ -43,6 +43,10 @@
from dpgen.generator.lib.vasp import incar_upper
from dpgen.generator.lib.pwscf import make_pwscf_input
#from dpgen.generator.lib.pwscf import cvt_1frame
+from dpgen.generator.lib.pwmat import make_pwmat_input_dict
+from dpgen.generator.lib.pwmat import write_input_dict
+from dpgen.generator.lib.pwmat import make_pwmat_input_user_dict
+from dpgen.generator.lib.pwmat import input_upper
from dpgen.generator.lib.siesta import make_siesta_input
from dpgen.generator.lib.gaussian import make_gaussian_input, take_cluster
from dpgen.generator.lib.cp2k import make_cp2k_input, make_cp2k_xyz
@@ -53,11 +57,6 @@
from dpgen.remote.group_jobs import group_local_jobs
from dpgen.remote.decide_machine import decide_train_machine, decide_fp_machine, decide_model_devi_machine
from dpgen.dispatcher.Dispatcher import Dispatcher, _split_tasks, make_dispatcher
-try:
- from dpgen.dispatcher.ALI import ALI
-except ImportError as e:
- dlog.info(e)
- pass
from dpgen.util import sepline
from dpgen import ROOT_PATH
from pymatgen.io.vasp import Incar,Kpoints,Potcar
@@ -194,6 +193,15 @@ def make_train (iter_index,
fp_task_min = jdata['fp_task_min']
model_devi_jobs = jdata['model_devi_jobs']
use_ele_temp = jdata.get('use_ele_temp', 0)
+ training_iter0_model = jdata.get('training_iter0_model_path', [])
+ training_init_model = jdata.get('training_init_model', False)
+ training_reuse_iter = jdata.get('training_reuse_iter')
+ training_reuse_old_ratio = jdata.get('training_reuse_old_ratio', 0.2)
+ training_reuse_stop_batch = jdata.get('training_reuse_stop_batch', 400000)
+ training_reuse_start_lr = jdata.get('training_reuse_start_lr', 1e-4)
+ training_reuse_start_pref_e = jdata.get('training_reuse_start_pref_e', 0.1)
+ training_reuse_start_pref_f = jdata.get('training_reuse_start_pref_f', 100)
+ model_devi_activation_func = jdata.get('model_devi_activation_func', None)
if iter_index > 0 and _check_empty_iter(iter_index-1, fp_task_min) :
log_task('prev data is empty, copy prev model')
@@ -229,12 +237,18 @@ def make_train (iter_index,
init_batch_size = []
if 'init_batch_size' in jdata:
init_batch_size_ = list(jdata['init_batch_size'])
+ if len(init_data_sys_) > len(init_batch_size_):
+ warnings.warn("The batch sizes are not enough. Assume auto for those not spefified.")
+ init_batch_size.extend(["auto" for aa in range(len(init_data_sys_)-len(init_batch_size))])
else:
init_batch_size_ = ["auto" for aa in range(len(jdata['init_data_sys']))]
if 'sys_batch_size' in jdata:
sys_batch_size = jdata['sys_batch_size']
else:
sys_batch_size = ["auto" for aa in range(len(jdata['sys_configs']))]
+
+ # make sure all init_data_sys has the batch size -- for the following `zip`
+ assert (len(init_data_sys_) <= len(init_batch_size_))
for ii, ss in zip(init_data_sys_, init_batch_size_) :
if jdata.get('init_multi_systems', False):
for single_sys in os.listdir(os.path.join(work_path, 'data.init', ii)):
@@ -243,8 +257,11 @@ def make_train (iter_index,
else:
init_data_sys.append(os.path.join('..', 'data.init', ii))
init_batch_size.append(detect_batch_size(ss, os.path.join(work_path, 'data.init', ii)))
+ old_range = None
if iter_index > 0 :
for ii in range(iter_index) :
+ if ii == iter_index - 1:
+ old_range = len(init_data_sys)
fp_path = os.path.join(make_iter_name(ii), fp_name)
fp_data_sys = glob.glob(os.path.join(fp_path, "data.*"))
for jj in fp_data_sys :
@@ -262,9 +279,7 @@ def make_train (iter_index,
init_data_sys.append(os.path.join('..', 'data.iters', jj, sys_single))
init_batch_size.append(detect_batch_size(sys_batch_size[sys_idx], os.path.join(jj, sys_single)))
else:
- tmp_box = np.loadtxt(os.path.join(jj, 'box.raw'))
- tmp_box = np.reshape(tmp_box, [-1,9])
- nframes = tmp_box.shape[0]
+ nframes = dpdata.System(jj, 'deepmd/npy').get_nframes()
if nframes < fp_task_min :
log_task('nframes (%d) in data sys %s is too small, skip' % (nframes, jj))
continue
@@ -274,8 +289,9 @@ def make_train (iter_index,
jinput = jdata['default_training_param']
try:
mdata["deepmd_version"]
- except:
+ except KeyError:
mdata = set_version(mdata)
+ # setup data systems
if LooseVersion(mdata["deepmd_version"]) < LooseVersion('1'):
# 0.x
jinput['systems'] = init_data_sys
@@ -286,6 +302,7 @@ def make_train (iter_index,
# 1.x
jinput['training']['systems'] = init_data_sys
jinput['training']['batch_size'] = init_batch_size
+ jinput['model']['type_map'] = jdata['type_map']
# electron temperature
if use_ele_temp == 0:
pass
@@ -297,14 +314,27 @@ def make_train (iter_index,
jinput['model']['fitting_net'].pop('numb_fparam', None)
else:
raise RuntimeError('invalid setting for use_ele_temp ' + str(use_ele_temp))
+ # set training reuse model
+ if training_reuse_iter is not None and iter_index >= training_reuse_iter:
+ jinput['training']['auto_prob_style'] \
+ ="prob_sys_size; 0:%d:%f; %d:%d:%f" \
+ %(old_range, training_reuse_old_ratio, old_range, len(init_data_sys), 1.-training_reuse_old_ratio)
+ if jinput['loss'].get('start_pref_e') is not None:
+ jinput['loss']['start_pref_e'] = training_reuse_start_pref_e
+ if jinput['loss'].get('start_pref_f') is not None:
+ jinput['loss']['start_pref_f'] = training_reuse_start_pref_f
+ jinput['learning_rate']['start_lr'] = training_reuse_start_lr
+ jinput['training']['stop_batch'] = training_reuse_stop_batch
+
for ii in range(numb_models) :
task_path = os.path.join(work_path, train_task_fmt % ii)
create_path(task_path)
os.chdir(task_path)
- for ii in init_data_sys :
- if not os.path.isdir(ii) :
- raise RuntimeError ("data sys %s does not exists, cwd is %s" % (ii, os.getcwd()))
+ for jj in init_data_sys :
+ if not os.path.isdir(jj) :
+ raise RuntimeError ("data sys %s does not exists, cwd is %s" % (jj, os.getcwd()))
os.chdir(cwd)
+ # set random seed for each model
if LooseVersion(mdata["deepmd_version"]) < LooseVersion('1'):
# 0.x
jinput['seed'] = random.randrange(sys.maxsize) % (2**32)
@@ -313,6 +343,14 @@ def make_train (iter_index,
jinput['model']['descriptor']['seed'] = random.randrange(sys.maxsize) % (2**32)
jinput['model']['fitting_net']['seed'] = random.randrange(sys.maxsize) % (2**32)
jinput['training']['seed'] = random.randrange(sys.maxsize) % (2**32)
+ # set model activation function
+ if model_devi_activation_func is not None:
+ if LooseVersion(mdata["deepmd_version"]) < LooseVersion('1'):
+ raise RuntimeError('model_devi_activation_func does not suppport deepmd version', mdata['deepmd_version'])
+ assert(type(model_devi_activation_func) is list and len(model_devi_activation_func) == numb_models)
+ jinput['model']['descriptor']['activation_function'] = model_devi_activation_func[ii]
+ jinput['model']['fitting_net']['activation_function'] = model_devi_activation_func[ii]
+ # dump the input.json
with open(os.path.join(task_path, train_input_file), 'w') as outfile:
json.dump(jinput, outfile, indent = 4)
@@ -324,16 +362,38 @@ def make_train (iter_index,
prev_task_path = os.path.join(prev_work_path, train_task_fmt%ii)
old_model_files = glob.glob(
os.path.join(prev_task_path, "model.ckpt*"))
- task_path = os.path.join(work_path, train_task_fmt % ii)
- task_old_path = os.path.join(task_path, 'old')
- create_path(task_old_path)
- cwd = os.getcwd()
- for jj in old_model_files:
- absjj = os.path.abspath(jj)
- basejj = os.path.basename(jj)
- os.chdir(task_old_path)
- os.symlink(os.path.relpath(absjj), basejj)
- os.chdir(cwd)
+ _link_old_models(work_path, old_model_files, ii)
+ else:
+ if type(training_iter0_model) == str:
+ training_iter0_model = [training_iter0_model]
+ iter0_models = []
+ for ii in training_iter0_model:
+ model_is = glob.glob(ii)
+ model_is.sort()
+ iter0_models += [os.path.abspath(ii) for ii in model_is]
+ if training_init_model:
+ assert(numb_models == len(iter0_models)), "training_iter0_model should be provided, and the number of models should be equal to %d" % numb_models
+ for ii in range(len(iter0_models)):
+ old_model_files = glob.glob(os.path.join(iter0_models[ii], 'model.ckpt*'))
+ _link_old_models(work_path, old_model_files, ii)
+
+
+def _link_old_models(work_path, old_model_files, ii):
+ """
+ link the `ii`th old model given by `old_model_files` to
+ the `ii`th training task in `work_path`
+ """
+ task_path = os.path.join(work_path, train_task_fmt % ii)
+ task_old_path = os.path.join(task_path, 'old')
+ create_path(task_old_path)
+ cwd = os.getcwd()
+ for jj in old_model_files:
+ absjj = os.path.abspath(jj)
+ basejj = os.path.basename(jj)
+ os.chdir(task_old_path)
+ os.symlink(os.path.relpath(absjj), basejj)
+ os.chdir(cwd)
+
def detect_batch_size(batch_size, system=None):
if type(batch_size) == int:
@@ -352,16 +412,19 @@ def run_train (iter_index,
numb_models = jdata['numb_models']
# train_param = jdata['train_param']
train_input_file = default_train_input_file
+ training_reuse_iter = jdata.get('training_reuse_iter')
+ training_init_model = jdata.get('training_init_model', False)
+ if training_reuse_iter is not None and iter_index >= training_reuse_iter:
+ training_init_model = True
try:
mdata["deepmd_version"]
- except:
+ except KeyError:
mdata = set_version(mdata)
if LooseVersion(mdata["deepmd_version"]) < LooseVersion('1'):
# 0.x
deepmd_path = mdata['deepmd_path']
else:
# 1.x
- python_path = mdata.get('python_path', None)
train_command = mdata.get('train_command', 'dp')
train_resources = mdata['train_resources']
@@ -385,22 +448,20 @@ def run_train (iter_index,
commands.append(command)
command = os.path.join(deepmd_path, 'bin/dp_frz')
commands.append(command)
- elif python_path:
- # 1.x
- command = '%s -m deepmd train %s' % (python_path, train_input_file)
- commands.append(command)
- command = '%s -m deepmd freeze' % python_path
- commands.append(command)
else:
+ # 1.x
## Commands are like `dp train` and `dp freeze`
## train_command should not be None
assert(train_command)
command = '%s train %s' % (train_command, train_input_file)
+ if training_init_model:
+ command = "{ if [ ! -f model.ckpt.index ]; then %s --init-model old/model.ckpt; else %s --restart model.ckpt; fi }" % (command, command)
+ else:
+ command = "{ if [ ! -f model.ckpt.index ]; then %s; else %s --restart model.ckpt; fi }" % (command, command)
commands.append(command)
command = '%s freeze' % train_command
commands.append(command)
-
#_tasks = [os.path.basename(ii) for ii in all_task]
# run_tasks = []
# for ii in all_task:
@@ -413,7 +474,13 @@ def run_train (iter_index,
run_tasks = [os.path.basename(ii) for ii in all_task]
forward_files = [train_input_file]
+ if training_init_model:
+ forward_files += [os.path.join('old', 'model.ckpt.meta'),
+ os.path.join('old', 'model.ckpt.index'),
+ os.path.join('old', 'model.ckpt.data-00000-of-00001')
+ ]
backward_files = ['frozen_model.pb', 'lcurve.out', 'train.log']
+ backward_files+= ['model.ckpt.meta', 'model.ckpt.index', 'model.ckpt.data-00000-of-00001', 'checkpoint']
init_data_sys_ = jdata['init_data_sys']
init_data_sys = []
for ii in init_data_sys_ :
@@ -429,21 +496,21 @@ def run_train (iter_index,
if jdata.get('init_multi_systems', False):
for single_sys in os.listdir(os.path.join(ii)):
trans_comm_data += glob.glob(os.path.join(ii, single_sys, 'set.*'))
- trans_comm_data += glob.glob(os.path.join(ii, single_sys, 'type.raw'))
+ trans_comm_data += glob.glob(os.path.join(ii, single_sys, 'type*.raw'))
trans_comm_data += glob.glob(os.path.join(ii, single_sys, 'nopbc'))
else:
trans_comm_data += glob.glob(os.path.join(ii, 'set.*'))
- trans_comm_data += glob.glob(os.path.join(ii, 'type.raw'))
+ trans_comm_data += glob.glob(os.path.join(ii, 'type*.raw'))
trans_comm_data += glob.glob(os.path.join(ii, 'nopbc'))
for ii in fp_data :
if jdata.get('use_clusters', False):
for single_sys in os.listdir(os.path.join(ii)):
trans_comm_data += glob.glob(os.path.join(ii, single_sys, 'set.*'))
- trans_comm_data += glob.glob(os.path.join(ii, single_sys, 'type.raw'))
+ trans_comm_data += glob.glob(os.path.join(ii, single_sys, 'type*.raw'))
trans_comm_data += glob.glob(os.path.join(ii, single_sys, 'nopbc'))
else:
trans_comm_data += glob.glob(os.path.join(ii, 'set.*'))
- trans_comm_data += glob.glob(os.path.join(ii, 'type.raw'))
+ trans_comm_data += glob.glob(os.path.join(ii, 'type*.raw'))
trans_comm_data += glob.glob(os.path.join(ii, 'nopbc'))
os.chdir(cwd)
@@ -452,14 +519,7 @@ def run_train (iter_index,
except:
train_group_size = 1
- task_chunks = _split_tasks(run_tasks, train_group_size)
- nchunks = len(task_chunks)
- if "ali_auth" in mdata:
- dispatcher = ALI(mdata['ali_auth'], mdata['train_resources'], mdata['train_machine'], nchunks, work_path)
- dispatcher.init()
- else:
- dispatcher = make_dispatcher(mdata['train_machine'])
-
+ dispatcher = make_dispatcher(mdata['train_machine'], mdata['train_resources'], work_path, run_tasks, train_group_size)
dispatcher.run_jobs(mdata['train_resources'],
commands,
work_path,
@@ -506,7 +566,7 @@ def parse_cur_job(cur_job) :
if 'npt' in ensemble :
temps = _get_param_alias(cur_job, ['Ts','temps'])
press = _get_param_alias(cur_job, ['Ps','press'])
- elif 'nvt' == ensemble :
+ elif 'nvt' == ensemble or 'nve' == ensemble:
temps = _get_param_alias(cur_job, ['Ts','temps'])
nsteps = _get_param_alias(cur_job, ['nsteps'])
trj_freq = _get_param_alias(cur_job, ['t_freq', 'trj_freq','traj_freq'])
@@ -664,6 +724,8 @@ def make_model_devi (iter_index,
else:
fmt = 'vasp/poscar'
system = dpdata.System(os.path.join(conf_path, poscar_name), fmt = fmt, type_map = jdata['type_map'])
+ if jdata.get('model_devi_nopbc', False):
+ system.remove_pbc()
system.to_lammps_lmp(os.path.join(conf_path, lmp_name))
conf_counter += 1
sys_counter += 1
@@ -672,6 +734,7 @@ def make_model_devi (iter_index,
if "template" in cur_job:
input_mode = "revise_template"
use_plm = jdata.get('model_devi_plumed', False)
+ use_plm_path = jdata.get('model_devi_plumed_path', False)
if input_mode == "native":
_make_model_devi_native(iter_index, jdata, mdata, conf_systems)
elif input_mode == "revise_template":
@@ -692,6 +755,7 @@ def _make_model_devi_revmat(iter_index, jdata, mdata, conf_systems):
raise RuntimeError("system index should be uniq")
mass_map = jdata['mass_map']
use_plm = jdata.get('model_devi_plumed', False)
+ use_plm_path = jdata.get('model_devi_plumed_path', False)
trj_freq = _get_param_alias(cur_job, ['t_freq', 'trj_freq','traj_freq'])
rev_keys, rev_mat, num_lmp = parse_cur_job_revmat(cur_job, use_plm = use_plm)
@@ -700,6 +764,9 @@ def _make_model_devi_revmat(iter_index, jdata, mdata, conf_systems):
if use_plm:
plm_templ = cur_job['template']['plm']
plm_templ = os.path.abspath(plm_templ)
+ if use_plm_path:
+ plm_path_templ = cur_job['template']['plm_path']
+ plm_path_templ = os.path.abspath(plm_path_templ)
iter_name = make_iter_name(iter_index)
train_path = os.path.join(iter_name, train_name)
@@ -711,7 +778,7 @@ def _make_model_devi_revmat(iter_index, jdata, mdata, conf_systems):
work_path = os.path.join(iter_name, model_devi_name)
try:
mdata["deepmd_version"]
- except:
+ except KeyError:
mdata = set_version(mdata)
deepmd_version = mdata['deepmd_version']
@@ -751,6 +818,8 @@ def _make_model_devi_revmat(iter_index, jdata, mdata, conf_systems):
plm_lines = revise_by_keys(plm_lines, rev_keys[num_lmp:], rev_item[num_lmp:])
with open('input.plumed', 'w') as fp:
fp.write(''.join(plm_lines))
+ if use_plm_path:
+ shutil.copyfile(plm_path_templ, 'plmpath.pdb')
# dump input of lammps
with open('input.lammps', 'w') as fp:
fp.write(''.join(lmp_lines))
@@ -788,6 +857,7 @@ def _make_model_devi_native(iter_index, jdata, mdata, conf_systems):
if 'model_devi_taup' in jdata :
model_devi_taup = jdata['model_devi_taup']
mass_map = jdata['mass_map']
+ nopbc = jdata.get('model_devi_nopbc', False)
iter_name = make_iter_name(iter_index)
train_path = os.path.join(iter_name, train_name)
@@ -840,7 +910,7 @@ def _make_model_devi_native(iter_index, jdata, mdata, conf_systems):
os.chdir(task_path)
try:
mdata["deepmd_version"]
- except:
+ except KeyError:
mdata = set_version(mdata)
deepmd_version = mdata['deepmd_version']
file_c = make_lammps_input(ensemble,
@@ -859,6 +929,7 @@ def _make_model_devi_native(iter_index, jdata, mdata, conf_systems):
pka_e = pka_e,
ele_temp_f = te_f,
ele_temp_a = te_a,
+ nopbc = nopbc,
deepmd_version = deepmd_version)
job = {}
job["ensemble"] = ensemble
@@ -887,6 +958,7 @@ def run_model_devi (iter_index,
model_devi_group_size = mdata['model_devi_group_size']
model_devi_resources = mdata['model_devi_resources']
use_plm = jdata.get('model_devi_plumed', False)
+ use_plm_path = jdata.get('model_devi_plumed_path', False)
iter_name = make_iter_name(iter_index)
work_path = os.path.join(iter_name, model_devi_name)
@@ -919,16 +991,13 @@ def run_model_devi (iter_index,
backward_files = ['model_devi.out', 'model_devi.log', 'traj']
if use_plm:
forward_files += ['input.plumed']
- backward_files += ['output.plumed']
+ # backward_files += ['output.plumed']
+ backward_files += ['output.plumed','COLVAR','dump.0.xyz']
+ if use_plm_path:
+ forward_files += ['plmpath.pdb']
cwd = os.getcwd()
- task_chunks = _split_tasks(run_tasks, model_devi_group_size)
- nchunks = len(task_chunks)
- if "ali_auth" in mdata:
- dispatcher = ALI(mdata['ali_auth'], mdata['model_devi_resources'], mdata['model_devi_machine'], nchunks, work_path)
- dispatcher.init()
- else:
- dispatcher = make_dispatcher(mdata['model_devi_machine'])
+ dispatcher = make_dispatcher(mdata['model_devi_machine'], mdata['model_devi_resources'], work_path, run_tasks, model_devi_group_size)
dispatcher.run_jobs(mdata['model_devi_resources'],
commands,
work_path,
@@ -946,6 +1015,40 @@ def post_model_devi (iter_index,
mdata) :
pass
+
+def _to_face_dist(box_):
+ box = np.reshape(box_, [3,3])
+ vol = np.abs(np.linalg.det(box))
+ dists = []
+ for [ii,jj] in [[0, 1], [1, 2], [2, 0]]:
+ vv = np.cross(box[ii], box[jj])
+ dists.append(vol / np.linalg.norm(vv))
+ return np.array(dists)
+
+def check_bad_box(conf_name,
+ criteria,
+ fmt = 'lammps/dump'):
+ all_c = criteria.split(';')
+ sys = dpdata.System(conf_name, fmt)
+ assert(sys.get_nframes() == 1)
+ is_bad = False
+ for ii in all_c:
+ [key, value] = ii.split(':')
+ if key == 'length_ratio':
+ lengths = np.linalg.norm(sys['cells'][0], axis = 1)
+ ratio = np.max(lengths) / np.min(lengths)
+ if ratio > float(value):
+ is_bad = True
+ elif key == 'height_ratio':
+ lengths = np.linalg.norm(sys['cells'][0], axis = 1)
+ dists = _to_face_dist(sys['cells'][0])
+ ratio = np.max(lengths) / np.min(dists)
+ if ratio > float(value):
+ is_bad = True
+ else:
+ raise RuntimeError('unknow key', key)
+ return is_bad
+
def _make_fp_vasp_inner (modd_path,
work_path,
model_devi_skip,
@@ -979,6 +1082,8 @@ def _make_fp_vasp_inner (modd_path,
cluster_cutoff = jdata['cluster_cutoff'] if jdata.get('use_clusters', False) else None
# skip save *.out if detailed_report_make_fp is False, default is True
detailed_report_make_fp = jdata.get("detailed_report_make_fp", True)
+ # skip bad box criteria
+ skip_bad_box = jdata.get('fp_skip_bad_box')
for ss in system_index :
fp_candidate = []
if detailed_report_make_fp:
@@ -989,6 +1094,9 @@ def _make_fp_vasp_inner (modd_path,
modd_system_task.sort()
cc = 0
counter = Counter()
+ counter['candidate'] = 0
+ counter['failed'] = 0
+ counter['accurate'] = 0
for tt in modd_system_task :
with warnings.catch_warnings():
warnings.simplefilter("ignore")
@@ -1044,7 +1152,23 @@ def _make_fp_vasp_inner (modd_path,
with open(os.path.join(work_path,'rest_failed.shuffled.%s.out'%ss), 'w') as fp:
for ii in fp_rest_failed:
fp.write(" ".join([str(nn) for nn in ii]) + "\n")
- numb_task = min(fp_task_max, len(fp_candidate))
+
+ # set number of tasks
+ accurate_ratio = float(counter['accurate']) / float(fp_sum)
+ fp_accurate_threshold = jdata.get('fp_accurate_threshold', 1)
+ fp_accurate_soft_threshold = jdata.get('fp_accurate_soft_threshold', fp_accurate_threshold)
+ if accurate_ratio < fp_accurate_soft_threshold :
+ this_fp_task_max = fp_task_max
+ elif accurate_ratio >= fp_accurate_soft_threshold and accurate_ratio < fp_accurate_threshold:
+ this_fp_task_max = int(fp_task_max * (accurate_ratio - fp_accurate_threshold) / (fp_accurate_soft_threshold - fp_accurate_threshold))
+ else:
+ this_fp_task_max = 0
+ numb_task = min(this_fp_task_max, len(fp_candidate))
+ if (numb_task < fp_task_min):
+ numb_task = 0
+ dlog.info("system {0:s} accurate_ratio: {1:8.4f} thresholds: {2:6.4f} and {3:6.4f} eff. task min and max {4:4d} {5:4d} number of fp tasks: {6:6d}".format(ss, accurate_ratio, fp_accurate_soft_threshold, fp_accurate_threshold, fp_task_min, this_fp_task_max, numb_task))
+ # make fp tasks
+ count_bad_box = 0
for cc in range(numb_task) :
tt = fp_candidate[cc][0]
ii = fp_candidate[cc][1]
@@ -1052,6 +1176,11 @@ def _make_fp_vasp_inner (modd_path,
conf_name = os.path.join(tt, "traj")
conf_name = os.path.join(conf_name, str(ii) + '.lammpstrj')
conf_name = os.path.abspath(conf_name)
+ if skip_bad_box is not None:
+ skip = check_bad_box(conf_name, skip_bad_box)
+ if skip:
+ count_bad_box += 1
+ continue
# link job.json
job_name = os.path.join(tt, "job.json")
@@ -1078,6 +1207,8 @@ def _make_fp_vasp_inner (modd_path,
for pair in fp_link_files :
os.symlink(pair[0], pair[1])
os.chdir(cwd)
+ if count_bad_box > 0:
+ dlog.info("system {0:s} skipped {1:6d} confs with bad box, {2:6d} remains".format(ss, count_bad_box, numb_task - count_bad_box))
if cluster_cutoff is None:
cwd = os.getcwd()
for ii in fp_tasks:
@@ -1102,6 +1233,65 @@ def make_vasp_incar(jdata, filename):
fp.write(incar)
return incar
+def make_pwmat_input(jdata, filename):
+ if 'fp_incar' in jdata.keys() :
+ fp_incar_path = jdata['fp_incar']
+ assert(os.path.exists(fp_incar_path))
+ fp_incar_path = os.path.abspath(fp_incar_path)
+ fr = open(fp_incar_path)
+ input = fr.read()
+ fr.close()
+ elif 'user_fp_params' in jdata.keys() :
+ fp_params = jdata['user_fp_params']
+ node1 = fp_params['node1']
+ node2 = fp_params['node2']
+ atom_config = fp_params['in.atom']
+ ecut = fp_params['ecut']
+ e_error = fp_params['e_error']
+ rho_error = fp_params['rho_error']
+ kspacing = fp_params['kspacing']
+ flag_symm = fp_params['flag_symm']
+ os.system("command -v poscar2config.x | wc -l > 1.txt")
+ fc = open('1.txt')
+ flag_command = fc.read()
+ fc.close()
+ if int(flag_command) == 1 :
+ os.system('poscar2config.x < POSCAR > tmp.config')
+ else:
+ os.system('cp ../../../out_data_post_fp_pwmat/02.fp/task.000.000000/poscar2config.x ./')
+ os.system('./poscar2config.x < POSCAR > tmp.config')
+ os.system('rm -rf tmp.config')
+ input_dict = make_pwmat_input_dict(node1, node2, atom_config, ecut, e_error,
+ rho_error, icmix = None, smearing = None,
+ sigma = None, kspacing = kspacing,
+ flag_symm = flag_symm
+ )
+
+ input = write_input_dict(input_dict)
+ else:
+ input = make_pwmat_input_user_dict(jdata['fp_params'])
+ if 'IN.PSP' in input or 'in.psp' in input:
+ with open(filename, 'w') as fp:
+ fp.write(input)
+ fp.write('job=scf\n')
+ if 'OUT.MLMD' in input or 'out.mlmd' in input:
+ return input
+ else:
+ fp.write('OUT.MLMD = T')
+ return input
+ else:
+ with open(filename, 'w') as fp:
+ fp.write(input)
+ fp.write('job=scf\n')
+ fp_pp_files = jdata['fp_pp_files']
+ for idx, ii in enumerate(fp_pp_files) :
+ fp.write('IN.PSP%d = %s\n' %(idx+1, ii))
+ if 'OUT.MLMD' in input or 'out.mlmd' in input:
+ return input
+ else:
+ fp.write('OUT.MLMD = T')
+ return input
+
def make_vasp_incar_ele_temp(jdata, filename, ele_temp, nbands_esti = None):
with open(filename) as fp:
incar = fp.read()
@@ -1116,7 +1306,7 @@ def make_vasp_incar_ele_temp(jdata, filename, ele_temp, nbands_esti = None):
incar['NBANDS'] = nbands
incar.write_file('INCAR')
-def _make_fp_vasp_incar (iter_index,
+def make_fp_vasp_incar (iter_index,
jdata,
nbands_esti = None) :
iter_name = make_iter_name(iter_index)
@@ -1138,9 +1328,44 @@ def _make_fp_vasp_incar (iter_index,
nbands_esti = nbands_esti)
os.chdir(cwd)
-def _make_fp_vasp_kp (iter_index,jdata):
+def _make_fp_pwmat_input (iter_index,
+ jdata) :
iter_name = make_iter_name(iter_index)
work_path = os.path.join(iter_name, fp_name)
+ fp_tasks = glob.glob(os.path.join(work_path, 'task.*'))
+ fp_tasks.sort()
+ if len(fp_tasks) == 0 :
+ return
+ cwd = os.getcwd()
+ for ii in fp_tasks:
+ os.chdir(ii)
+ make_pwmat_input(jdata, 'etot.input')
+ os.system("sed -i '1,2c 4 1' etot.input")
+ os.chdir(cwd)
+
+def make_fp_vasp_cp_cvasp(iter_index,jdata):
+ # Move cvasp interface to jdata
+ if ('cvasp' in jdata) and (jdata['cvasp'] == True):
+ pass
+ else:
+ return
+ iter_name = make_iter_name(iter_index)
+ work_path = os.path.join(iter_name, fp_name)
+ fp_tasks = glob.glob(os.path.join(work_path, 'task.*'))
+ fp_tasks.sort()
+ if len(fp_tasks) == 0 :
+ return
+ cwd = os.getcwd()
+ for ii in fp_tasks:
+ os.chdir(ii)
+ #copy cvasp.py
+ shutil.copyfile(cvasp_file, 'cvasp.py')
+ os.chdir(cwd)
+
+def make_fp_vasp_kp (iter_index,jdata):
+ iter_name = make_iter_name(iter_index)
+ work_path = os.path.join(iter_name, fp_name)
+ fp_aniso_kspacing = jdata.get('fp_aniso_kspacing')
fp_tasks = glob.glob(os.path.join(work_path, 'task.*'))
fp_tasks.sort()
@@ -1154,10 +1379,13 @@ def _make_fp_vasp_kp (iter_index,jdata):
with open('INCAR') as fp:
incar = fp.read()
standard_incar = incar_upper(Incar.from_string(incar))
- try:
- kspacing = standard_incar['KSPACING']
- except:
- raise RuntimeError ("KSPACING must be given in INCAR")
+ if fp_aniso_kspacing is None:
+ try:
+ kspacing = standard_incar['KSPACING']
+ except KeyError:
+ raise RuntimeError ("KSPACING must be given in INCAR")
+ else :
+ kspacing = fp_aniso_kspacing
try:
gamma = standard_incar['KGAMMA']
if isinstance(gamma,bool):
@@ -1167,7 +1395,7 @@ def _make_fp_vasp_kp (iter_index,jdata):
gamma=True
else:
gamma=False
- except:
+ except KeyError:
raise RuntimeError ("KGAMMA must be given in INCAR")
# check poscar
assert(os.path.exists('POSCAR'))
@@ -1253,10 +1481,6 @@ def _make_fp_vasp_configs(iter_index,
work_path = os.path.join(iter_name, fp_name)
create_path(work_path)
- #copy cvasp.py
- # Move cvasp interface to jdata
- if ('cvasp' in jdata) and (jdata['cvasp'] == True):
- shutil.copyfile(cvasp_file, os.path.join(work_path,'cvasp.py'))
modd_path = os.path.join(iter_name, model_devi_name)
task_min = -1
@@ -1293,9 +1517,11 @@ def make_fp_vasp (iter_index,
# 1, create potcar
sys_link_fp_vasp_pp(iter_index, jdata)
# 2, create incar
- _make_fp_vasp_incar(iter_index, jdata, nbands_esti = nbe)
+ make_fp_vasp_incar(iter_index, jdata, nbands_esti = nbe)
# 3, create kpoints
- _make_fp_vasp_kp(iter_index, jdata)
+ make_fp_vasp_kp(iter_index, jdata)
+ # 4, copy cvasp
+ make_fp_vasp_cp_cvasp(iter_index,jdata)
def make_fp_pwscf(iter_index,
@@ -1410,6 +1636,21 @@ def make_fp_cp2k (iter_index,
# link pp files
_link_fp_vasp_pp(iter_index, jdata)
+def make_fp_pwmat (iter_index,
+ jdata) :
+ # make config
+ fp_tasks = _make_fp_vasp_configs(iter_index, jdata)
+ if len(fp_tasks) == 0 :
+ return
+ # abs path for fp_incar if it exists
+ if 'fp_incar' in jdata:
+ jdata['fp_incar'] = os.path.abspath(jdata['fp_incar'])
+ # order is critical!
+ # 1, link pp files
+ _link_fp_vasp_pp(iter_index, jdata)
+ # 2, create pwmat input
+ _make_fp_pwmat_input(iter_index, jdata)
+
def make_fp (iter_index,
jdata,
mdata) :
@@ -1425,6 +1666,8 @@ def make_fp (iter_index,
make_fp_gaussian(iter_index, jdata)
elif fp_style == "cp2k" :
make_fp_cp2k(iter_index, jdata)
+ elif fp_style == "pwmat" :
+ make_fp_pwmat(iter_index, jdata)
else :
raise RuntimeError ("unsupported fp style")
@@ -1484,6 +1727,17 @@ def _cp2k_check_fin(ii):
return False
return True
+def _pwmat_check_fin (ii) :
+ if os.path.isfile(os.path.join(ii, 'REPORT')) :
+ with open(os.path.join(ii, 'REPORT'), 'r') as fp :
+ content = fp.read()
+ count = content.count('time')
+ if count != 1 :
+ return False
+ else :
+ return False
+ return True
+
def run_fp_inner (iter_index,
jdata,
mdata,
@@ -1495,6 +1749,7 @@ def run_fp_inner (iter_index,
fp_command = mdata['fp_command']
fp_group_size = mdata['fp_group_size']
fp_resources = mdata['fp_resources']
+ mark_failure = fp_resources.get('mark_failure', False)
iter_name = make_iter_name(iter_index)
work_path = os.path.join(iter_name, fp_name)
@@ -1509,14 +1764,7 @@ def run_fp_inner (iter_index,
# if not check_fin(ii) :
# fp_run_tasks.append(ii)
run_tasks = [os.path.basename(ii) for ii in fp_run_tasks]
- cwd = os.getcwd()
- task_chunks = _split_tasks(run_tasks, fp_group_size)
- nchunks = len(task_chunks)
- if "ali_auth" in mdata:
- dispatcher = ALI(mdata['ali_auth'], mdata['fp_resources'], mdata['fp_machine'], nchunks, work_path)
- dispatcher.init()
- else:
- dispatcher = make_dispatcher(mdata['fp_machine'])
+ dispatcher = make_dispatcher(mdata['fp_machine'], mdata['fp_resources'], work_path, run_tasks, fp_group_size)
dispatcher.run_jobs(mdata['fp_resources'],
[fp_command],
work_path,
@@ -1525,6 +1773,7 @@ def run_fp_inner (iter_index,
forward_common_files,
forward_files,
backward_files,
+ mark_failure = mark_failure,
outlog = log_file,
errlog = log_file)
@@ -1537,15 +1786,15 @@ def run_fp (iter_index,
fp_pp_files = jdata['fp_pp_files']
if fp_style == "vasp" :
- forward_files = ['POSCAR', 'INCAR', 'POTCAR']
+ forward_files = ['POSCAR', 'INCAR', 'POTCAR','KPOINTS']
backward_files = ['OUTCAR','vasprun.xml']
# Move cvasp interface to jdata
if ('cvasp' in jdata) and (jdata['cvasp'] == True):
mdata['fp_resources']['cvasp'] = True
if ('cvasp' in mdata["fp_resources"] ) and (mdata["fp_resources"]["cvasp"]==True):
- #dlog.info("cvasp is on !")
- forward_common_files=['cvasp.py']
- forward_files.append('KPOINTS')
+ dlog.info("cvasp is on !")
+ forward_files.append('cvasp.py')
+ forward_common_files=[]
else:
forward_common_files=[]
run_fp_inner(iter_index, jdata, mdata, forward_files, backward_files, _vasp_check_fin,
@@ -1566,10 +1815,37 @@ def run_fp (iter_index,
forward_files = ['input.inp', 'coord.xyz']
backward_files = ['output']
run_fp_inner(iter_index, jdata, mdata, forward_files, backward_files, _cp2k_check_fin, log_file = 'output')
+ elif fp_style == "pwmat" :
+ forward_files = ['atom.config', 'etot.input'] + fp_pp_files
+ backward_files = ['REPORT', 'OUT.MLMD', 'output']
+ run_fp_inner(iter_index, jdata, mdata, forward_files, backward_files, _pwmat_check_fin, log_file = 'output')
else :
raise RuntimeError ("unsupported fp style")
+def post_fp_check_fail(iter_index,
+ jdata,
+ rfailed = None) :
+ ratio_failed = rfailed if rfailed else jdata.get('ratio_failed',0.05)
+ iter_name = make_iter_name(iter_index)
+ work_path = os.path.join(iter_name, fp_name)
+ fp_tasks = glob.glob(os.path.join(work_path, 'task.*'))
+ fp_tasks.sort()
+ if len(fp_tasks) == 0 :
+ return
+ # check fail according to tag_failure
+ fp_failed_tags = glob.glob(os.path.join(work_path, 'task.*', 'tag_failure*'))
+ fp_failed_tasks = [os.path.dirname(ii) for ii in fp_failed_tags]
+ fp_failed_tasks = list(set(fp_failed_tasks))
+
+ ntask = len(fp_tasks)
+ nfail = len(fp_failed_tasks)
+ rfail = float(nfail) / float(ntask)
+ dlog.info("failed tasks: %6d in %6d %6.2f %% " % (nfail, ntask, rfail * 100.))
+ if rfail > ratio_failed:
+ raise RuntimeError("find too many unsuccessfully terminated jobs")
+
+
def post_fp_vasp (iter_index,
jdata,
rfailed=None):
@@ -1649,12 +1925,10 @@ def post_fp_vasp (iter_index,
else:
raise RuntimeError('invalid setting of use_ele_temp ' + str(use_ele_temp))
- dlog.info("failed frame number: %s "%icount)
- dlog.info("total frame number: %s "%tcount)
- reff=icount/tcount
- dlog.info('ratio of failed frame: {:.2%}'.format(reff))
+ rfail=float(icount)/float(tcount)
+ dlog.info("failed frame: %6d in %6d %6.2f %% " % (icount, tcount, rfail * 100.))
- if reff>ratio_failed:
+ if rfail>ratio_failed:
raise RuntimeError("find too many unsuccessfully terminated jobs")
@@ -1830,9 +2104,64 @@ def post_fp_cp2k (iter_index,
all_sys.to_deepmd_npy(sys_data_path, set_size = len(sys_output))
+def post_fp_pwmat (iter_index,
+ jdata,
+ rfailed=None):
+
+ ratio_failed = rfailed if rfailed else jdata.get('ratio_failed',0.05)
+ model_devi_jobs = jdata['model_devi_jobs']
+ assert (iter_index < len(model_devi_jobs))
+
+ iter_name = make_iter_name(iter_index)
+ work_path = os.path.join(iter_name, fp_name)
+ fp_tasks = glob.glob(os.path.join(work_path, 'task.*'))
+ fp_tasks.sort()
+ if len(fp_tasks) == 0 :
+ return
+
+ system_index = []
+ for ii in fp_tasks :
+ system_index.append(os.path.basename(ii).split('.')[1])
+ system_index.sort()
+ set_tmp = set(system_index)
+ system_index = list(set_tmp)
+ system_index.sort()
+
+ cwd = os.getcwd()
+
+ tcount=0
+ icount=0
+ for ss in system_index :
+ sys_output = glob.glob(os.path.join(work_path, "task.%s.*/OUT.MLMD"%ss))
+ sys_output.sort()
+ tcount += len(sys_output)
+ all_sys = None
+ for oo in sys_output :
+ _sys = dpdata.LabeledSystem(oo, type_map = jdata['type_map'])
+ if len(_sys) == 1:
+ if all_sys is None:
+ all_sys = _sys
+ else:
+ all_sys.append(_sys)
+ else:
+ icount+=1
+ if all_sys is not None:
+ sys_data_path = os.path.join(work_path, 'data.%s'%ss)
+ all_sys.to_deepmd_raw(sys_data_path)
+ all_sys.to_deepmd_npy(sys_data_path, set_size = len(sys_output))
+ dlog.info("failed frame number: %s "%icount)
+ dlog.info("total frame number: %s "%tcount)
+ reff=icount/tcount
+ dlog.info('ratio of failed frame: {:.2%}'.format(reff))
+
+ if reff>ratio_failed:
+ raise RuntimeError("find too many unsuccessfully terminated jobs")
+
+
def post_fp (iter_index,
jdata) :
fp_style = jdata['fp_style']
+ post_fp_check_fail(iter_index, jdata)
if fp_style == "vasp" :
post_fp_vasp(iter_index, jdata)
elif fp_style == "pwscf" :
@@ -1843,6 +2172,8 @@ def post_fp (iter_index,
post_fp_gaussian(iter_index, jdata)
elif fp_style == 'cp2k' :
post_fp_cp2k(iter_index, jdata)
+ elif fp_style == 'pwmat' :
+ post_fp_pwmat(iter_index, jdata)
else :
raise RuntimeError ("unsupported fp style")
# clean traj
diff --git a/dpgen/main.py b/dpgen/main.py
index f5aff6583..7d180b596 100644
--- a/dpgen/main.py
+++ b/dpgen/main.py
@@ -10,6 +10,7 @@
from dpgen.data.gen import gen_init_bulk
from dpgen.data.surf import gen_init_surf
from dpgen.data.reaction import gen_init_reaction
+from dpgen.collect.collect import gen_collect
from dpgen.simplify.simplify import gen_simplify
from dpgen.auto_test.run import gen_test
from dpgen.database.run import db_run
@@ -115,6 +116,24 @@ def main():
help="being loud")
parser_rr.set_defaults(func=run_report)
+ # collect
+ parser_coll = subparsers.add_parser(
+ "collect",
+ help="Collect data.")
+ parser_coll.add_argument("JOB_DIR", type=str,
+ help="the directory of the DP-GEN job")
+ parser_coll.add_argument("OUTPUT", type=str,
+ help="the output directory of data")
+ parser_coll.add_argument('-p',"--parameter", type=str, default = 'param.json',
+ help="the json file provides DP-GEN paramters, should be located in JOB_DIR")
+ parser_coll.add_argument('-v',"--verbose", action = 'store_true',
+ help="print number of data in each system")
+ parser_coll.add_argument('-m',"--merge", action = 'store_true',
+ help="merge the systems with the same chemical formula")
+ parser_coll.add_argument('-s',"--shuffle", action = 'store_true',
+ help="shuffle the data systems")
+ parser_coll.set_defaults(func=gen_collect)
+
# simplify
parser_run = subparsers.add_parser(
"simplify",
diff --git a/dpgen/remote/RemoteJob.py b/dpgen/remote/RemoteJob.py
index 6997c583a..2450b59f5 100644
--- a/dpgen/remote/RemoteJob.py
+++ b/dpgen/remote/RemoteJob.py
@@ -755,7 +755,8 @@ def submit(self,
while self.check_limit(task_max=resources['task_max']):
time.sleep(60)
self._submit(job_dirs, cmd, args, resources)
- time.sleep(20) # For preventing the crash of the tasks while submitting.
+ if resources.get('wait_time', False):
+ time.sleep(resources['wait_time']) # For preventing the crash of the tasks while submitting.
def _submit(self,
job_dirs,
@@ -807,11 +808,11 @@ def check_status(self) :
status_word = status_line.split()[2]
# ref: https://www.ibm.com/support/knowledgecenter/en/SSETD4_9.1.2/lsf_command_ref/bjobs.1.html
- if status_word in ["PEND", "WAIT"] :
+ if status_word in ["PEND", "WAIT", "PSUSP"] :
return JobStatus.waiting
- elif status_word in ["RUN"] :
+ elif status_word in ["RUN", "USUSP"] :
return JobStatus.running
- elif status_word in ["DONE","EXIT"] :
+ elif status_word in ["DONE","EXIT"] :
if self._check_finish_tag() :
return JobStatus.finished
else :
@@ -845,14 +846,19 @@ def _make_script(self,
ret = ''
ret += "#!/bin/bash -l\n#BSUB -e %J.err\n#BSUB -o %J.out\n"
if res['numb_gpu'] == 0:
- ret += '#BSUB -R span[ptile=%d]\n#BSUB -n %d\n' % (
- res['node_cpu'], res['numb_node'] * res['task_per_node'])
+ ret += '#BSUB -n %d\n#BSUB -R span[ptile=%d]\n' % (
+ res['numb_node'] * res['task_per_node'], res['node_cpu'])
else:
if res['node_cpu']:
ret += '#BSUB -R span[ptile=%d]\n' % res['node_cpu']
- # It is selected only for the situation that GPU is related to CPU node.
- ret += '#BSUB -R "select[ngpus >0] rusage[ngpus_excl_p=1]"\n#BSUB -n %d\n' % (
- res['numb_gpu'])
+ if 'new_lsf_gpu' in res and res['new_lsf_gpu'] == True:
+ # supportted in LSF >= 10.1.0 SP6
+ # ref: https://www.ibm.com/support/knowledgecenter/en/SSWRJV_10.1.0/lsf_resource_sharing/use_gpu_res_reqs.html
+ ret += '#BSUB -n %d\n#BSUB -gpu "num=%d:mode=shared:j_exclusive=yes"\n' % (
+ res['numb_gpu'], res['task_per_node'])
+ else:
+ ret += '#BSUB -n %d\n#BSUB -R "select[ngpus >0] rusage[ngpus_excl_p=%d]"\n' % (
+ res['numb_gpu'], res['task_per_node'])
if res['time_limit']:
ret += '#BSUB -W %s\n' % (res['time_limit'].split(':')[
0] + ':' + res['time_limit'].split(':')[1])
diff --git a/dpgen/simplify/simplify.py b/dpgen/simplify/simplify.py
index ba99b0b6c..9856dc58a 100644
--- a/dpgen/simplify/simplify.py
+++ b/dpgen/simplify/simplify.py
@@ -15,7 +15,7 @@
import argparse
import pickle
import glob
-
+import fnmatch
import dpdata
import numpy as np
@@ -24,7 +24,7 @@
from dpgen.util import sepline
from dpgen.remote.decide_machine import decide_train_machine
from dpgen.dispatcher.Dispatcher import Dispatcher, make_dispatcher
-from dpgen.generator.run import make_train, run_train, post_train, run_fp, post_fp, fp_name, model_devi_name, train_name
+from dpgen.generator.run import make_train, run_train, post_train, run_fp, post_fp, fp_name, model_devi_name, train_name, train_task_fmt, sys_link_fp_vasp_pp, make_fp_vasp_incar, make_fp_vasp_kp, make_fp_vasp_cp_cvasp, data_system_fmt, model_devi_task_fmt, fp_task_fmt
# TODO: maybe the following functions can be moved to dpgen.util
from dpgen.generator.lib.utils import log_iter, make_iter_name, create_path, record_iter
from dpgen.remote.decide_machine import decide_train_machine, decide_fp_machine, decide_model_devi_machine
@@ -35,6 +35,19 @@
rest_data_name = "data.rest"
accurate_data_name = "data.accurate"
detail_file_name_prefix = "details"
+sys_name_fmt = 'sys.' + data_system_fmt
+sys_name_pattern = 'sys.[0-9]*[0-9]'
+
+def expand_sys_str(root_dir):
+ matches = []
+ for root, dirnames, filenames in os.walk(root_dir, followlinks=True):
+ for filename in fnmatch.filter(filenames, 'type.raw'):
+ matches.append(root)
+ matches.sort()
+ dirnames = [os.path.basename(ii) for ii in matches]
+ if (len(list(set(dirnames))) != len(matches)) :
+ raise RuntimeError('duplicated system name: it is highly recommend to place all systems in the same level of directory and has different names')
+ return matches
def get_system_cls(jdata):
@@ -43,29 +56,80 @@ def get_system_cls(jdata):
return dpdata.System
-def get_systems(path, jdata):
+def get_multi_system(path, jdata):
system = get_system_cls(jdata)
systems = dpdata.MultiSystems(
*[system(os.path.join(path, s), fmt='deepmd/npy') for s in os.listdir(path)])
return systems
+def get_systems(path, jdata):
+ system_cls = get_system_cls(jdata)
+ system_paths = expand_sys_str(path)
+ systems = {}
+ for ii in system_paths:
+ systems[os.path.basename(ii)] = system_cls(ii, fmt='deepmd/npy')
+ return systems
+
+
+def get_system_idx(path):
+ system_paths = expand_sys_str(path)
+ sys_idx_map = {}
+ for idx,ii in enumerate(system_paths):
+ sys_idx_map[os.path.basename(ii)] = idx
+ return sys_idx_map
+
+
+def init_model(iter_index, jdata, mdata):
+ training_init_model = jdata.get('training_init_model', False)
+ if not training_init_model:
+ return
+ iter0_models = []
+ training_iter0_model = jdata.get('training_iter0_model_path', [])
+ if type(training_iter0_model) == str:
+ training_iter0_model = [training_iter0_model]
+ for ii in training_iter0_model:
+ model_is = glob.glob(ii)
+ model_is.sort()
+ iter0_models += [os.path.abspath(ii) for ii in model_is]
+ numb_models = jdata['numb_models']
+ assert(numb_models == len(iter0_models)), "training_iter0_model_path should be provided, and the number of models should be equal to %d" % numb_models
+ work_path = os.path.join(make_iter_name(iter_index), train_name)
+ create_path(work_path)
+ cwd = os.getcwd()
+ for ii in range(len(iter0_models)):
+ train_path = os.path.join(work_path, train_task_fmt % ii)
+ create_path(train_path)
+ os.chdir(train_path)
+ ckpt_files = glob.glob(os.path.join(iter0_models[ii], 'model.ckpt*'))
+ for jj in ckpt_files:
+ os.symlink(jj, os.path.basename(jj))
+ os.chdir(cwd)
+
+
def init_pick(iter_index, jdata, mdata):
"""pick up init data from dataset randomly"""
pick_data = jdata['pick_data']
init_pick_number = jdata['init_pick_number']
+ use_clusters = jdata.get('use_clusters', False)
# use MultiSystems with System
# TODO: support System and LabeledSystem
# TODO: support other format
- systems = get_systems(pick_data, jdata)
+ if use_clusters:
+ systems = get_multi_system(pick_data, jdata)
+ else:
+ systems = get_systems(pick_data, jdata)
# label the system
labels = []
- for key, system in systems.systems.items():
+ if use_clusters:
+ items = systems.systems.items()
+ else:
+ items = systems.items()
+ for key, system in items:
labels.extend([(key, j) for j in range(len(system))])
# random pick
iter_name = make_iter_name(iter_index)
- create_path(iter_name)
work_path = os.path.join(iter_name, model_devi_name)
create_path(work_path)
idx = np.arange(len(labels))
@@ -74,27 +138,56 @@ def init_pick(iter_index, jdata, mdata):
rest_idx = idx[init_pick_number:]
# dump the init data
- picked_systems = dpdata.MultiSystems()
- for j in pick_idx:
- sys_name, sys_id = labels[j]
- picked_systems.append(systems[sys_name][sys_id])
sys_data_path = os.path.join(work_path, picked_data_name)
-
- picked_systems.to_deepmd_raw(sys_data_path)
- picked_systems.to_deepmd_npy(sys_data_path, set_size=init_pick_number)
+ _init_dump_selected_frames(systems, labels, pick_idx, sys_data_path, jdata)
# dump the rest data
- rest_systems = dpdata.MultiSystems()
- for j in rest_idx:
- sys_name, sys_id = labels[j]
- rest_systems.append(systems[sys_name][sys_id])
sys_data_path = os.path.join(work_path, rest_data_name)
- rest_systems.to_deepmd_raw(sys_data_path)
- rest_systems.to_deepmd_npy(sys_data_path, set_size=rest_idx.size)
+ _init_dump_selected_frames(systems, labels, rest_idx, sys_data_path, jdata)
+
+
+def _add_system(systems, key, system):
+ if key in systems.keys():
+ systems[key].append(system)
+ else:
+ systems[key] = system
+ return systems
+
+
+def _init_dump_selected_frames(systems, labels, selc_idx, sys_data_path, jdata):
+ pick_data = jdata['pick_data']
+ use_clusters = jdata.get('use_clusters', False)
+ if use_clusters:
+ selc_systems = dpdata.MultiSystems()
+ for j in selc_idx:
+ sys_name, sys_id = labels[j]
+ selc_systems.append(systems[sys_name][sys_id])
+ selc_systems.to_deepmd_raw(sys_data_path)
+ selc_systems.to_deepmd_npy(sys_data_path, set_size=selc_idx.size)
+ else:
+ selc_systems = {}
+ for j in selc_idx:
+ sys_name, sys_id = labels[j]
+ selc_systems = _add_system(selc_systems, sys_name, systems[sys_name][sys_id])
+ sys_idx_map = get_system_idx(pick_data)
+ for kk in selc_systems.keys():
+ sub_path = os.path.join(sys_data_path, sys_name_fmt % sys_idx_map[kk])
+ selc_systems[kk].to_deepmd_raw(sub_path)
+ selc_systems[kk].to_deepmd_npy(sub_path, set_size=selc_idx.size)
+ with open(os.path.join(sys_data_path, 'sys_idx_map.json'), 'w') as fp:
+ json.dump(sys_idx_map, fp, indent=4)
+
+def _dump_system_dict(systems, path):
+ for kk in systems:
+ sub_path = os.path.join(path, sys_name_fmt % (int(kk)))
+ systems[kk].to_deepmd_raw(sub_path)
+ systems[kk].to_deepmd_npy(sub_path, set_size=systems[kk].get_nframes())
def make_model_devi(iter_index, jdata, mdata):
"""calculate the model deviation of the rest idx"""
+ pick_data = jdata['pick_data']
+ use_clusters = jdata.get('use_clusters', False)
iter_name = make_iter_name(iter_index)
work_path = os.path.join(iter_name, model_devi_name)
create_path(work_path)
@@ -110,16 +203,29 @@ def make_model_devi(iter_index, jdata, mdata):
rest_data_path = os.path.join(last_iter_name, model_devi_name, rest_data_name)
if not os.path.exists(rest_data_path):
return False
- for jj, subsystem in enumerate(os.listdir(rest_data_path)):
- task_name = "task.%03d.%06d" % (0, jj)
- task_path = os.path.join(work_path, task_name)
- create_path(task_path)
- os.symlink(os.path.abspath(os.path.join(rest_data_path, subsystem)),
- os.path.abspath(os.path.join(task_path, rest_data_name)))
+ if use_clusters:
+ for jj, subsystem in enumerate(os.listdir(rest_data_path)):
+ task_name = "task." + model_devi_task_fmt % (0, jj)
+ task_path = os.path.join(work_path, task_name)
+ create_path(task_path)
+ os.symlink(os.path.abspath(os.path.join(rest_data_path, subsystem)),
+ os.path.abspath(os.path.join(task_path, rest_data_name)))
+ else:
+ rest_data_path = os.path.abspath(rest_data_path)
+ sys_path = glob.glob(os.path.join(rest_data_path, sys_name_pattern))
+ cwd = os.getcwd()
+ for ii in sys_path:
+ task_name = "task." + model_devi_task_fmt % (int(os.path.basename(ii).split('.')[1]), 0)
+ task_path = os.path.join(work_path, task_name)
+ create_path(task_path)
+ os.chdir(task_path)
+ os.symlink(os.path.relpath(ii), rest_data_name)
+ os.chdir(cwd)
+ os.chdir(cwd)
return True
-def run_model_devi(iter_index, jdata, mdata, dispatcher):
+def run_model_devi(iter_index, jdata, mdata):
"""submit dp test tasks"""
iter_name = make_iter_name(iter_index)
work_path = os.path.join(iter_name, model_devi_name)
@@ -163,6 +269,7 @@ def run_model_devi(iter_index, jdata, mdata, dispatcher):
forward_files = [rest_data_name]
backward_files = sum([[pf+".e.out", pf+".f.out", pf+".v.out"] for pf in detail_file_names], [])
+ dispatcher = make_dispatcher(mdata['model_devi_machine'], mdata['model_devi_resources'], work_path, run_tasks, model_devi_group_size)
dispatcher.run_jobs(mdata['model_devi_resources'],
commands,
work_path,
@@ -177,20 +284,31 @@ def run_model_devi(iter_index, jdata, mdata, dispatcher):
def post_model_devi(iter_index, jdata, mdata):
"""calculate the model deviation"""
+ use_clusters = jdata.get('use_clusters', False)
iter_name = make_iter_name(iter_index)
work_path = os.path.join(iter_name, model_devi_name)
tasks = glob.glob(os.path.join(work_path, "task.*"))
+ tasks.sort()
e_trust_lo = jdata['e_trust_lo']
e_trust_hi = jdata['e_trust_hi']
f_trust_lo = jdata['f_trust_lo']
f_trust_hi = jdata['f_trust_hi']
- sys_accurate = dpdata.MultiSystems()
- sys_candinate = dpdata.MultiSystems()
- sys_failed = dpdata.MultiSystems()
+ if use_clusters:
+ sys_accurate = dpdata.MultiSystems()
+ sys_candinate = dpdata.MultiSystems()
+ sys_failed = dpdata.MultiSystems()
+ else:
+ sys_accurate = {}
+ sys_candinate = {}
+ sys_failed = {}
+ all_names = set()
for task in tasks:
+ if not use_clusters:
+ sys_name = os.path.basename(task).split('.')[1]
+ all_names.add(sys_name)
# e.out
details_e = glob.glob(os.path.join(task, "{}.*.e.out".format(detail_file_name_prefix)))
e_all = np.array([np.loadtxt(detail_e, ndmin=2)[:, 1] for detail_e in details_e])
@@ -211,86 +329,233 @@ def post_model_devi(iter_index, jdata, mdata):
system_cls = get_system_cls(jdata)
for subsys, e_devi, f_devi in zip(system_cls(os.path.join(task, rest_data_name), fmt='deepmd/npy'), e_std, f_std):
if (e_devi < e_trust_hi and e_devi >= e_trust_lo) or (f_devi < f_trust_hi and f_devi >= f_trust_lo) :
- sys_candinate.append(subsys)
+ if use_clusters:
+ sys_candinate.append(subsys)
+ else:
+ sys_candinate = _add_system(sys_candinate, sys_name, subsys)
elif (e_devi >= e_trust_hi ) or (f_devi >= f_trust_hi ):
- sys_failed.append(subsys)
+ if use_clusters:
+ sys_failed.append(subsys)
+ else:
+ sys_failed = _add_system(sys_failed, sys_name, subsys)
elif (e_devi < e_trust_lo and f_devi < f_trust_lo ):
- sys_accurate.append(subsys)
- counter = {"candidate": sys_candinate.get_nframes(), "accurate": sys_accurate.get_nframes(), "failed": sys_failed.get_nframes()}
- fp_sum = sum(counter.values())
- for cc_key, cc_value in counter.items():
- dlog.info("{0:9s} : {1:6d} in {2:6d} {3:6.2f} %".format(cc_key, cc_value, fp_sum, cc_value/fp_sum*100))
+ if use_clusters:
+ sys_accurate.append(subsys)
+ else:
+ sys_accurate = _add_system(sys_accurate, sys_name, subsys)
+ else:
+ raise RuntimeError('reach a place that should NOT be reached...')
+ if use_clusters:
+ counter = {"candidate": sys_candinate.get_nframes(), "accurate": sys_accurate.get_nframes(), "failed": sys_failed.get_nframes()}
+ fp_sum = sum(counter.values())
+ for cc_key, cc_value in counter.items():
+ dlog.info("{0:9s} : {1:6d} in {2:6d} {3:6.2f} %".format(cc_key, cc_value, fp_sum, cc_value/fp_sum*100))
+ else:
+ all_names = list(all_names)
+ all_names.sort()
+ counter = {"candidate": 0, "accurate": 0, "failed": 0}
+ for kk in all_names:
+ sys_counter = {"candidate": 0, "accurate": 0, "failed": 0}
+ if kk in sys_candinate.keys():
+ sys_counter['candidate'] += sys_candinate[kk].get_nframes()
+ if kk in sys_accurate.keys():
+ sys_counter['accurate'] += sys_accurate[kk].get_nframes()
+ if kk in sys_failed.keys():
+ sys_counter['failed'] += sys_failed[kk].get_nframes()
+ fp_sum = sum(sys_counter.values())
+ for cc_key, cc_value in sys_counter.items():
+ if fp_sum != 0:
+ dlog.info("sys{0:s} {1:9s} : {2:6d} in {3:6d} {4:6.2f} %".format(kk, cc_key, cc_value, fp_sum, cc_value/fp_sum*100))
+ else:
+ dlog.info("sys{0:s} {1:9s} : {2:6d} in {3:6d} {4:6.2f} %".format(kk, cc_key, cc_value, fp_sum, 0*100))
+ for ii in ['candidate', 'accurate', 'failed']:
+ counter[ii] += sys_counter[ii]
+ if counter['candidate'] == 0 and counter['failed'] > 0:
+ raise RuntimeError('no candidate but still have failed cases, stop. You may want to refine the training or to increase the trust level hi')
+
# label the candidate system
labels = []
- for key, system in sys_candinate.systems.items():
+ if use_clusters:
+ items = sys_candinate.systems.items()
+ else:
+ items = sys_candinate.items()
+ for key, system in items:
labels.extend([(key, j) for j in range(len(system))])
# candinate: pick up randomly
iter_pick_number = jdata['iter_pick_number']
idx = np.arange(counter['candidate'])
+ assert(len(idx) == len(labels))
np.random.shuffle(idx)
pick_idx = idx[:iter_pick_number]
rest_idx = idx[iter_pick_number:]
+ dlog.info("total candidate {0:6d} picked {1:6d} ({2:6.2f} %) rest {3:6d} ({4:6.2f} % )".format\
+ (counter['candidate'], len(pick_idx), float(len(pick_idx))/counter['candidate']*100., len(rest_idx), float(len(rest_idx))/counter['candidate']*100.))
# dump the picked candinate data
- picked_systems = dpdata.MultiSystems()
- for j in pick_idx:
- sys_name, sys_id = labels[j]
- picked_systems.append(sys_candinate[sys_name][sys_id])
- sys_data_path = os.path.join(work_path, picked_data_name)
-
- picked_systems.to_deepmd_raw(sys_data_path)
- picked_systems.to_deepmd_npy(sys_data_path, set_size=iter_pick_number)
+ if use_clusters:
+ picked_systems = dpdata.MultiSystems()
+ for j in pick_idx:
+ sys_name, sys_id = labels[j]
+ picked_systems.append(sys_candinate[sys_name][sys_id])
+ sys_data_path = os.path.join(work_path, picked_data_name)
+ picked_systems.to_deepmd_raw(sys_data_path)
+ picked_systems.to_deepmd_npy(sys_data_path, set_size=iter_pick_number)
+ else:
+ selc_systems = {}
+ for j in pick_idx:
+ sys_name, sys_id = labels[j]
+ selc_systems = _add_system(selc_systems, sys_name, sys_candinate[sys_name][sys_id])
+ sys_data_path = os.path.join(work_path, picked_data_name)
+ _dump_system_dict(selc_systems, sys_data_path)
# dump the rest data (not picked candinate data and failed data)
- rest_systems = dpdata.MultiSystems()
- for j in rest_idx:
- sys_name, sys_id = labels[j]
- rest_systems.append(sys_candinate[sys_name][sys_id])
- rest_systems += sys_failed
- sys_data_path = os.path.join(work_path, rest_data_name)
- rest_systems.to_deepmd_raw(sys_data_path)
- rest_systems.to_deepmd_npy(sys_data_path, set_size=rest_idx.size)
+ if use_clusters:
+ rest_systems = dpdata.MultiSystems()
+ for j in rest_idx:
+ sys_name, sys_id = labels[j]
+ rest_systems.append(sys_candinate[sys_name][sys_id])
+ rest_systems += sys_failed
+ sys_data_path = os.path.join(work_path, rest_data_name)
+ rest_systems.to_deepmd_raw(sys_data_path)
+ rest_systems.to_deepmd_npy(sys_data_path, set_size=rest_idx.size)
+ else:
+ selc_systems = {}
+ for j in rest_idx:
+ sys_name, sys_id = labels[j]
+ selc_systems = _add_system(selc_systems, sys_name, sys_candinate[sys_name][sys_id])
+ for kk in sys_failed.keys():
+ selc_systems = _add_system(selc_systems, kk, sys_failed[kk])
+ sys_data_path = os.path.join(work_path, rest_data_name)
+ _dump_system_dict(selc_systems, sys_data_path)
# dump the accurate data -- to another directory
- sys_data_path = os.path.join(work_path, accurate_data_name)
- sys_accurate.to_deepmd_raw(sys_data_path)
- sys_accurate.to_deepmd_npy(sys_data_path, set_size=sys_accurate.get_nframes())
+ if use_clusters:
+ sys_data_path = os.path.join(work_path, accurate_data_name)
+ sys_accurate.to_deepmd_raw(sys_data_path)
+ sys_accurate.to_deepmd_npy(sys_data_path, set_size=sys_accurate.get_nframes())
+ else:
+ sys_data_path = os.path.join(work_path, accurate_data_name)
+ _dump_system_dict(sys_accurate, sys_data_path)
-def make_fp(iter_index, jdata, mdata):
+def make_fp_labeled(iter_index, jdata):
+ dlog.info("already labeled, skip make_fp and link data directly")
+ pick_data = jdata['pick_data']
+ use_clusters = jdata.get('use_clusters', False)
iter_name = make_iter_name(iter_index)
work_path = os.path.join(iter_name, fp_name)
create_path(work_path)
picked_data_path = os.path.join(iter_name, model_devi_name, picked_data_name)
- if jdata.get("labeled", False):
- dlog.info("already labeled, skip make_fp and link data directly")
+ if use_clusters:
os.symlink(os.path.abspath(picked_data_path), os.path.abspath(
- os.path.join(work_path, "task.%03d" % 0)))
+ os.path.join(work_path, "task." + data_system_fmt % 0)))
os.symlink(os.path.abspath(picked_data_path), os.path.abspath(
- os.path.join(work_path, "data.%03d" % 0)))
- return
- systems = get_systems(picked_data_path, jdata)
- fp_style = jdata['fp_style']
+ os.path.join(work_path, "data." + data_system_fmt % 0)))
+ else:
+ picked_data_path = os.path.abspath(picked_data_path)
+ sys_path = glob.glob(os.path.join(picked_data_path, sys_name_pattern))
+ cwd = os.getcwd()
+ os.chdir(work_path)
+ for ii in sys_path:
+ sys_idx = os.path.basename(ii).split('.')[1]
+ data_dir = 'data.' + data_system_fmt % int(sys_idx)
+ task_dir = 'task.' + data_system_fmt % int(sys_idx)
+ os.symlink(os.path.relpath(ii), data_dir)
+ os.symlink(os.path.relpath(ii), task_dir)
+ os.chdir(cwd)
+
+
+def make_fp_configs(iter_index, jdata):
+ pick_data = jdata['pick_data']
+ use_clusters = jdata.get('use_clusters', False)
+ iter_name = make_iter_name(iter_index)
+ work_path = os.path.join(iter_name, fp_name)
+ create_path(work_path)
+ picked_data_path = os.path.join(iter_name, model_devi_name, picked_data_name)
+ if use_clusters:
+ systems = get_multi_system(picked_data_path, jdata)
+ jj = 0
+ for system in systems:
+ for subsys in system:
+ task_name = "task." + fp_task_fmt % (0, jj)
+ task_path = os.path.join(work_path, task_name)
+ create_path(task_path)
+ subsys.to('vasp/poscar', os.path.join(task_path, 'POSCAR'))
+ jj += 1
+ else:
+ picked_data_path = os.path.abspath(picked_data_path)
+ sys_path = glob.glob(os.path.join(picked_data_path, sys_name_pattern))
+ for ii in sys_path:
+ tmp_sys = dpdata.System(ii, fmt = 'deepmd/npy')
+ sys_idx = os.path.basename(ii).split('.')[1]
+ jj = 0
+ for ss in tmp_sys:
+ task_name = "task." + fp_task_fmt % (int(sys_idx), jj)
+ task_path = os.path.join(work_path, task_name)
+ create_path(task_path)
+ ss.to('vasp/poscar', os.path.join(task_path, 'POSCAR'))
+ job = {}
+ with open(os.path.join(task_path, 'job.json'), 'w') as fp:
+ json.dump(job, fp, indent=4)
+ jj += 1
+
+
+def make_fp_gaussian(iter_index, jdata):
+ work_path = os.path.join(make_iter_name(iter_index), fp_name)
+ fp_tasks = glob.glob(os.path.join(work_path, 'task.*'))
+ cwd = os.getcwd()
if 'user_fp_params' in jdata.keys() :
fp_params = jdata['user_fp_params']
else:
fp_params = jdata['fp_params']
- jj = 0
- for system in systems:
- for subsys in system:
- sys_data = subsys.data
- task_name = "task.%03d.%06d" % (0, jj)
- task_path = os.path.join(work_path, task_name)
- create_path(task_path)
- if fp_style == "gaussian" :
- ret = make_gaussian_input(sys_data, fp_params)
- with open(os.path.join(task_path, 'input'), 'w') as fp:
- fp.write(ret)
- else :
- # TODO: support other formats
- raise RuntimeError ("unsupported fp style")
- jj += 1
+ cwd = os.getcwd()
+ for ii in fp_tasks:
+ os.chdir(ii)
+ sys_data = dpdata.System('POSCAR').data
+ ret = make_gaussian_input(sys_data, fp_params)
+ with open('input', 'w') as fp:
+ fp.write(ret)
+ os.chdir(cwd)
+
+
+def make_fp_vasp(iter_index, jdata):
+ # abs path for fp_incar if it exists
+ if 'fp_incar' in jdata:
+ jdata['fp_incar'] = os.path.abspath(jdata['fp_incar'])
+ # get nbands esti if it exists
+ if 'fp_nbands_esti_data' in jdata:
+ nbe = NBandsEsti(jdata['fp_nbands_esti_data'])
+ else:
+ nbe = None
+ # order is critical!
+ # 1, create potcar
+ sys_link_fp_vasp_pp(iter_index, jdata)
+ # 2, create incar
+ make_fp_vasp_incar(iter_index, jdata, nbands_esti = nbe)
+ # 3, create kpoints
+ make_fp_vasp_kp(iter_index, jdata)
+ # 4, copy cvasp
+ make_fp_vasp_cp_cvasp(iter_index,jdata)
+
+
+def make_fp_calculation(iter_index, jdata):
+ fp_style = jdata['fp_style']
+ if fp_style == 'vasp':
+ make_fp_vasp(iter_index, jdata)
+ elif fp_style == 'gaussian':
+ make_fp_gaussian(iter_index, jdata)
+ else :
+ raise RuntimeError('unsupported fp_style ' + fp_style)
+
+
+def make_fp(iter_index, jdata, mdata):
+ labeled = jdata.get("labeled", False)
+ if labeled:
+ make_fp_labeled(iter_index, jdata)
+ else:
+ make_fp_configs(iter_index, jdata)
+ make_fp_calculation(iter_index, jdata)
def run_iter(param_file, machine_file):
@@ -365,6 +630,7 @@ def run_iter(param_file, machine_file):
if ii == 0 and jj < 6:
if jj == 0:
log_iter("init_pick", ii, jj)
+ init_model(ii, jdata, mdata)
init_pick(ii, jdata, mdata)
dlog.info("first iter, skip step 1-5")
elif jj == 0:
@@ -373,7 +639,7 @@ def run_iter(param_file, machine_file):
elif jj == 1:
log_iter("run_train", ii, jj)
mdata = decide_train_machine(mdata)
- disp = make_dispatcher(mdata['train_machine'])
+ #disp = make_dispatcher(mdata['train_machine'])
run_train(ii, jdata, mdata)
elif jj == 2:
log_iter("post_train", ii, jj)
@@ -386,8 +652,8 @@ def run_iter(param_file, machine_file):
elif jj == 4:
log_iter("run_model_devi", ii, jj)
mdata = decide_model_devi_machine(mdata)
- disp = make_dispatcher(mdata['model_devi_machine'])
- run_model_devi(ii, jdata, mdata, disp)
+ #disp = make_dispatcher(mdata['model_devi_machine'])
+ run_model_devi(ii, jdata, mdata)
elif jj == 5:
log_iter("post_model_devi", ii, jj)
post_model_devi(ii, jdata, mdata)
@@ -400,7 +666,7 @@ def run_iter(param_file, machine_file):
dlog.info("already have labeled data, skip run_fp")
else:
mdata = decide_fp_machine(mdata)
- disp = make_dispatcher(mdata['fp_machine'])
+ #disp = make_dispatcher(mdata['fp_machine'])
run_fp(ii, jdata, mdata)
elif jj == 8:
log_iter("post_fp", ii, jj)
diff --git a/examples/machine/DeePMD-kit-1.0/machine-ali.json b/examples/machine/DeePMD-kit-1.0/machine-ali.json
new file mode 100644
index 000000000..e539ee74e
--- /dev/null
+++ b/examples/machine/DeePMD-kit-1.0/machine-ali.json
@@ -0,0 +1,115 @@
+{
+ "train": [
+ {
+ "machine": {
+ "batch": "shell",
+ "hostname": "",
+ "password": "975481DING!",
+ "port": 22,
+ "username": "root",
+ "work_path": "/root/dpgen_work",
+ "ali_auth": {
+ "AccessKey_ID":"",
+ "AccessKey_Secret":"",
+ "regionID": "cn-shenzhen",
+ "img_name": "kit",
+ "machine_type_price": [
+ {"machine_type": "ecs.gn6v-c8g1.2xlarge", "price_limit": 20.00, "numb": 1, "priority": 0},
+ {"machine_type": "ecs.gn5-c4g1.xlarge", "price_limit": 20.00, "numb": 1, "priority": 1}
+ ],
+ "instance_name": "CH4",
+ "pay_strategy": "spot"
+ }
+ },
+ "resources": {
+ "numb_gpu": 1,
+ "numb_node": 1,
+ "task_per_node": 12,
+ "partition": "gpu",
+ "exclude_list": [],
+ "mem_limit": 32,
+ "source_list": [],
+ "module_list": [],
+ "time_limit": "23:0:0"
+ },
+ "command": "/root/deepmd-kit/bin/dp",
+ "group_size": 2
+ }
+ ],
+
+ "model_devi": [
+ {
+ "machine": {
+ "batch": "shell",
+ "hostname": "",
+ "password": "975481DING!",
+ "port": 22,
+ "username": "root",
+ "work_path": "/root/dpgen_work",
+ "ali_auth": {
+ "AccessKey_ID":"",
+ "AccessKey_Secret":"",
+ "regionID": "cn-shenzhen",
+ "img_name": "kit",
+ "machine_type_price": [
+ {"machine_type": "ecs.gn6v-c8g1.2xlarge", "price_limit": 20.00, "numb": 1, "priority": 0},
+ {"machine_type": "ecs.gn5-c4g1.xlarge", "price_limit": 20.00, "numb": 1, "priority": 1}
+ ],
+ "instance_name": "CH4",
+ "pay_strategy": "spot"
+ }
+ },
+ "resources": {
+ "numb_gpu": 1,
+ "task_per_node": 4,
+ "partition": "gpu",
+ "exclude_list": [],
+ "mem_limit": 11,
+ "source_list": [],
+ "module_list": [],
+ "time_limit": "23:0:0"
+ },
+ "command": "/root/deepmd-kit/bin/lmp",
+ "group_size": 2
+ }
+ ],
+
+ "fp": [
+ {
+ "machine": {
+ "batch": "shell",
+ "hostname": "",
+ "password": "975481DING!",
+ "port": 22,
+ "username": "root",
+ "work_path": "/root/dpgen_work",
+ "ali_auth": {
+ "AccessKey_ID":"",
+ "AccessKey_Secret":"",
+ "regionID": "cn-shenzhen",
+ "img_name": "vasp",
+ "machine_type_price": [
+ {"machine_type": "ecs.c6.4xlarge", "price_limit": 0.2, "numb": 16, "priority": 0},
+ {"machine_type": "ecs.g6.4xlarge", "price_limit": 0.2, "numb": 16, "priority": 1}
+ ],
+ "instance_name": "CH4",
+ "pay_strategy": "spot"
+ }
+ },
+ "resources": {
+ "numb_gpu": 0,
+ "task_per_node": 16,
+ "with_mpi": "false",
+ "source_list": ["/opt/intel/parallel_studio_xe_2018/psxevars.sh"],
+ "module_list": [],
+ "partition": "cpu",
+ "envs" : {"PATH" : "/root/deepmd-pkg/vasp.5.4.4/bin:$PATH"}
+ },
+ "command": "mpirun -n 16 /root/deepmd-pkg/vasp.5.4.4/bin/vasp_std",
+ "group_size": 1
+ }
+ ]
+}
+
+
+
diff --git a/examples/run/dp-lammps-pwmat/machine-slurm-pwmat-single.json b/examples/run/dp-lammps-pwmat/machine-slurm-pwmat-single.json
new file mode 100644
index 000000000..9138f0cbf
--- /dev/null
+++ b/examples/run/dp-lammps-pwmat/machine-slurm-pwmat-single.json
@@ -0,0 +1,80 @@
+{
+ "train": [
+ {
+ "machine": {
+ "machine_type": "slurm",
+ "hostname": "mstation",
+ "port": 22,
+ "username": "test",
+ "password": "PWmat2019",
+ "work_path": "/home/test/software/dpgen/examples/run/dp-lammps-pwmat/work_train"
+ },
+ "resources": {
+ "numb_node": 1,
+ "numb_gpu": 4,
+ "task_per_node": 4,
+ "partition": "control",
+ "exclude_list": [],
+ "source_list": [
+ "/home/test/software/dpgen/examples/run/dp-lammps-pwmat/train.env"
+ ],
+ "module_list": [],
+ "time_limit": "23:0:0"
+ },
+ "deepmd_path" : "/home/test/anaconda2/envs/python3/"
+ }
+ ],
+ "model_devi": [
+ {
+ "machine": {
+ "machine_type": "slurm",
+ "hostname": "mstation",
+ "port": 22,
+ "username": "test",
+ "password": "PWmat2019",
+ "work_path": "/home/test/software/dpgen/examples/run/dp-lammps-pwmat/work_model"
+ },
+ "resources": {
+ "numb_node": 1,
+ "numb_gpu": 4,
+ "task_per_node": 4,
+ "partition": "control",
+ "exclude_list": [],
+ "source_list": [
+ "/home/test/software/dpgen/examples/run/dp-lammps-pwmat/train.env"
+ ],
+ "module_list": [],
+ "time_limit": "23:0:0"
+ },
+ "command": "srun --mpi=pmi2 lmp_mpi",
+ "group_size": 10
+ }
+ ],
+ "fp": [
+ {
+ "machine": {
+ "machine_type": "slurm",
+ "hostname": "mstation",
+ "port": 22,
+ "username": "test",
+ "password": "PWmat2019",
+ "work_path": "/home/test/software/dpgen/examples/run/dp-lammps-pwmat/work_fp"
+ },
+ "resources": {
+ "task_per_node": 4,
+ "numb_gpu": 4,
+ "exclude_list": [],
+ "with_mpi": false,
+ "source_list": [],
+ "module_list": [
+ "cuda/8.0"
+ ],
+ "time_limit": "120:0:0",
+ "partition": "control",
+ "_comment": "that's All"
+ },
+ "command": "mpirun -np 4 PWmat",
+ "group_size": 5
+ }
+ ]
+}
diff --git a/examples/run/dp-lammps-pwmat/param_CH4.json b/examples/run/dp-lammps-pwmat/param_CH4.json
new file mode 100644
index 000000000..cda28a5ec
--- /dev/null
+++ b/examples/run/dp-lammps-pwmat/param_CH4.json
@@ -0,0 +1,99 @@
+{
+ "type_map": ["H","C"],
+ "mass_map": [1, 12],
+
+ "init_data_prefix": "/home/test/software/dpgen/examples/run/dp-lammps-pwmat/",
+
+ "init_data_sys": [
+ "ch4/00.data"
+ ],
+ "init_batch_size": [
+ 8
+ ],
+ "sys_configs_prefix":
+ "/home/test/software/dpgen/examples/run/dp-lammps-pwmat/",
+ "sys_configs": [
+ ["/home/test/software/dpgen/examples/run/dp-lammps-pwmat/scale-1.000/00000*/POSCAR"],
+ ["/home/test/software/dpgen/examples/run/dp-lammps-pwmat/scale-1.000/00001*/POSCAR"]
+ ],
+
+ "sys_batch_size": [
+ 8, 8, 8, 8
+ ],
+
+
+ "_comment": " 00.train ",
+ "numb_models": 4,
+ "default_training_param" : {
+ "_comment": " model parameters",
+ "use_smooth": true,
+ "sel_a": [16,4],
+ "rcut_smth": 0.50,
+ "rcut": 5,
+ "filter_neuron": [10, 20, 40],
+ "filter_resnet_dt": false,
+ "n_axis_neuron": 12,
+ "n_neuron": [120,120,120],
+ "resnet_dt": true,
+ "coord_norm": true,
+ "type_fitting_net": false,
+
+ "_comment": " traing controls",
+ "systems": ["./00.data/"],
+ "set_prefix": "set",
+ "stop_batch": 2000,
+ "batch_size": 1,
+ "start_lr": 0.001,
+ "decay_steps": 100,
+ "decay_rate": 0.95,
+ "seed": 0,
+
+ "start_pref_e": 0.02,
+ "limit_pref_e": 2,
+ "start_pref_f": 1000,
+ "limit_pref_f": 1,
+ "start_pref_v": 0.0,
+ "limit_pref_v": 0.0,
+
+ "_comment": " display and restart",
+ "_comment": " frequencies counted in batch",
+ "disp_file": "lcurve.out",
+ "disp_freq": 1000,
+ "numb_test": 4,
+ "save_freq": 1000,
+ "save_ckpt": "model.ckpt",
+ "load_ckpt": "model.ckpt",
+ "disp_training": true,
+ "time_training": true,
+ "profiling": false,
+ "profiling_file": "timeline.json",
+
+ "_comment": "that's all"
+ },
+
+ "_comment": " 01.model_devi ",
+ "_comment": "model_devi_skip: the first x of the recorded frames",
+ "model_devi_dt": 0.002,
+ "model_devi_skip": 0,
+ "model_devi_f_trust_lo": 0.05,
+ "model_devi_f_trust_hi": 0.15,
+ "model_devi_e_trust_lo": 1e10,
+ "model_devi_e_trust_hi": 1e10,
+ "model_devi_clean_traj": true,
+ "model_devi_jobs": [
+ {"sys_idx": [0],
+ "temps": [ 300], "press": [0.0], "trj_freq": 10, "nsteps": 300, "ensemble": "nvt", "_idx": "00"},
+ {"sys_idx": [1],
+ "temps": [ 300], "press": [0.0], "trj_freq": 10, "nsteps": 3000, "ensemble": "nvt", "_idx": "01"}
+ ],
+
+ "_comment": " 02.fp ",
+ "fp_style": "pwmat",
+ "shuffle_poscar": false,
+ "fp_task_max": 20,
+ "fp_task_min": 8,
+ "fp_pp_path": ".",
+ "fp_pp_files": ["C.SG15.PBE.UPF", "H.SG15.PBE.UPF"],
+ "fp_incar" : "etot.input",
+ "_comment": " that's all "
+}
diff --git a/examples/run/dp-lammps-vasp/CH4/POT_C b/examples/run/dp-lammps-vasp/CH4/POT_C
new file mode 100644
index 000000000..8b1378917
--- /dev/null
+++ b/examples/run/dp-lammps-vasp/CH4/POT_C
@@ -0,0 +1 @@
+
diff --git a/examples/run/dp-lammps-vasp/CH4/POT_H b/examples/run/dp-lammps-vasp/CH4/POT_H
new file mode 100644
index 000000000..8b1378917
--- /dev/null
+++ b/examples/run/dp-lammps-vasp/CH4/POT_H
@@ -0,0 +1 @@
+
diff --git a/examples/run/dp-lammps-vasp/CH4/param_CH4.json b/examples/run/dp-lammps-vasp/CH4/param_CH4.json
index 2ec6216d5..f69b4652d 100644
--- a/examples/run/dp-lammps-vasp/CH4/param_CH4.json
+++ b/examples/run/dp-lammps-vasp/CH4/param_CH4.json
@@ -88,12 +88,13 @@
],
"_comment": " 02.fp ",
+ "cvasp": false,
"fp_style": "vasp",
"shuffle_poscar": false,
"fp_task_max": 20,
"fp_task_min": 5,
"fp_pp_path": "/gpfs/share/home/1600017784/yuzhi/methane/",
- "fp_pp_files": ["POTCAR"],
+ "fp_pp_files": ["POT_H","POT_C"],
"fp_incar" : "/gpfs/share/home/1600017784/yuzhi/methane/INCAR_methane",
"_comment": " that's all "
}
diff --git a/setup.py b/setup.py
index 71286b666..1016662a9 100755
--- a/setup.py
+++ b/setup.py
@@ -17,7 +17,7 @@
with open(path.join('dpgen', '_date.py'), 'w') as fp :
fp.write('date = \'%s\'' % today)
-install_requires=['numpy>=1.14.3', 'dpdata>=0.1.12', 'pymatgen>=2019.1.13', 'ase', 'monty>2.0.0', 'paramiko', 'custodian']
+install_requires=['numpy>=1.14.3', 'dpdata>=0.1.14', 'pymatgen>=2019.1.13', 'ase', 'monty>2.0.0', 'paramiko', 'custodian']
setuptools.setup(
name=NAME,
@@ -42,6 +42,7 @@
'dpgen/database',
'dpgen/tools',
'dpgen/simplify',
+ 'dpgen/collect',
],
# data_files = [('dpgen/tools/', ['dpgen/tools/update_time.sh', ])],
# package_data={'example':['*.json']},
diff --git a/tests/auto_test/00.equi/si/mp-149/vasp-relax_incar/CONTCAR b/tests/auto_test/00.equi/si/mp-149/vasp-relax_incar/CONTCAR
new file mode 100644
index 000000000..4fe904343
--- /dev/null
+++ b/tests/auto_test/00.equi/si/mp-149/vasp-relax_incar/CONTCAR
@@ -0,0 +1,25 @@
+Si8
+ 1.00000000000000
+ 5.4687279999999996 0.0000000000000000 0.0000000000000000
+ 0.0000000000000000 5.4687279999999996 0.0000000000000000
+ 0.0000000000000000 0.0000000000000000 5.4687279999999996
+ Si
+ 8
+Direct
+ 0.2500000000000000 0.2500000000000000 0.2500000000000000
+ 0.5000000000000000 0.5000000000000000 0.0000000000000000
+ 0.2500000000000000 0.7500000000000000 0.7500000000000000
+ 0.5000000000000000 0.0000000000000000 0.5000000000000000
+ 0.7500000000000000 0.2500000000000000 0.7500000000000000
+ 0.0000000000000000 0.5000000000000000 0.5000000000000000
+ 0.7500000000000000 0.7500000000000000 0.2500000000000000
+ 0.0000000000000000 0.0000000000000000 0.0000000000000000
+
+ 0.00000000E+00 0.00000000E+00 0.00000000E+00
+ 0.00000000E+00 0.00000000E+00 0.00000000E+00
+ 0.00000000E+00 0.00000000E+00 0.00000000E+00
+ 0.00000000E+00 0.00000000E+00 0.00000000E+00
+ 0.00000000E+00 0.00000000E+00 0.00000000E+00
+ 0.00000000E+00 0.00000000E+00 0.00000000E+00
+ 0.00000000E+00 0.00000000E+00 0.00000000E+00
+ 0.00000000E+00 0.00000000E+00 0.00000000E+00
diff --git a/tests/auto_test/00.equi/si/mp-149/vasp-relax_incar/OUTCAR b/tests/auto_test/00.equi/si/mp-149/vasp-relax_incar/OUTCAR
new file mode 100644
index 000000000..0bbd2d456
--- /dev/null
+++ b/tests/auto_test/00.equi/si/mp-149/vasp-relax_incar/OUTCAR
@@ -0,0 +1,204 @@
+
+
+ INCAR:
+ POTCAR: PAW_PBE Si 05Jan2001
+
+ POTCAR: PAW_PBE Si 05Jan2001
+ VRHFIN =Si: s2p2
+ LEXCH = PE
+ EATOM = 103.0669 eV, 7.5752 Ry
+
+ TITEL = PAW_PBE Si 05Jan2001
+ LULTRA = F use ultrasoft PP ?
+ IUNSCR = 1 unscreen: 0-lin 1-nonlin 2-no
+ RPACOR = 1.500 partial core radius
+ POMASS = 28.085; ZVAL = 4.000 mass and valenz
+ RCORE = 1.900 outmost cutoff radius
+ RWIGS = 2.480; RWIGS = 1.312 wigner-seitz radius (au A)
+ ENMAX = 245.345; ENMIN = 184.009 eV
+ ICORE = 2 local potential
+ LCOR = T correct aug charges
+ LPAW = T paw PP
+ EAUG = 322.069
+ DEXC = 0.000
+ RMAX = 1.950 core radius for proj-oper
+ RAUG = 1.300 factor for augmentation sphere
+ RDEP = 1.993 radius for radial grids
+ RDEPT = 1.837 core radius for aug-charge
+
+ Atomic configuration
+ 6 entries
+ n l j E occ.
+ 1 0 0.50 -1785.8828 2.0000
+ 2 0 0.50 -139.4969 2.0000
+ 2 1 1.50 -95.5546 6.0000
+ 3 0 0.50 -10.8127 2.0000
+ 3 1 0.50 -4.0811 2.0000
+ 3 2 1.50 -4.0817 0.0000
+ Description
+ l E TYP RCUT TYP RCUT
+ 0 -10.8127223 23 1.900
+ 0 -7.6451159 23 1.900
+ 1 -4.0811372 23 1.900
+ 1 2.4879257 23 1.900
+ 2 -4.0817478 7 1.900
+ local pseudopotential read in
+ partial core-charges read in
+ partial kinetic energy density read in
+ atomic valenz-charges read in
+ non local Contribution for L= 0 read in
+ real space projection operators read in
+ non local Contribution for L= 0 read in
+ real space projection operators read in
+ non local Contribution for L= 1 read in
+ real space projection operators read in
+ non local Contribution for L= 1 read in
+ real space projection operators read in
+ PAW grid and wavefunctions read in
+
+ number of l-projection operators is LMAX = 4
+ number of lm-projection operators is LMMAX = 8
+
+ PAW_PBE Si 05Jan2001 :
+ energy of atom 1 EATOM= -103.0669
+ kinetic energy error for atom= 0.0003 (will be added to EATOM!!)
+
+
+ POSCAR: Si8
+ positions in direct lattice
+ velocities in cartesian coordinates
+ exchange correlation table for LEXCH = 8
+ RHO(1)= 0.500 N(1) = 2000
+ RHO(2)= 100.500 N(2) = 4000
+
+
+
+------------------------ aborting loop because EDIFF is reached ----------------------------------------
+
+
+ CHARGE: cpu time 0.2963: real time 0.2963
+ FORLOC: cpu time 0.0049: real time 0.0049
+ FORNL : cpu time 0.9680: real time 0.9681
+ STRESS: cpu time 3.5438: real time 3.5442
+ FORCOR: cpu time 0.0514: real time 0.0514
+ FORHAR: cpu time 0.0141: real time 0.0141
+ MIXING: cpu time 0.0020: real time 0.0020
+ OFIELD: cpu time 0.0000: real time 0.0000
+
+ FORCE on cell =-STRESS in cart. coord. units (eV):
+ Direction XX YY ZZ XY YZ ZX
+ --------------------------------------------------------------------------------------
+ Alpha Z 13.17272 13.17272 13.17272
+ Ewald -302.59373 -302.59373 -302.59373 0.00000 0.00000 -0.00000
+ Hartree 20.22818 20.22818 20.22818 -0.00000 0.00000 -0.00001
+ E(xc) -100.98430 -100.98430 -100.98430 0.00000 -0.00000 0.00000
+ Local -118.00021 -118.00021 -118.00020 0.00001 -0.00000 0.00002
+ n-local 309.78388 309.78388 309.78388 -0.00000 0.00000 -0.00000
+ augment -46.59684 -46.59684 -46.59684 -0.00000 -0.00000 -0.00000
+ Kinetic 224.99250 224.99250 224.99250 0.00000 0.00000 0.00001
+ Fock 0.00000 0.00000 0.00000 0.00000 0.00000 0.00000
+ -------------------------------------------------------------------------------------
+ Total 0.00220 0.00220 0.00220 0.00001 0.00000 0.00002
+ in kB 0.02151 0.02155 0.02156 0.00008 0.00001 0.00019
+ external pressure = 0.02 kB Pullay stress = 0.00 kB
+
+
+ VOLUME and BASIS-vectors are now :
+ -----------------------------------------------------------------------------
+ energy-cutoff : 650.00
+ volume of cell : 163.55
+ direct lattice vectors reciprocal lattice vectors
+ 5.468728000 0.000000000 0.000000000 0.182857878 0.000000000 0.000000000
+ 0.000000000 5.468728000 0.000000000 0.000000000 0.182857878 0.000000000
+ 0.000000000 0.000000000 5.468728000 0.000000000 0.000000000 0.182857878
+
+ length of vectors
+ 5.468728000 5.468728000 5.468728000 0.182857878 0.182857878 0.182857878
+
+
+ FORCES acting on ions
+ electron-ion (+dipol) ewald-force non-local-force convergence-correction
+ -----------------------------------------------------------------------------------------------
+ -.236E-04 -.111E-04 -.458E-04 0.249E-13 0.107E-13 -.355E-14 -.871E-05 -.478E-05 0.920E-05 0.337E-04 0.158E-04 0.387E-04
+ -.255E-04 -.104E-04 -.182E-04 0.180E-13 0.384E-14 -.710E-14 0.241E-05 0.915E-05 -.846E-06 0.256E-04 0.474E-05 0.222E-04
+ -.805E-04 0.107E-05 0.374E-04 0.107E-13 0.355E-14 0.000E+00 0.146E-04 0.648E-05 -.650E-05 0.733E-04 -.793E-05 -.346E-04
+ -.760E-05 -.183E-04 -.565E-05 0.341E-14 -.357E-14 -.362E-14 -.135E-05 0.548E-05 0.133E-04 0.888E-05 0.154E-04 -.541E-05
+ 0.555E-04 0.406E-05 0.204E-04 0.355E-14 0.355E-14 0.355E-14 -.750E-05 -.781E-05 0.356E-05 -.512E-04 0.202E-05 -.277E-04
+ -.321E-05 0.309E-04 0.493E-04 -.178E-13 -.350E-14 -.718E-16 0.836E-05 -.964E-05 -.109E-04 -.534E-05 -.228E-04 -.407E-04
+ 0.481E-04 0.281E-04 -.852E-05 -.107E-13 -.355E-14 0.711E-14 -.257E-05 -.775E-05 -.798E-05 -.479E-04 -.223E-04 0.139E-04
+ 0.234E-04 -.320E-04 -.298E-04 -.320E-13 -.106E-13 0.356E-14 -.448E-05 0.995E-05 0.233E-05 -.207E-04 0.245E-04 0.287E-04
+ -----------------------------------------------------------------------------------------------
+ -.134E-04 -.763E-05 -.867E-06 0.124E-15 0.361E-15 -.131E-15 0.759E-06 0.107E-05 0.221E-05 0.165E-04 0.934E-05 -.503E-05
+
+
+ POSITION TOTAL-FORCE (eV/Angst)
+ -----------------------------------------------------------------------------------
+ 1.36718 1.36718 1.36718 0.000001 -0.000000 0.000003
+ 2.73436 2.73436 0.00000 0.000002 0.000003 0.000004
+ 1.36718 4.10155 4.10155 0.000007 -0.000001 -0.000003
+ 2.73436 0.00000 2.73436 -0.000001 0.000002 0.000003
+ 4.10155 1.36718 4.10155 -0.000004 -0.000002 -0.000003
+ 0.00000 2.73436 2.73436 -0.000001 -0.000002 -0.000002
+ 4.10155 4.10155 1.36718 -0.000003 -0.000002 -0.000002
+ 0.00000 0.00000 0.00000 -0.000002 0.000002 0.000002
+ -----------------------------------------------------------------------------------
+ total drift: 0.000004 0.000003 -0.000004
+
+
+--------------------------------------------------------------------------------------------------------
+
+
+
+ FREE ENERGIE OF THE ION-ELECTRON SYSTEM (eV)
+ ---------------------------------------------------
+ free energy TOTEN = -43.40017315 eV
+
+ energy without entropy= -43.40017315 energy(sigma->0) = -43.40017315
+
+
+
+--------------------------------------------------------------------------------------------------------
+
+
+ POTLOK: cpu time 0.0496: real time 0.0496
+
+
+--------------------------------------------------------------------------------------------------------
+
+
+
+
+--------------------------------------------------------------------------------------------------------
+
+
+
+ reached required accuracy - stopping structural energy minimisation
+ LOOP+: cpu time 73.7767: real time 73.8176
+ 4ORBIT: cpu time 0.0000: real time 0.0000
+
+ total amount of memory used by VASP MPI-rank0 119933. kBytes
+=======================================================================
+
+ base : 30000. kBytes
+ nonl-proj : 43479. kBytes
+ fftplans : 4240. kBytes
+ grid : 9842. kBytes
+ one-center: 24. kBytes
+ wavefun : 32348. kBytes
+
+
+
+ General timing and accounting informations for this job:
+ ========================================================
+
+ Total CPU time used (sec): 75.718
+ User time (sec): 75.184
+ System time (sec): 0.534
+ Elapsed time (sec): 75.841
+
+ Maximum memory used (kb): 186724.
+ Average memory used (kb): 0.
+
+ Minor page faults: 71438
+ Major page faults: 0
+ Voluntary context switches: 409
diff --git a/tests/auto_test/confs/si/mp-149/POSCAR b/tests/auto_test/confs/si/mp-149/POSCAR
new file mode 100644
index 000000000..4fe904343
--- /dev/null
+++ b/tests/auto_test/confs/si/mp-149/POSCAR
@@ -0,0 +1,25 @@
+Si8
+ 1.00000000000000
+ 5.4687279999999996 0.0000000000000000 0.0000000000000000
+ 0.0000000000000000 5.4687279999999996 0.0000000000000000
+ 0.0000000000000000 0.0000000000000000 5.4687279999999996
+ Si
+ 8
+Direct
+ 0.2500000000000000 0.2500000000000000 0.2500000000000000
+ 0.5000000000000000 0.5000000000000000 0.0000000000000000
+ 0.2500000000000000 0.7500000000000000 0.7500000000000000
+ 0.5000000000000000 0.0000000000000000 0.5000000000000000
+ 0.7500000000000000 0.2500000000000000 0.7500000000000000
+ 0.0000000000000000 0.5000000000000000 0.5000000000000000
+ 0.7500000000000000 0.7500000000000000 0.2500000000000000
+ 0.0000000000000000 0.0000000000000000 0.0000000000000000
+
+ 0.00000000E+00 0.00000000E+00 0.00000000E+00
+ 0.00000000E+00 0.00000000E+00 0.00000000E+00
+ 0.00000000E+00 0.00000000E+00 0.00000000E+00
+ 0.00000000E+00 0.00000000E+00 0.00000000E+00
+ 0.00000000E+00 0.00000000E+00 0.00000000E+00
+ 0.00000000E+00 0.00000000E+00 0.00000000E+00
+ 0.00000000E+00 0.00000000E+00 0.00000000E+00
+ 0.00000000E+00 0.00000000E+00 0.00000000E+00
diff --git a/tests/auto_test/test_elastic.py b/tests/auto_test/test_elastic.py
new file mode 100644
index 000000000..53a8d57a2
--- /dev/null
+++ b/tests/auto_test/test_elastic.py
@@ -0,0 +1,52 @@
+import os,sys,json,glob,shutil
+import dpdata
+import numpy as np
+import unittest
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+__package__ = 'auto_test'
+from .context import make_kspacing_kpoints
+from .context import setUpModule
+
+from pymatgen.io.vasp import Incar
+from dpgen.auto_test.gen_02_elastic import make_vasp
+
+class Test02(unittest.TestCase):
+ def tearDown(self):
+ if os.path.exists('02.elastic'):
+ shutil.rmtree('02.elastic')
+
+ def test_make_vasp (self):
+ jdata = {
+ 'relax_incar' : 'vasp_input/INCAR.rlx',
+ 'potcar_map': {'Si': 'vasp_input/POTCAR' },
+ }
+ make_vasp(jdata, 'confs/si/mp-149')
+
+ target_path = '02.elastic/si/mp-149/vasp-relax_incar'
+ equi_path = '00.equi/si/mp-149/vasp-relax_incar'
+ dfm_dirs = glob.glob(os.path.join(target_path, 'dfm*'))
+
+ # check root INCAR
+ incar0 = Incar.from_file(os.path.join('vasp_input', 'INCAR.rlx'))
+ incar1 = Incar.from_file(os.path.join(target_path, 'INCAR'))
+ self.assertFalse(incar0 == incar1)
+ incar0['ISIF'] = 2
+ self.assertTrue(incar0 == incar1)
+ # check root POTCAR
+ with open(os.path.join('vasp_input', 'POTCAR')) as fp:
+ pot0 = fp.read()
+ with open(os.path.join(target_path, 'POTCAR')) as fp:
+ pot1 = fp.read()
+ self.assertEqual(pot0, pot1)
+ # check root POSCAR
+ self.assertEqual(os.path.realpath(os.path.join(target_path, 'POSCAR')),
+ os.path.realpath(os.path.join(equi_path, 'CONTCAR')))
+ # check subdir
+ for ii in dfm_dirs:
+ self.assertEqual(os.path.realpath(os.path.join(ii, 'INCAR')),
+ os.path.realpath(os.path.join(target_path, 'INCAR')))
+ self.assertEqual(os.path.realpath(os.path.join(ii, 'KPOINTS')),
+ os.path.realpath(os.path.join(target_path, 'KPOINTS')))
+ self.assertEqual(os.path.realpath(os.path.join(ii, 'POTCAR')),
+ os.path.realpath(os.path.join(target_path, 'POTCAR')))
diff --git a/tests/auto_test/test_eos.py b/tests/auto_test/test_eos.py
new file mode 100644
index 000000000..1cddb92c7
--- /dev/null
+++ b/tests/auto_test/test_eos.py
@@ -0,0 +1,81 @@
+import os,sys,json,glob,shutil
+import dpdata
+import numpy as np
+import unittest
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+__package__ = 'auto_test'
+from .context import make_kspacing_kpoints
+from .context import setUpModule
+
+from pymatgen.io.vasp import Incar
+from dpgen.auto_test.gen_01_eos import make_vasp
+
+class Test01(unittest.TestCase):
+ def tearDown(self):
+ if os.path.exists('01.eos'):
+ shutil.rmtree('01.eos')
+
+ def test_make_vasp_rlx_cell_shape (self):
+ jdata = {
+ 'relax_incar' : 'vasp_input/INCAR.rlx',
+ 'potcar_map': {'Si': 'vasp_input/POTCAR' },
+ 'vol_start': 15,
+ 'vol_end': 25,
+ 'vol_step': 1.0,
+ 'eos_relax_cell_shape': True,
+ }
+ make_vasp(jdata, 'confs/si/mp-149')
+
+ target_path = '01.eos/si/mp-149/vasp-relax_incar'
+ equi_path = '00.equi/si/mp-149/vasp-relax_incar'
+ dfm_dirs = glob.glob(os.path.join(target_path, 'vol*'))
+
+ # check root INCAR
+ incar0 = Incar.from_file(os.path.join('vasp_input', 'INCAR.rlx'))
+ incar1 = Incar.from_file(os.path.join(target_path, 'INCAR'))
+ self.assertFalse(incar0 == incar1)
+ incar0['ISIF'] = 4
+ self.assertTrue(incar0 == incar1)
+ # check root POTCAR
+ with open(os.path.join('vasp_input', 'POTCAR')) as fp:
+ pot0 = fp.read()
+ with open(os.path.join(target_path, 'POTCAR')) as fp:
+ pot1 = fp.read()
+ self.assertEqual(pot0, pot1)
+ # check subdir
+ for ii in dfm_dirs:
+ self.assertTrue(os.path.isfile(os.path.join(ii, 'KPOINTS')))
+ self.assertEqual(os.path.realpath(os.path.join(ii, 'POSCAR.orig')),
+ os.path.realpath(os.path.join(equi_path, 'CONTCAR')))
+ self.assertEqual(os.path.realpath(os.path.join(ii, 'INCAR')),
+ os.path.realpath(os.path.join(target_path, 'INCAR')))
+ self.assertEqual(os.path.realpath(os.path.join(ii, 'POTCAR')),
+ os.path.realpath(os.path.join(target_path, 'POTCAR')))
+ sys = dpdata.System(os.path.join(ii, 'POSCAR'))
+ vol = float(ii.split('/')[-1].split('-')[1])
+ natoms = sys.get_natoms()
+ self.assertAlmostEqual(vol, np.linalg.det(sys['cells'][0]) / natoms)
+
+
+ def test_make_vasp_norlx_cell_shape (self):
+ jdata = {
+ 'relax_incar' : 'vasp_input/INCAR.rlx',
+ 'potcar_map': {'Si': 'vasp_input/POTCAR' },
+ 'vol_start': 15,
+ 'vol_end': 25,
+ 'vol_step': 1.0,
+ 'eos_relax_cell_shape': False,
+ }
+ make_vasp(jdata, 'confs/si/mp-149')
+
+ target_path = '01.eos/si/mp-149/vasp-relax_incar'
+ equi_path = '00.equi/si/mp-149/vasp-relax_incar'
+ dfm_dirs = glob.glob(os.path.join(target_path, 'vol*'))
+
+ # check root INCAR
+ incar0 = Incar.from_file(os.path.join('vasp_input', 'INCAR.rlx'))
+ incar1 = Incar.from_file(os.path.join(target_path, 'INCAR'))
+ self.assertFalse(incar0 == incar1)
+ incar0['ISIF'] = 2
+ self.assertTrue(incar0 == incar1)
diff --git a/tests/auto_test/vasp_input/INCAR b/tests/auto_test/vasp_input/INCAR
new file mode 100644
index 000000000..1e2a28323
--- /dev/null
+++ b/tests/auto_test/vasp_input/INCAR
@@ -0,0 +1,30 @@
+#Parameters
+SYSTEM = dpgen
+PREC = A
+ISART = 0
+ICHARG = 2
+#Electronic Relaxation
+ENCUT = 650
+NELM = 100
+NELMIN = 6
+NELMDL = -5
+EDIFF = 1e-06
+LREAL = False
+ALGO = Fast # or normal
+#Ionic relaxation
+IBRION = -1
+ISIF = 2
+ISYM = 0
+NSW = 0
+ISMEAR = 0
+SIGMA = 0.1
+# Write flags
+LWAVE = False
+LCHARG = False
+#parallel related
+#KPAR = 4
+#NPAR = 1
+KSPACING = 0.1
+KGAMMA = False
+
+PSTRESS = 0.0
diff --git a/tests/auto_test/vasp_input/INCAR.md b/tests/auto_test/vasp_input/INCAR.md
new file mode 100644
index 000000000..b7b00d64b
--- /dev/null
+++ b/tests/auto_test/vasp_input/INCAR.md
@@ -0,0 +1,38 @@
+#Parameters
+SYSTEM = dpgen
+PREC = A
+ISART = 0
+ICHARG = 2
+#Electronic Relaxation
+ENCUT = 650
+NELM = 100
+NELMIN = 6
+NELMDL = -5
+EDIFF = 1e-06
+LREAL = False
+ALGO = Fast # or normal
+#Ionic relaxation
+IBRION = 0
+ISIF = 2
+#EDIFFG = -0.01 # useless for MD
+ISYM = 0
+NSW = 10
+ISMEAR = 0
+SIGMA = 0.1
+# MD related
+SMASS = 0
+POTIM = 2
+TEBEG = 100
+TEEND = 100
+NBLOCK = 1
+KBLOCK = 100
+# Write flags
+LWAVE = False
+LCHARG = False
+#parallel related
+#KPAR = 4
+#NPAR = 1
+KSPACING = 0.1
+KGAMMA = False
+
+PSTRESS = 0.0
diff --git a/tests/auto_test/vasp_input/INCAR.rlx b/tests/auto_test/vasp_input/INCAR.rlx
new file mode 100644
index 000000000..b871170ff
--- /dev/null
+++ b/tests/auto_test/vasp_input/INCAR.rlx
@@ -0,0 +1,32 @@
+#Parameters
+SYSTEM = dpgen
+PREC = A
+ISART = 0
+ICHARG = 2
+#Electronic Relaxation
+ENCUT = 650
+NELM = 100
+NELMIN = 6
+NELMDL = -5
+EDIFF = 1e-06
+LREAL = False
+ALGO = Fast # or normal
+#Ionic relaxation
+IBRION = 2
+POTIM = 0.2
+ISIF = 3
+EDIFFG = -0.01
+ISYM = 0
+NSW = 100
+ISMEAR = 0
+SIGMA = 0.1
+# Write flags
+LWAVE = False
+LCHARG = False
+#parallel related
+#KPAR = 4
+#NPAR = 1
+KSPACING = 0.2
+KGAMMA = False
+
+PSTRESS = 0.0
diff --git a/tests/auto_test/vasp_input/POTCAR b/tests/auto_test/vasp_input/POTCAR
new file mode 100644
index 000000000..7bfb856bd
--- /dev/null
+++ b/tests/auto_test/vasp_input/POTCAR
@@ -0,0 +1 @@
+bf8505dd-84b6-4fcc-a32c-3e1768b8a119
diff --git a/tests/data/surf_poscar.json b/tests/data/surf_poscar.json
index b5727ff83..3399b920f 100644
--- a/tests/data/surf_poscar.json
+++ b/tests/data/surf_poscar.json
@@ -12,14 +12,11 @@
1
],
"layer_numb": 3,
- "vacuum_max": 9,
- "vacuum_resol": [
- 0.5,
- 1
- ],
- "mid_point": 4.0,
- "head_ratio": 0.6,
- "vacuum_numb": 20,
+ "vacuum_max": 5,
+ "vacuum_resol": [0.5,2],
+ "mid_point": 2.0,
+ "_head_ratio": 0.6,
+ "_vacuum_numb": 20,
"millers": [
[
1,
diff --git a/tests/data/test_gen_surf.py b/tests/data/test_gen_surf.py
index 5392ccb74..63e9ed7a4 100644
--- a/tests/data/test_gen_surf.py
+++ b/tests/data/test_gen_surf.py
@@ -1,6 +1,6 @@
import os,sys,json,glob,shutil
import unittest
-from pymatgen import Structure
+from pymatgen import Structure,Element
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
__package__ = 'data'
@@ -24,6 +24,7 @@ def setUp(self):
make_vasp_relax(jdata)
make_scale(jdata)
pert_scaled(jdata)
+ self.jdata=jdata
def tearDown(self):
shutil.rmtree(self.root_dir)
@@ -38,7 +39,8 @@ def test(self):
surf=poscar.split('/')[-3]
st1=Structure.from_file(surf+'.POSCAR')
st2=Structure.from_file(poscar)
- self.assertEqual(st1,st2)
+ vacuum_size=float(Element(self.jdata['elements'][0]).atomic_radius*2)
+ self.assertTrue(st1.lattice.c+vacuum_size-st2.lattice.c<0.01)
for surf in self.surfs:
elongs=glob.glob("surf.al.fcc.01x01x01/01.scale_pert/"+surf+"/sys-*/scale-1.000/el*")
diff --git a/tests/data/test_gen_surf_poscar.py b/tests/data/test_gen_surf_poscar.py
index 5c49b0578..1ed44eac3 100644
--- a/tests/data/test_gen_surf_poscar.py
+++ b/tests/data/test_gen_surf_poscar.py
@@ -1,6 +1,6 @@
import os,sys,json,glob,shutil
import unittest
-from pymatgen import Structure
+from pymatgen import Structure,Element
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
__package__ = 'data'
@@ -10,9 +10,8 @@
class TestGenSurfPOSCAR(unittest.TestCase):
def setUp(self):
self.surfs=["surf-100"]
- self.elongs=["elong-0.500", "elong-1.000", "elong-1.500", "elong-2.000", "elong-2.500",\
- "elong-3.000", "elong-3.500", "elong-4.000", "elong-5.000", "elong-6.000",\
- "elong-7.000", "elong-8.000" ]
+ self.elongs=["elong-0.500", "elong-1.000", "elong-1.500",
+ "elong-2.000", "elong-4.000"]
with open (param_file, 'r') as fp :
jdata = json.load (fp)
out_dir = out_dir_name(jdata)
@@ -24,6 +23,7 @@ def setUp(self):
make_vasp_relax(jdata)
make_scale(jdata)
pert_scaled(jdata)
+ self.jdata=jdata
def tearDown(self):
shutil.rmtree(self.root_dir)
@@ -38,7 +38,9 @@ def test(self):
surf=poscar.split('/')[-3]
st1=Structure.from_file(surf+'.POSCAR')
st2=Structure.from_file(poscar)
- self.assertEqual(st1,st2)
+ vacuum_size=float(Element(self.jdata['elements'][0]).atomic_radius*2)
+ self.assertTrue(st1.lattice.c+vacuum_size-st2.lattice.c<0.01)
+
for surf in self.surfs:
elongs=glob.glob("POSCAR.01x01x01/01.scale_pert/"+surf+"/sys-*/scale-1.000/el*")
diff --git a/tests/dispatcher/shell/test_shell_local.py b/tests/dispatcher/shell/test_shell_local.py
index b4d02ba67..4c47136c1 100644
--- a/tests/dispatcher/shell/test_shell_local.py
+++ b/tests/dispatcher/shell/test_shell_local.py
@@ -74,7 +74,7 @@ def test_sub_success(self) :
self.assertTrue(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task0/tag_1_finished')))
self.assertTrue(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task1/tag_0_finished')))
self.assertTrue(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task1/tag_1_finished')))
- self.assertTrue(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'tag_finished')))
+ self.assertTrue(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, '%s_tag_finished' % self.shell.context.job_uuid)))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task0/test1')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task1/test1')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task0/test2')))
@@ -101,7 +101,7 @@ def test_sub_scancel(self) :
self.assertFalse(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task0/tag_1_finished')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task1/tag_0_finished')))
self.assertFalse(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task1/tag_1_finished')))
- self.assertFalse(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'tag_finished')))
+ self.assertFalse(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, '%s_tag_finished' % self.shell.context.job_uuid)))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task0/test1')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task1/test1')))
self.assertFalse(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task0/test2')))
@@ -117,7 +117,7 @@ def test_sub_scancel(self) :
self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task0/tag_1_finished')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task1/tag_0_finished')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task1/tag_1_finished')))
- self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'tag_finished')))
+ self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, '%s_tag_finished' % self.shell.context.job_uuid)))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task0/test1')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task1/test1')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task0/test2')))
diff --git a/tests/dispatcher/shell/test_shell_ssh.py b/tests/dispatcher/shell/test_shell_ssh.py
index 10da439a4..d80bb5688 100644
--- a/tests/dispatcher/shell/test_shell_ssh.py
+++ b/tests/dispatcher/shell/test_shell_ssh.py
@@ -59,7 +59,7 @@ def test_sub_success(self) :
self.assertTrue(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task0/tag_1_finished')))
self.assertTrue(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task1/tag_0_finished')))
self.assertTrue(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task1/tag_1_finished')))
- self.assertTrue(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'tag_finished')))
+ self.assertTrue(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, '%s_tag_finished' % self.shell.context.job_uuid)))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task0/test1')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task1/test1')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task0/test2')))
diff --git a/tests/dispatcher/slurm/test_dispatcher_lazy_local.py b/tests/dispatcher/slurm/test_dispatcher_lazy_local.py
index 7d21a2fe7..89fd9b9a4 100644
--- a/tests/dispatcher/slurm/test_dispatcher_lazy_local.py
+++ b/tests/dispatcher/slurm/test_dispatcher_lazy_local.py
@@ -22,15 +22,14 @@ def setUp(self) :
for ii in ['loc/task0', 'loc/task1', 'loc/task2']:
with open(os.path.join(ii, 'test0'),'w') as fp:
fp.write('this is test0 from ' + ii + '\n')
- work_profile = None
+ work_profile = {}
self.disp = Dispatcher(work_profile, 'lazy-local', 'slurm')
def tearDown(self):
- # shutil.rmtree('loc')
- # shutil.rmtree('rmt')
- # if os.path.exists('dpgen.log'):
- # os.remove('dpgen.log')
- pass
+ shutil.rmtree('loc')
+ shutil.rmtree('rmt')
+ if os.path.exists('dpgen.log'):
+ os.remove('dpgen.log')
def test_sub_success(self):
tasks = ['task0', 'task1', 'task2']
diff --git a/tests/dispatcher/slurm/test_slurm_local.py b/tests/dispatcher/slurm/test_slurm_local.py
index bca72a54a..0aeca1f75 100644
--- a/tests/dispatcher/slurm/test_slurm_local.py
+++ b/tests/dispatcher/slurm/test_slurm_local.py
@@ -50,7 +50,7 @@ def test_sub_success(self) :
self.assertTrue(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/tag_1_finished')))
self.assertTrue(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/tag_0_finished')))
self.assertTrue(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/tag_1_finished')))
- self.assertTrue(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'tag_finished')))
+ self.assertTrue(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, '%s_tag_finished' % self.slurm.context.job_uuid)))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/test1')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/test1')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/test2')))
@@ -76,7 +76,7 @@ def test_sub_scancel(self) :
self.assertFalse(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/tag_1_finished')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/tag_0_finished')))
self.assertFalse(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/tag_1_finished')))
- self.assertFalse(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'tag_finished')))
+ self.assertFalse(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, '%s_tag_finished' % self.slurm.context.job_uuid)))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/test1')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/test1')))
self.assertFalse(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/test2')))
@@ -92,7 +92,7 @@ def test_sub_scancel(self) :
self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/tag_1_finished')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/tag_0_finished')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/tag_1_finished')))
- self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'tag_finished')))
+ self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, '%s_tag_finished' % self.slurm.context.job_uuid)))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/test1')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/test1')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/test2')))
diff --git a/tests/dispatcher/slurm/test_slurm_ssh.py b/tests/dispatcher/slurm/test_slurm_ssh.py
index 204e2191f..774650110 100644
--- a/tests/dispatcher/slurm/test_slurm_ssh.py
+++ b/tests/dispatcher/slurm/test_slurm_ssh.py
@@ -37,7 +37,7 @@ def test_gen_sub_script(self):
self.slurm.context.write_file('run.sub', ret)
with open('run.sub', 'w') as fp:
fp.write(ret)
- ret1 = self.slurm.sub_script(job_dirs, ['touch', 'touch'], [['test1 ', 'test2 '], ['test1 ', 'test2 ']])
+ ret1 = self.slurm.sub_script(job_dirs, ['touch', 'touch'], [['test1 ', 'test1 '], ['test2 ', 'test2 ']])
with open('run.sub.1', 'w') as fp:
fp.write(ret1)
my_file_cmp(self, 'run.sub.1', 'run.sub')
@@ -55,7 +55,7 @@ def test_sub_success(self) :
self.assertTrue(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/tag_1_finished')))
self.assertTrue(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/tag_0_finished')))
self.assertTrue(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/tag_1_finished')))
- self.assertTrue(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'tag_finished')))
+ self.assertTrue(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, '%s_tag_finished' % self.slurm.context.job_uuid)))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/test1')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/test1')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/test2')))
@@ -81,7 +81,7 @@ def test_sub_scancel(self) :
self.assertFalse(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/tag_1_finished')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/tag_0_finished')))
self.assertFalse(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/tag_1_finished')))
- self.assertFalse(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'tag_finished')))
+ self.assertFalse(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, '%s_tag_finished' % self.slurm.context.job_uuid)))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/test1')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/test1')))
self.assertFalse(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/test2')))
@@ -97,7 +97,7 @@ def test_sub_scancel(self) :
self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/tag_1_finished')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/tag_0_finished')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/tag_1_finished')))
- self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'tag_finished')))
+ self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, '%s_tag_finished' % self.slurm.context.job_uuid)))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/test1')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/test1')))
self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/test2')))
diff --git a/tests/dispatcher/test_lazy_local_context.py b/tests/dispatcher/test_lazy_local_context.py
index 6f34f2f14..8de1875d8 100644
--- a/tests/dispatcher/test_lazy_local_context.py
+++ b/tests/dispatcher/test_lazy_local_context.py
@@ -39,6 +39,83 @@ def test_download(self):
self.job.upload(tasks, ['test0', 'dir0'])
self.job.download(tasks, ['test0', 'dir0'])
+ def test_download_check_mark(self):
+ # upload files
+ self.job = LazyLocalContext('loc', None)
+ tasks = ['task0', 'task1']
+ self.job.upload(tasks, ['test0', 'dir0'])
+ record_uuid = []
+ # generate extra donwload files
+ for ii in tasks :
+ for jj in ['test6', 'test7'] :
+ if (ii == 'task1' and jj == 'test7') or \
+ (ii == 'task0' and jj == 'test6'):
+ continue
+ with open(os.path.join('loc',ii,jj), 'w') as fp:
+ tmp = str(uuid.uuid4())
+ fp.write(tmp)
+ record_uuid.append(tmp)
+ self.job.download(tasks, ['test6', 'test7', 'dir1'], check_exists = True, mark_failure = True)
+ # check dlded
+ cc = 0
+ for ii in tasks :
+ for jj in ['test6', 'test7'] :
+ if (ii == 'task1' and jj == 'test7') or \
+ (ii == 'task0' and jj == 'test6') :
+ self.assertFalse(os.path.exists(os.path.join('loc', ii, jj)),
+ msg = 'found ' + os.path.join('loc', ii, jj))
+ self.assertTrue(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj)),
+ msg = 'failed to find ' + os.path.join('loc', ii, 'tag_failure_download_%s' % jj))
+ continue
+ with open(os.path.join('loc',ii,jj), 'r') as fp:
+ tmp = fp.read()
+ self.assertEqual(tmp, record_uuid[cc])
+ cc += 1
+ for ii in tasks :
+ for jj in ['dir1'] :
+ self.assertFalse(os.path.exists(os.path.join('loc', ii, jj)))
+ self.assertTrue(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj)))
+
+
+ def test_download_check_nomark(self):
+ # upload files
+ self.job = LazyLocalContext('loc', None)
+ tasks = ['task0', 'task1']
+ self.job.upload(tasks, ['test0', 'dir0'])
+ record_uuid = []
+ # generate extra donwload files
+ for ii in tasks :
+ for jj in ['test6', 'test7'] :
+ if (ii == 'task1' and jj == 'test7') or \
+ (ii == 'task0' and jj == 'test6'):
+ continue
+ with open(os.path.join('loc',ii,jj), 'w') as fp:
+ tmp = str(uuid.uuid4())
+ fp.write(tmp)
+ record_uuid.append(tmp)
+ self.job.download(tasks, ['test6', 'test7', 'dir1'], check_exists = True, mark_failure = False)
+ # check dlded
+ cc = 0
+ for ii in tasks :
+ for jj in ['test6', 'test7'] :
+ if (ii == 'task1' and jj == 'test7') or \
+ (ii == 'task0' and jj == 'test6') :
+ self.assertFalse(os.path.exists(os.path.join('loc', ii, jj)),
+ msg = 'found ' + os.path.join('loc', ii, jj))
+ self.assertFalse(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj)),
+ msg = 'found ' + os.path.join('loc', ii, 'tag_failure_download_%s' % jj))
+ continue
+ with open(os.path.join('loc',ii,jj), 'r') as fp:
+ tmp = fp.read()
+ self.assertEqual(tmp, record_uuid[cc])
+ cc += 1
+ for ii in tasks :
+ for jj in ['dir1'] :
+ self.assertFalse(os.path.exists(os.path.join('loc', ii, jj)))
+ self.assertFalse(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj)))
+
+
+
def test_block_call(self) :
self.job = LazyLocalContext('loc', None)
tasks = ['task0', 'task1']
diff --git a/tests/dispatcher/test_local_context.py b/tests/dispatcher/test_local_context.py
index 49d4e2efc..694d48fa2 100644
--- a/tests/dispatcher/test_local_context.py
+++ b/tests/dispatcher/test_local_context.py
@@ -217,6 +217,88 @@ def test_download(self):
rmtf = os.path.join('rmt', self.job.job_uuid, ii, jj, kk)
self.assertEqual(os.path.realpath(locf),
os.path.realpath(rmtf))
+
+ def test_download_check_mark(self):
+ # upload files
+ work_profile = LocalSession({'work_path':'rmt'})
+ self.job = LocalContext('loc', work_profile)
+ tasks = ['task0', 'task1']
+ self.job.upload(tasks, ['test0', 'dir0'])
+ # generate extra donwload files
+ record_uuid = []
+ for ii in tasks :
+ for jj in ['test7', 'test8'] :
+ if (ii == 'task1' and jj == 'test7') or \
+ (ii == 'task0' and jj == 'test6') :
+ continue
+ with open(os.path.join('rmt',self.job.job_uuid,ii,jj), 'w') as fp:
+ tmp = str(uuid.uuid4())
+ fp.write(tmp)
+ record_uuid.append(tmp)
+ # donwload
+ files = ['test7', 'test8', 'dir1']
+ self.job.download(tasks, files, check_exists = True, mark_failure = True)
+ # check dlded
+ cc = 0
+ for ii in tasks :
+ for jj in ['test7', 'test8'] :
+ if (ii == 'task1' and jj == 'test7') or \
+ (ii == 'task0' and jj == 'test6') :
+ self.assertFalse(os.path.exists(os.path.join('loc', ii, jj)),
+ msg = 'found ' + os.path.join('loc', ii, jj))
+ self.assertTrue(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj)),
+ msg = 'failed to find ' + os.path.join('loc', ii, 'tag_failure_download_%s' % jj))
+ continue
+ with open(os.path.join('loc',ii,jj), 'r') as fp:
+ tmp = fp.read()
+ self.assertEqual(tmp, record_uuid[cc])
+ cc += 1
+ for ii in tasks :
+ for jj in ['dir1'] :
+ self.assertFalse(os.path.exists(os.path.join('loc', ii, jj)))
+ self.assertTrue(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj)))
+
+
+ def test_download_check_nomark(self):
+ # upload files
+ work_profile = LocalSession({'work_path':'rmt'})
+ self.job = LocalContext('loc', work_profile)
+ tasks = ['task0', 'task1']
+ self.job.upload(tasks, ['test0', 'dir0'])
+ # generate extra donwload files
+ record_uuid = []
+ for ii in tasks :
+ for jj in ['test7', 'test8'] :
+ if (ii == 'task1' and jj == 'test7') or \
+ (ii == 'task0' and jj == 'test6') :
+ continue
+ with open(os.path.join('rmt',self.job.job_uuid,ii,jj), 'w') as fp:
+ tmp = str(uuid.uuid4())
+ fp.write(tmp)
+ record_uuid.append(tmp)
+ # donwload
+ files = ['test7', 'test8', 'dir1']
+ self.job.download(tasks, files, check_exists = True, mark_failure = False)
+ # check dlded
+ cc = 0
+ for ii in tasks :
+ for jj in ['test7', 'test8'] :
+ if (ii == 'task1' and jj == 'test7') or \
+ (ii == 'task0' and jj == 'test6') :
+ self.assertFalse(os.path.exists(os.path.join('loc', ii, jj)),
+ msg = 'found ' + os.path.join('loc', ii, jj))
+ self.assertFalse(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj)),
+ msg = 'found ' + os.path.join('loc', ii, 'tag_failure_download_%s' % jj))
+ continue
+ with open(os.path.join('loc',ii,jj), 'r') as fp:
+ tmp = fp.read()
+ self.assertEqual(tmp, record_uuid[cc])
+ cc += 1
+ for ii in tasks :
+ for jj in ['dir1'] :
+ self.assertFalse(os.path.exists(os.path.join('loc', ii, jj)))
+ self.assertFalse(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj)))
+
def test_block_call(self) :
work_profile = LocalSession({'work_path':'rmt'})
diff --git a/tests/dispatcher/test_ssh_context.py b/tests/dispatcher/test_ssh_context.py
index 909923d2d..550fcaa12 100644
--- a/tests/dispatcher/test_ssh_context.py
+++ b/tests/dispatcher/test_ssh_context.py
@@ -110,6 +110,87 @@ def test_donwload(self):
cc += 1
+ def test_donwload_check_mark(self):
+ tasks = ['task0', 'task1']
+ self.job.upload(tasks, ['test0', 'dir0'])
+ # generate extra donwload files
+ record_uuid = []
+ for ii in tasks :
+ for jj in ['test6', 'test7'] :
+ if (ii == 'task1' and jj == 'test7') or \
+ (ii == 'task0' and jj == 'test6'):
+ continue
+ with open(os.path.join('rmt',self.job.job_uuid,ii,jj), 'w') as fp:
+ tmp = str(uuid.uuid4())
+ fp.write(tmp)
+ record_uuid.append(tmp)
+ # donwload
+ files = ['test6', 'test7', 'dir1']
+ self.job.download(tasks, files, check_exists = True, mark_failure = True)
+ # check dlded
+ cc = 0
+ for ii in tasks :
+ for jj in ['test6', 'test7'] :
+ if (ii == 'task1' and jj == 'test7') or \
+ (ii == 'task0' and jj == 'test6') :
+ self.assertFalse(os.path.exists(os.path.join('loc', ii, jj)),
+ msg = 'found ' + os.path.join('loc', ii, jj))
+ self.assertTrue(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj)),
+ msg = 'failed to find ' + os.path.join('loc', ii, 'tag_failure_download_%s' % jj))
+ continue
+ with open(os.path.join('loc',ii,jj), 'r') as fp:
+ tmp = fp.read()
+ self.assertEqual(tmp, record_uuid[cc])
+ cc += 1
+ for ii in tasks :
+ for jj in ['dir1'] :
+ self.assertFalse(os.path.exists(os.path.join('loc', ii, jj)))
+ self.assertTrue(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj)))
+
+
+ def test_donwload_check_nomark(self):
+ tasks = ['task0', 'task1']
+ self.job.upload(tasks, ['test0', 'dir0'])
+ # generate extra donwload files
+ record_uuid = []
+ for ii in tasks :
+ for jj in ['test6', 'test7'] :
+ if ii == 'task1' and jj == 'test7' :
+ continue
+ if ii == 'task0' and jj == 'test6' :
+ continue
+ with open(os.path.join('rmt',self.job.job_uuid,ii,jj), 'w') as fp:
+ tmp = str(uuid.uuid4())
+ fp.write(tmp)
+ record_uuid.append(tmp)
+ # donwload
+ files = ['test6', 'test7', 'dir1']
+ self.job.download(tasks, files, check_exists = True, mark_failure = False)
+ # check dlded
+ cc = 0
+ for ii in tasks :
+ for jj in ['test6', 'test7'] :
+ if ii == 'task1' and jj == 'test7' :
+ self.assertFalse(os.path.exists(os.path.join('loc', ii, jj)),
+ msg = 'found ' + os.path.join('loc', ii, jj))
+ self.assertFalse(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj)),
+ msg = 'found ' + os.path.join('loc', ii, 'tag_failure_download_%s' % jj))
+ continue
+ if ii == 'task0' and jj == 'test6' :
+ self.assertFalse(os.path.exists(os.path.join('loc', ii, jj)),
+ msg = 'found ' + os.path.join('loc', ii, jj))
+ self.assertFalse(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj)),
+ msg = 'found ' + os.path.join('loc', ii, 'tag_failure_download_%s' % jj))
+ continue
+ with open(os.path.join('loc',ii,jj), 'r') as fp:
+ tmp = fp.read()
+ self.assertEqual(tmp, record_uuid[cc])
+ cc += 1
+ for ii in tasks :
+ for jj in ['dir1'] :
+ self.assertFalse(os.path.exists(os.path.join('loc', ii, jj)))
+ self.assertFalse(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj)))
+
def test_block_call(self) :
tasks = ['task0', 'task1']
files = ['test0', 'test1']
@@ -147,3 +228,4 @@ def test_file(self) :
tmp1 = self.job.read_file('aaa')
self.assertEqual(tmp, tmp1)
+
diff --git a/tests/generator/C.SG15.PBE.UPF b/tests/generator/C.SG15.PBE.UPF
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/generator/H.SG15.PBE.UPF b/tests/generator/H.SG15.PBE.UPF
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/generator/N.SG15.PBE.UPF b/tests/generator/N.SG15.PBE.UPF
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/generator/check_bad_box/bad.height.POSCAR b/tests/generator/check_bad_box/bad.height.POSCAR
new file mode 100644
index 000000000..7bffef9c3
--- /dev/null
+++ b/tests/generator/check_bad_box/bad.height.POSCAR
@@ -0,0 +1,16 @@
+Si8
+1.0
+1.5044712174663648e+01 0.0000000000000000e+00 0.0000000000000000e+00
+2.1550292128393154e+00 4.6052924123850527e+00 0.0000000000000000e+00
+-4.8723010805707583e+00 -2.0033682930713113e+00 1.5924563614826326e+00
+Si
+8
+Cartesian
+ 4.9743339600 0.3464058271 1.0711743625
+ -0.3457810400 2.5396598271 1.1364543625
+ 8.0859689600 0.8673568271 0.7907843625
+ 8.6812089600 -1.3051601729 1.3749843625
+ 3.4236889600 2.1462798271 0.9487443625
+ 10.9770889600 0.2924648271 0.3900343625
+ 2.4104489600 3.8329998271 -0.0757256375
+ 5.9863139600 2.6933598271 -0.0617356375
diff --git a/tests/generator/check_bad_box/bad.length.lammpstrj b/tests/generator/check_bad_box/bad.length.lammpstrj
new file mode 100644
index 000000000..72219f300
--- /dev/null
+++ b/tests/generator/check_bad_box/bad.length.lammpstrj
@@ -0,0 +1,17 @@
+ITEM: TIMESTEP
+5280
+ITEM: NUMBER OF ATOMS
+8
+ITEM: BOX BOUNDS xy xz yz pp pp pp
+-9.8598738203143235e+00 2.5613607315592056e+01 2.2564569351176282e+00
+2.0941875002685144e+00 4.1475715876280130e+00 8.0019818936636522e+00
+1.1929631605542741e+00 4.3055923370457059e+00 7.7239367239652201e-01
+ITEM: ATOMS id type x y z
+1 1 -4.99391 3.35438 2.10847
+2 1 -0.969424 2.43249 2.0988
+3 1 7.28407 2.8735 2.62347
+4 1 12.7535 2.79479 2.52587
+5 1 11.2843 3.48997 4.38948
+6 1 18.5323 3.79371 3.76776
+7 1 22.4848 3.28742 3.92357
+8 1 4.68871 3.71 3.677
diff --git a/tests/generator/check_bad_box/good.lammpstrj b/tests/generator/check_bad_box/good.lammpstrj
new file mode 100644
index 000000000..de83eb30d
--- /dev/null
+++ b/tests/generator/check_bad_box/good.lammpstrj
@@ -0,0 +1,17 @@
+ITEM: TIMESTEP
+1000
+ITEM: NUMBER OF ATOMS
+8
+ITEM: BOX BOUNDS xy xz yz pp pp pp
+1.8139178622650964e-01 5.3315679568790921e+00 6.0140898224392482e-02
+1.5585645904442166e-01 5.2801927596078837e+00 -4.2475821618813588e-02
+1.7257600456519429e-01 5.3259794930348008e+00 -3.3316196847694178e-02
+ITEM: ATOMS id type x y z
+1 1 1.56205 1.38625 1.45027
+2 1 2.81674 2.78587 0.383858
+3 1 1.42276 3.82071 4.00808
+4 1 2.70217 5.14268 2.79767
+5 1 3.8694 1.49549 4.01017
+6 1 5.26061 2.60465 2.6905
+7 1 4.17094 4.22066 1.50687
+8 1 0.254383 0.28016 5.15933
diff --git a/tests/generator/context.py b/tests/generator/context.py
index 0668e7fe2..d79bbfeb4 100644
--- a/tests/generator/context.py
+++ b/tests/generator/context.py
@@ -18,6 +18,7 @@
machine_file = 'machine-local.json'
machine_file_v1 = 'machine-local-v1.json'
param_diy_file = 'param-mg-vasp-diy.json'
+param_pwmat_file = 'param-pyridine-pwmat.json'
def my_file_cmp(test, f0, f1):
with open(f0) as fp0 :
diff --git a/tests/generator/data/mg.fcc.02x02x02 b/tests/generator/data/mg.fcc.02x02x02
new file mode 120000
index 000000000..32b1ad2c2
--- /dev/null
+++ b/tests/generator/data/mg.fcc.02x02x02
@@ -0,0 +1 @@
+al.fcc.02x02x02
\ No newline at end of file
diff --git a/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000000/OUT.MLMD b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000000/OUT.MLMD
new file mode 100644
index 000000000..7477360a4
--- /dev/null
+++ b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000000/OUT.MLMD
@@ -0,0 +1,20 @@
+ 6 atoms,Iteration (fs) = -0.1000000000E+01, Etot,Ep,Ek (eV) = -0.8463607047E+03 -0.8463607047E+03 0.0000000000E+00, SCF = 36
+ Lattice vector (Angstrom)
+ 0.1080440044E+02 0.0000000000E+00 0.0000000000E+00
+ 0.0000000000E+00 0.1049750233E+02 0.0000000000E+00
+ 0.0000000000E+00 0.0000000000E+00 0.1095603371E+02
+ Position (normalized), move_x, move_y, move_z
+ 6 0.095086720000000 0.162577930000000 0.146264240000000 1 1 1
+ 6 0.366858130000000 0.378822980000000 0.293448980000000 1 1 1
+ 1 0.404086650000000 0.560402750000000 0.498746690000000 1 1 1
+ 1 0.141461450000000 0.215485070000000 0.346088290000000 1 1 1
+ 7 0.626253130000000 0.697621110000000 0.674972590000000 1 1 1
+ 7 0.745915950000000 0.874152660000000 0.525583090000000 1 1 1
+ Force (eV/Angstrom)
+ 6 0.000000000000000 0.000000000000000 0.000000000000000
+ 6 0.000000000000000 0.000000000000000 0.000000000000000
+ 1 0.000000000000000 0.000000000000000 0.000000000000000
+ 1 0.000000000000000 0.000000000000000 0.000000000000000
+ 7 0.000000000000000 0.000000000000000 0.000000000000000
+ 7 0.000000000000000 0.000000000000000 0.000000000000000
+ ----------------------------------------------END
diff --git a/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000000/etot.input b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000000/etot.input
new file mode 100644
index 000000000..0e919853e
--- /dev/null
+++ b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000000/etot.input
@@ -0,0 +1,17 @@
+4 1
+JOB = SCF
+IN.ATOM = atom.config
+IN.PSP1 = C.SG15.PBE.UPF
+IN.PSP2 = H.SG15.PBE.UPF
+IN.PSP3 = N.SG15.PBE.UPF
+ECUT = 50
+E_ERROR = 1.0E-4
+RHO_ERROR = 1.0E-4
+SCF_ITER0_1 = 6 4 3 0.0000 0.025 2
+SCF_ITER0_2 = 94 4 3 1.0000 0.025 2
+XCFUNCTIONAL = PBE
+CONVERGENCE = EASY
+MP_N123 = 2 2 2 0 0 0 2
+OUT.RHO = F
+OUT.WG = F
+OUT.MLMD = T
diff --git a/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000000/info b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000000/info
new file mode 100644
index 000000000..fe7207fc2
--- /dev/null
+++ b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000000/info
@@ -0,0 +1 @@
+/home/linfengz/TIG/deepgen/pyridine/00/iter.000022/02.fp/task.006.000076/input
diff --git a/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000000/poscar2config.x b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000000/poscar2config.x
new file mode 100755
index 000000000..868a02263
Binary files /dev/null and b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000000/poscar2config.x differ
diff --git a/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000001/OUT.MLMD b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000001/OUT.MLMD
new file mode 100644
index 000000000..7477360a4
--- /dev/null
+++ b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000001/OUT.MLMD
@@ -0,0 +1,20 @@
+ 6 atoms,Iteration (fs) = -0.1000000000E+01, Etot,Ep,Ek (eV) = -0.8463607047E+03 -0.8463607047E+03 0.0000000000E+00, SCF = 36
+ Lattice vector (Angstrom)
+ 0.1080440044E+02 0.0000000000E+00 0.0000000000E+00
+ 0.0000000000E+00 0.1049750233E+02 0.0000000000E+00
+ 0.0000000000E+00 0.0000000000E+00 0.1095603371E+02
+ Position (normalized), move_x, move_y, move_z
+ 6 0.095086720000000 0.162577930000000 0.146264240000000 1 1 1
+ 6 0.366858130000000 0.378822980000000 0.293448980000000 1 1 1
+ 1 0.404086650000000 0.560402750000000 0.498746690000000 1 1 1
+ 1 0.141461450000000 0.215485070000000 0.346088290000000 1 1 1
+ 7 0.626253130000000 0.697621110000000 0.674972590000000 1 1 1
+ 7 0.745915950000000 0.874152660000000 0.525583090000000 1 1 1
+ Force (eV/Angstrom)
+ 6 0.000000000000000 0.000000000000000 0.000000000000000
+ 6 0.000000000000000 0.000000000000000 0.000000000000000
+ 1 0.000000000000000 0.000000000000000 0.000000000000000
+ 1 0.000000000000000 0.000000000000000 0.000000000000000
+ 7 0.000000000000000 0.000000000000000 0.000000000000000
+ 7 0.000000000000000 0.000000000000000 0.000000000000000
+ ----------------------------------------------END
diff --git a/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000001/etot.input b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000001/etot.input
new file mode 100644
index 000000000..0e919853e
--- /dev/null
+++ b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000001/etot.input
@@ -0,0 +1,17 @@
+4 1
+JOB = SCF
+IN.ATOM = atom.config
+IN.PSP1 = C.SG15.PBE.UPF
+IN.PSP2 = H.SG15.PBE.UPF
+IN.PSP3 = N.SG15.PBE.UPF
+ECUT = 50
+E_ERROR = 1.0E-4
+RHO_ERROR = 1.0E-4
+SCF_ITER0_1 = 6 4 3 0.0000 0.025 2
+SCF_ITER0_2 = 94 4 3 1.0000 0.025 2
+XCFUNCTIONAL = PBE
+CONVERGENCE = EASY
+MP_N123 = 2 2 2 0 0 0 2
+OUT.RHO = F
+OUT.WG = F
+OUT.MLMD = T
diff --git a/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000001/info b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000001/info
new file mode 100644
index 000000000..02dfaee17
--- /dev/null
+++ b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000001/info
@@ -0,0 +1 @@
+/home/linfengz/TIG/deepgen/pyridine/00/iter.000022/02.fp/task.006.000070/input
diff --git a/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000001/poscar2config.x b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000001/poscar2config.x
new file mode 100755
index 000000000..868a02263
Binary files /dev/null and b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000001/poscar2config.x differ
diff --git a/tests/generator/out_data_post_fp_pwmat/orig/box.raw b/tests/generator/out_data_post_fp_pwmat/orig/box.raw
new file mode 100644
index 000000000..f6099445f
--- /dev/null
+++ b/tests/generator/out_data_post_fp_pwmat/orig/box.raw
@@ -0,0 +1,2 @@
+1.080440044000000022e+01 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.049750232999999966e+01 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.095603370999999981e+01
+1.080440044000000022e+01 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.049750232999999966e+01 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.095603370999999981e+01
diff --git a/tests/generator/out_data_post_fp_pwmat/orig/coord.raw b/tests/generator/out_data_post_fp_pwmat/orig/coord.raw
new file mode 100644
index 000000000..9bb2eff6c
--- /dev/null
+++ b/tests/generator/out_data_post_fp_pwmat/orig/coord.raw
@@ -0,0 +1,2 @@
+1.528406152623037917e+00 2.262055024405213111e+00 3.791754971876255986e+00 4.365913979058126060e+00 5.882829173863406425e+00 5.464285548390919622e+00 1.027354999406156821e+00 1.706662198981576894e+00 1.602475944007530240e+00 3.963682141189577290e+00 3.976695115207542930e+00 3.215036917045115850e+00 6.766289593323377538e+00 7.323279227682186310e+00 7.395022449366008388e+00 8.059174618383018185e+00 9.176419585125698219e+00 5.758306051445964080e+00
+1.528406152623037917e+00 2.262055024405213111e+00 3.791754971876255986e+00 4.365913979058126060e+00 5.882829173863406425e+00 5.464285548390919622e+00 1.027354999406156821e+00 1.706662198981576894e+00 1.602475944007530240e+00 3.963682141189577290e+00 3.976695115207542930e+00 3.215036917045115850e+00 6.766289593323377538e+00 7.323279227682186310e+00 7.395022449366008388e+00 8.059174618383018185e+00 9.176419585125698219e+00 5.758306051445964080e+00
diff --git a/tests/generator/out_data_post_fp_pwmat/orig/energy.raw b/tests/generator/out_data_post_fp_pwmat/orig/energy.raw
new file mode 100644
index 000000000..b43284887
--- /dev/null
+++ b/tests/generator/out_data_post_fp_pwmat/orig/energy.raw
@@ -0,0 +1,2 @@
+-8.463607047000000421e+02
+-8.463607047000000421e+02
diff --git a/tests/generator/out_data_post_fp_pwmat/orig/force.raw b/tests/generator/out_data_post_fp_pwmat/orig/force.raw
new file mode 100644
index 000000000..7e93007b7
--- /dev/null
+++ b/tests/generator/out_data_post_fp_pwmat/orig/force.raw
@@ -0,0 +1,2 @@
+0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00
+0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00
diff --git a/tests/generator/out_data_post_fp_pwmat/orig/type.raw b/tests/generator/out_data_post_fp_pwmat/orig/type.raw
new file mode 100644
index 000000000..0970af7bc
--- /dev/null
+++ b/tests/generator/out_data_post_fp_pwmat/orig/type.raw
@@ -0,0 +1,6 @@
+0
+0
+1
+1
+2
+2
diff --git a/tests/generator/out_data_post_fp_pwmat/orig/type_map.raw b/tests/generator/out_data_post_fp_pwmat/orig/type_map.raw
new file mode 100644
index 000000000..57b04d593
--- /dev/null
+++ b/tests/generator/out_data_post_fp_pwmat/orig/type_map.raw
@@ -0,0 +1,3 @@
+H
+C
+N
diff --git a/tests/generator/param-mg-vasp.json b/tests/generator/param-mg-vasp.json
index 908be9afb..f2ecdcf8f 100644
--- a/tests/generator/param-mg-vasp.json
+++ b/tests/generator/param-mg-vasp.json
@@ -7,8 +7,8 @@
],
"init_batch_size": [16],
"sys_configs": [
- ["data/mg.fcc.02x02x02/01.scale_pert/sys-0032/scale*/000010/POSCAR"],
- ["data/mg.fcc.02x02x02/01.scale_pert/sys-0032/scale*/00000[8-9]/POSCAR"]
+ ["data/mg.fcc.02x02x02/01.scale_pert/sys-0032/scale*/000000/POSCAR"],
+ ["data/mg.fcc.02x02x02/01.scale_pert/sys-0032/scale*/000001/POSCAR"]
],
"_comment": "0 1 2 3",
"_comment": "4 5 6 7",
diff --git a/tests/generator/param-pyridine-pwmat.json b/tests/generator/param-pyridine-pwmat.json
new file mode 100644
index 000000000..cc2e59b39
--- /dev/null
+++ b/tests/generator/param-pyridine-pwmat.json
@@ -0,0 +1,123 @@
+{
+ "type_map": ["H", "C", "N"],
+ "mass_map": [2, 16, 14],
+
+ "init_data_prefix": "/home/linfengz/SCR/wanghan/deepgen.pyridine/init",
+ "init_data_sys": ["Pyridine-I",
+ "Pyridine-II"
+ ],
+ "init_batch_size": [1, 1],
+ "sys_configs": [
+ ["/home/linfengz/SCR/wanghan/data/pyridine/pyI.POSCAR.01x01x01/01.scale_pert/sys-0080-0080-0016/scale-1.000/00009?/POSCAR"],
+ ["/home/linfengz/SCR/wanghan/data/pyridine/pyI.POSCAR.01x01x01/01.scale_pert/sys-0080-0080-0016/scale-1.000/0000[7-8]?/POSCAR"],
+ ["/home/linfengz/SCR/wanghan/data/pyridine/pyI.POSCAR.01x01x01/01.scale_pert/sys-0080-0080-0016/scale-1.000/0000[5-6]?/POSCAR"],
+ ["/home/linfengz/SCR/wanghan/data/pyridine/pyI.POSCAR.01x01x01/01.scale_pert/sys-0080-0080-0016/scale-1.000/0000[0-4]?/POSCAR"],
+ ["/home/linfengz/SCR/wanghan/data/pyridine/pyII.POSCAR.01x01x01/01.scale_pert/sys-0080-0080-0016/scale-1.000/00009?/POSCAR"],
+ ["/home/linfengz/SCR/wanghan/data/pyridine/pyII.POSCAR.01x01x01/01.scale_pert/sys-0080-0080-0016/scale-1.000/0000[7-8]?/POSCAR"],
+ ["/home/linfengz/SCR/wanghan/data/pyridine/pyII.POSCAR.01x01x01/01.scale_pert/sys-0080-0080-0016/scale-1.000/0000[5-6]?/POSCAR"],
+ ["/home/linfengz/SCR/wanghan/data/pyridine/pyII.POSCAR.01x01x01/01.scale_pert/sys-0080-0080-0016/scale-1.000/0000[0-4]?/POSCAR"]
+ ],
+ "_comment": "0 1 2 3",
+ "_comment": "4 5 6 7",
+ "sys_batch_size": [1, 1, 1, 1,
+ 1, 1, 1, 1
+ ],
+
+ "_comment": " 00.train ",
+ "numb_models": 4,
+ "train_param": "input.json",
+ "default_training_param" : {
+ "_comment": " model parameters",
+ "use_smooth": true,
+ "sel_a": [81, 81, 20],
+ "rcut_smth": 0.50,
+ "rcut": 6.50,
+ "filter_neuron": [25, 50, 100],
+ "filter_resnet_dt": false,
+ "n_axis_neuron": 12,
+ "n_neuron": [240, 240, 240],
+ "resnet_dt": true,
+ "coord_norm": true,
+ "type_fitting_net": false,
+
+ "_comment": " traing controls",
+ "systems": [],
+ "set_prefix": "set",
+ "stop_batch": 400000,
+ "batch_size": 1,
+ "start_lr": 0.002,
+ "decay_steps": 2000,
+ "decay_rate": 0.95,
+ "seed": 0,
+
+ "start_pref_e": 0.02,
+ "limit_pref_e": 2,
+ "start_pref_f": 1000,
+ "limit_pref_f": 1,
+ "start_pref_v": 0.0,
+ "limit_pref_v": 0.0,
+
+ "_comment": " display and restart",
+ "_comment": " frequencies counted in batch",
+ "disp_file": "lcurve.out",
+ "disp_freq": 2000,
+ "numb_test": 10,
+ "save_freq": 20000,
+ "save_ckpt": "model.ckpt",
+ "load_ckpt": "model.ckpt",
+ "disp_training": true,
+ "time_training": true,
+ "profiling": false,
+ "profiling_file": "timeline.json",
+
+ "_comment": "that's all"
+ },
+
+ "_comment": " 01.model_devi ",
+ "_comment": "model_devi_skip: the first x of the recorded frames",
+ "model_devi_dt": 0.001,
+ "model_devi_skip": 0,
+ "model_devi_f_trust_lo": 0.050,
+ "model_devi_f_trust_hi": 0.150,
+ "model_devi_e_trust_lo": 1e10,
+ "model_devi_e_trust_hi": 1e10,
+ "model_devi_clean_traj": false,
+ "model_devi_jobs": [
+ {"sys_idx": [0,4], "temps": [ 50], "press": [1e0,1e1,1e2,1e3,1e4,2e4,4e4], "trj_freq": 10, "nsteps": 1000, "ensemble": "npt", "_idx": "00"},
+ {"sys_idx": [1,5], "temps": [ 50], "press": [1e0,1e1,1e2,1e3,1e4,2e4,4e4], "trj_freq": 10, "nsteps": 1000, "ensemble": "npt", "_idx": "01"},
+ {"sys_idx": [0,4], "temps": [ 50], "press": [1e0,1e1,1e2,1e3,1e4,2e4,4e4], "trj_freq": 10, "nsteps": 1000, "ensemble": "npt", "_idx": "02"},
+ {"sys_idx": [1,5], "temps": [ 50], "press": [1e0,1e1,1e2,1e3,1e4,2e4,4e4], "trj_freq": 10, "nsteps": 1000, "ensemble": "npt", "_idx": "03"},
+ {"sys_idx": [0,4], "temps": [ 100], "press": [1e0,1e1,1e2,1e3,1e4,2e4,4e4], "trj_freq": 10, "nsteps": 1000, "ensemble": "npt", "_idx": "04"},
+ {"sys_idx": [1,5], "temps": [ 100], "press": [1e0,1e1,1e2,1e3,1e4,2e4,4e4], "trj_freq": 10, "nsteps": 1000, "ensemble": "npt", "_idx": "05"},
+ {"sys_idx": [0,4], "temps": [ 100], "press": [1e0,1e1,1e2,1e3,1e4,2e4,4e4], "trj_freq": 10, "nsteps": 1000, "ensemble": "npt", "_idx": "06"},
+ {"sys_idx": [1,5], "temps": [ 100], "press": [1e0,1e1,1e2,1e3,1e4,2e4,4e4], "trj_freq": 10, "nsteps": 1000, "ensemble": "npt", "_idx": "07"}
+ ],
+
+ "_comment": " 02.fp ",
+ "fp_style": "pwmat",
+ "shuffle_poscar": false,
+ "fp_task_max": 100,
+ "fp_task_min": 10,
+ "fp_pp_path": ".",
+ "fp_pp_files": ["C.SG15.PBE.UPF", "H.SG15.PBE.UPF", "N.SG15.PBE.UPF"],
+ "user_fp_params": {
+ "node1": 4,
+ "node2": 1,
+ "job": "SCF",
+ "in.atom": "atom.config",
+ "in.psp1": "C.SG15.PBE.UPF",
+ "in.psp2": "H.SG15.PBE.UPF",
+ "in.psp3": "N.SG15.PBE.UPF",
+ "ecut": 50,
+ "flag_symm": 2,
+ "e_error": 1.0E-4,
+ "rho_error": 1.0E-4,
+ "scf_iter0_1": "6 4 3 0.0000 0.025 2",
+ "scf_iter0_2": "94 4 3 1.0000 0.025 2",
+ "xcfunctional": "PBE",
+ "convergence": "EASY",
+ "kspacing": 0.1
+
+ },
+ "_comment": " that's all "
+}
diff --git a/tests/generator/test_check_bad_box.py b/tests/generator/test_check_bad_box.py
new file mode 100644
index 000000000..ba210128e
--- /dev/null
+++ b/tests/generator/test_check_bad_box.py
@@ -0,0 +1,22 @@
+import os,sys,json,glob,shutil
+import dpdata
+import numpy as np
+import unittest
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+__package__ = 'generator'
+from .context import check_bad_box
+
+class TestCheckBadBox(unittest.TestCase):
+ def test_length_ratio(self):
+ dirname = os.path.dirname(__file__)
+ conf_bad = os.path.join(dirname, 'check_bad_box', 'bad.length.lammpstrj')
+ conf_good = os.path.join(dirname, 'check_bad_box', 'good.lammpstrj')
+ self.assertTrue(check_bad_box(conf_bad, 'length_ratio:5'))
+ self.assertFalse(check_bad_box(conf_good, 'length_ratio:5'))
+
+ def test_height_ratio(self):
+ dirname = os.path.dirname(__file__)
+ conf_bad = os.path.join(dirname, 'check_bad_box', 'bad.height.POSCAR')
+ self.assertTrue(check_bad_box(conf_bad, 'height_ratio:5', fmt = 'vasp/POSCAR'))
+ self.assertFalse(check_bad_box(conf_bad, 'length_ratio:5', fmt = 'vasp/POSCAR'))
diff --git a/tests/generator/test_make_fp.py b/tests/generator/test_make_fp.py
index e77e94d36..8e679f1e9 100644
--- a/tests/generator/test_make_fp.py
+++ b/tests/generator/test_make_fp.py
@@ -2,6 +2,7 @@
import dpdata
import numpy as np
import unittest
+import importlib
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
__package__ = 'generator'
@@ -26,6 +27,7 @@
from .comp_sys import test_coord
from .comp_sys import test_cell
from pymatgen.io.vasp import Kpoints,Incar
+from .context import param_pwmat_file
import scipy.constants as pc
vasp_incar_ref = "PREC=A\n\
@@ -185,7 +187,23 @@
&FORCES ON\n\
&END FORCES\n\
&END PRINT\n\
-&END FORCE_EVAL\n"
+&END FORCE_EVAL\n";
+
+pwmat_input_ref = "4 1\n\
+in.atom=atom.config\n\
+ecut=50\n\
+e_error=0.0001\n\
+rho_error=0.0001\n\
+scf_iter0_1=6 4 3 0.0000 0.025 2\n\
+scf_iter0_2=94 4 3 1.0000 0.025 2\n\
+mp_n123=147 57 39 0 0 0 2\n\
+out.wg=F\n\
+out.rho=F\n\
+out.mlmd=T\n\
+job=scf\n\
+IN.PSP1 = C.SG15.PBE.UPF\n\
+IN.PSP2 = H.SG15.PBE.UPF\n\
+IN.PSP3 = N.SG15.PBE.UPF\n";
@@ -469,6 +487,18 @@ def _check_cp2k_input_head(testCase, idx) :
+def _check_pwmat_input(testCase, idx):
+ fp_path = os.path.join('iter.%06d' % idx, '02.fp')
+ tasks = glob.glob(os.path.join(fp_path, 'task.*'))
+ cwd = os.getcwd()
+ for ii in tasks :
+ os.chdir(ii)
+ os.system("sed -i '8c mp_n123=147 57 39 0 0 0 2' etot.input")
+ with open('etot.input') as fp:
+ lines = fp.read()
+ testCase.assertEqual(lines.strip(), pwmat_input_ref.strip())
+ os.chdir(cwd)
+
class TestMakeFPPwscf(unittest.TestCase):
def test_make_fp_pwscf(self):
if os.path.isdir('iter.000000') :
@@ -710,11 +740,12 @@ def test_make_fp_vasp_ele_temp(self):
class TestMakeFPGaussian(unittest.TestCase):
- def test_make_fp_gaussian(self):
+ def make_fp_gaussian(self, multiplicity="auto"):
if os.path.isdir('iter.000000') :
shutil.rmtree('iter.000000')
with open (param_gaussian_file, 'r') as fp :
jdata = json.load (fp)
+ jdata['user_fp_params']['multiplicity'] = multiplicity
with open (machine_file, 'r') as fp:
mdata = json.load (fp)
md_descript = []
@@ -736,6 +767,13 @@ def test_make_fp_gaussian(self):
_check_potcar(self, 0, jdata['fp_pp_path'], jdata['fp_pp_files'])
shutil.rmtree('iter.000000')
+ @unittest.skipIf(importlib.util.find_spec("openbabel") is None, "requires openbabel")
+ def test_make_fp_gaussian(self):
+ self.make_fp_gaussian()
+
+ def test_make_fp_gaussian_multiplicity_one(self):
+ self.make_fp_gaussian(multiplicity=1)
+
def test_detect_multiplicity(self):
# oxygen O2 3
self._check_multiplicity(['O', 'O'], 3)
@@ -779,6 +817,33 @@ def test_make_fp_cp2k(self):
shutil.rmtree('iter.000000')
+class TestMakeFPPWmat(unittest.TestCase):
+ def test_make_fp_pwmat(self):
+ if os.path.isdir('iter.000000') :
+ shutil.rmtree('iter.000000')
+ with open (param_pwmat_file, 'r') as fp :
+ jdata = json.load (fp)
+ with open (machine_file, 'r') as fp:
+ mdata = json.load (fp)
+ md_descript = []
+ nsys = 2
+ nmd = 3
+ n_frame = 10
+ for ii in range(nsys) :
+ tmp = []
+ for jj in range(nmd) :
+ tmp.append(np.arange(0, 0.29, 0.29/10))
+ md_descript.append(tmp)
+ atom_types = [0, 1, 2, 2, 0, 1]
+ type_map = jdata['type_map']
+ _make_fake_md(0, md_descript, atom_types, type_map)
+ make_fp(0, jdata, {})
+ _check_sel(self, 0, jdata['fp_task_max'], jdata['model_devi_f_trust_lo'], jdata['model_devi_f_trust_hi'])
+ _check_poscars(self, 0, jdata['fp_task_max'], jdata['type_map'])
+ _check_pwmat_input(self, 0)
+ _check_potcar(self, 0, jdata['fp_pp_path'], jdata['fp_pp_files'])
+ shutil.rmtree('iter.000000')
+
if __name__ == '__main__':
unittest.main()
diff --git a/tests/generator/test_make_md.py b/tests/generator/test_make_md.py
index 3ee930344..4f67b1fbe 100644
--- a/tests/generator/test_make_md.py
+++ b/tests/generator/test_make_md.py
@@ -146,8 +146,38 @@ def test_make_model_devi (self) :
_check_confs(self, 0, jdata)
_check_traj_dir(self, 0)
_check_pt(self, 0, jdata)
- shutil.rmtree('iter.000000')
-
+ #shutil.rmtree('iter.000000')
+
+ def test_make_model_devi_nopbc_npt (self) :
+ if os.path.isdir('iter.000000') :
+ shutil.rmtree('iter.000000')
+ with open (param_file, 'r') as fp :
+ jdata = json.load (fp)
+ jdata['model_devi_nopbc'] = True
+ with open (machine_file, 'r') as fp:
+ mdata = json.load (fp)
+ _make_fake_models(0, jdata['numb_models'])
+ cwd = os.getcwd()
+ with self.assertRaises(RuntimeError) :
+ make_model_devi(0, jdata, mdata)
+ os.chdir(cwd)
+
+ def test_make_model_devi_nopbc_nvt (self) :
+ if os.path.isdir('iter.000000') :
+ shutil.rmtree('iter.000000')
+ with open (param_file, 'r') as fp :
+ jdata = json.load (fp)
+ jdata['model_devi_nopbc'] = True
+ jdata['model_devi_jobs'][0]['ensemble'] = 'nvt'
+ with open (machine_file, 'r') as fp:
+ mdata = json.load (fp)
+ _make_fake_models(0, jdata['numb_models'])
+ make_model_devi(0, jdata, mdata)
+ _check_pb(self, 0)
+ # _check_confs(self, 0, jdata)
+ _check_traj_dir(self, 0)
+ _check_pt(self, 0, jdata)
+ # shutil.rmtree('iter.000000')
class TestMakeModelDeviRevMat(unittest.TestCase):
diff --git a/tests/generator/test_make_train.py b/tests/generator/test_make_train.py
index c64487f55..3fa820d82 100644
--- a/tests/generator/test_make_train.py
+++ b/tests/generator/test_make_train.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
-import os,sys,json,glob,shutil
+import os,sys,json,glob,shutil,dpdata
import numpy as np
import unittest
@@ -101,7 +101,7 @@ def _check_model_input_dict(testCase, input_dict, init_data_sys, init_batch_size
testCase.assertEqual(input_dict[ii], default_training_param[ii])
-def _check_model_inputs_v1(testCase, iter_idx, jdata) :
+def _check_model_inputs_v1(testCase, iter_idx, jdata, reuse = False) :
train_param = jdata.get('train_param', 'input.json')
numb_models = jdata['numb_models']
use_ele_temp = jdata.get('use_ele_temp', 0)
@@ -133,6 +133,18 @@ def _check_model_inputs_v1(testCase, iter_idx, jdata) :
_check_model_input_dict(testCase, jdata0['loss'], init_data_sys, init_batch_size, default_training_param['loss'])
_check_model_input_dict(testCase, jdata0['learning_rate'], init_data_sys, init_batch_size, default_training_param['learning_rate'])
_check_model_input_dict(testCase, jdata0['training'], init_data_sys, init_batch_size, default_training_param['training'])
+ if reuse:
+ testCase.assertEqual(jdata['training_reuse_stop_batch'],
+ jdata0['training']['stop_batch'])
+ testCase.assertEqual(jdata['training_reuse_start_lr'],
+ jdata0['learning_rate']['start_lr'])
+ testCase.assertEqual(jdata['training_reuse_start_pref_e'],
+ jdata0['loss']['start_pref_e'])
+ testCase.assertEqual(jdata['training_reuse_start_pref_f'],
+ jdata0['loss']['start_pref_f'])
+ old_ratio = jdata['training_reuse_old_ratio']
+ testCase.assertEqual(jdata0['training']['auto_prob_style'],
+ "prob_sys_size; 0:1:%f; 1:2:%f" % (old_ratio, 1-old_ratio))
def _make_fake_fp(iter_idx, sys_idx, nframes):
@@ -143,13 +155,14 @@ def _make_fake_fp(iter_idx, sys_idx, nframes):
os.makedirs(dirname, exist_ok = True)
dirname = os.path.join('iter.%06d' % iter_idx,
'02.fp',
- 'data.%03d' % sys_idx)
+ 'data.%03d' % sys_idx)
os.makedirs(dirname, exist_ok = True)
- box_str = ['0' for ii in range(9)]
- box_str = ' '.join(box_str)
- with open(os.path.join(dirname, 'box.raw'), 'w') as fp :
- for ii in range(nframes) :
- fp.write(box_str + '\n')
+ tmp_sys = dpdata.LabeledSystem('out_data_post_fp_vasp/02.fp/task.000.000000/OUTCAR')
+ tmp_sys1 = tmp_sys.sub_system([0])
+ tmp_sys2 = tmp_sys1
+ for ii in range(1, nframes):
+ tmp_sys2.append(tmp_sys1)
+ tmp_sys2.to('deepmd/npy', dirname)
def _check_pb_link(testCase, iter_idx, numb_models) :
@@ -235,6 +248,38 @@ def test_1_data_v1(self) :
# remove testing dirs
shutil.rmtree('iter.000001')
shutil.rmtree('iter.000000')
+
+
+ def test_1_data_reuse_v1(self) :
+ with open (param_file_v1, 'r') as fp :
+ jdata = json.load (fp)
+ jdata.pop('use_ele_temp', None)
+ jdata['training_reuse_iter'] = 1
+ jdata['training_reuse_old_ratio'] = 0.8
+ jdata['training_reuse_stop_batch'] = 400000
+ jdata['training_reuse_start_lr'] = 1e-4
+ jdata['training_reuse_start_pref_e'] = 0.1
+ jdata['training_reuse_start_pref_f'] = 100
+ with open (machine_file_v1, 'r') as fp:
+ mdata = json.load (fp)
+ make_train(0, jdata, mdata)
+ # make fake fp results #data == fp_task_min
+ _make_fake_fp(0, 0, jdata['fp_task_min'])
+ # make iter1 train
+ make_train(1, jdata, mdata)
+ # check data is linked
+ self.assertTrue(os.path.isdir(os.path.join('iter.000001', '00.train', 'data.iters', 'iter.000000', '02.fp')))
+ # check old models are linked
+ self.assertTrue(os.path.isdir(os.path.join('iter.000001', '00.train', '000', 'old')))
+ self.assertTrue(os.path.isdir(os.path.join('iter.000001', '00.train', '001', 'old')))
+ self.assertTrue(os.path.isdir(os.path.join('iter.000001', '00.train', '002', 'old')))
+ self.assertTrue(os.path.isdir(os.path.join('iter.000001', '00.train', '003', 'old')))
+ # check models inputs
+ _check_model_inputs_v1(self, 1, jdata, reuse = True)
+ # remove testing dirs
+ shutil.rmtree('iter.000001')
+ shutil.rmtree('iter.000000')
+
def test_1_data_v1_eletron_temp(self) :
with open (param_file_v1_et, 'r') as fp :
diff --git a/tests/generator/test_post_fp.py b/tests/generator/test_post_fp.py
index 652f627ab..629f416b8 100644
--- a/tests/generator/test_post_fp.py
+++ b/tests/generator/test_post_fp.py
@@ -25,6 +25,7 @@
from .comp_sys import test_coord
from .comp_sys import test_cell
from .comp_sys import CompLabeledSys
+from .context import param_pwmat_file
class TestPostFPVasp(unittest.TestCase):
@@ -225,6 +226,22 @@ def setUp(self):
self.system_2 = dpdata.LabeledSystem('iter.000000/02.fp/data.000', fmt = 'deepmd/raw')
+class TestPostFPPWmat(unittest.TestCase, CompLabeledSys):
+ def setUp(self):
+ self.places = 5
+ self.e_places = 5
+ self.f_places = 5
+ self.v_places = 2
+ assert os.path.isdir('out_data_post_fp_pwmat'), 'out data for post fp pwmat should exist'
+ if os.path.isdir('iter.000000') :
+ shutil.rmtree('iter.000000')
+ shutil.copytree('out_data_post_fp_pwmat', 'iter.000000')
+ with open (param_pwmat_file, 'r') as fp :
+ jdata = json.load (fp)
+ post_fp(0, jdata)
+ self.system_1 = dpdata.LabeledSystem('iter.000000/orig', fmt = 'deepmd/raw')
+ self.system_2 = dpdata.LabeledSystem('iter.000000/02.fp/data.000', fmt = 'deepmd/raw')
+
if __name__ == '__main__':
unittest.main()