diff --git a/.gitignore b/.gitignore index 60febdeb4..59c551707 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,4 @@ dpgen.egg-info .eggs .coverage dbconfig.json +.vscode/* \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index b0963bb18..bad74118a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,6 @@ install: - pip install . # command to run tests script: - - coverage run --source=./dpgen -m unittest && coverage report + - coverage run --source=./dpgen -m unittest -v && coverage report after_success: - codecov diff --git a/README.md b/README.md index d9fed8fe8..044ef0cf0 100644 --- a/README.md +++ b/README.md @@ -21,9 +21,14 @@ ## About DP-GEN [![GitHub release](https://img.shields.io/github/release/deepmodeling/dpgen.svg?maxAge=86400)](https://github.com/deepmodeling/dpgen/releases/) -[![arxiv:1910.12690](http://img.shields.io/badge/arXiv-1910.12690-B31B1B.svg?maxAge=86400)](https://arxiv.org/abs/1910.12690) +[![doi:10.1016/j.cpc.2020.107206](https://zenodo.org/badge/DOI/10.1016/j.cpc.2020.107206.svg)](https://doi.org/10.1016/j.cpc.2020.107206) + +DP-GEN (Deep Generator) is a software written in Python, delicately designed to generate a deep learning based model of interatomic potential energy and force field. DP-GEN is depedent on DeepMD-kit (https://github.com/deepmodeling/deepmd-kit/blob/master/README.md). With highly scalable interface with common softwares for molecular simulation, DP-GEN is capable to automatically prepare scripts and maintain job queues on HPC machines (High Performance Cluster) and analyze results. + +If you use this software in any publication, please cite: + +Yuzhi Zhang, Haidi Wang, Weijie Chen, Jinzhe Zeng, Linfeng Zhang, Han Wang, and Weinan E, DP-GEN: A concurrent learning platform for the generation of reliable deep learning based potential energy models, Computer Physics Communications, 2020, 107206. -DP-GEN (Deep Generator) is a software written in Python, delicately designed to generate a deep learning based model of interatomic potential energy and force field. DP-GEN is depedent on DeepMD-kit (https://github.com/deepmodeling/deepmd-kit/blob/master/README.md). With highly scalable interface with common softwares for molecular simulation, DP-GEN is capable to automatically prepare scripts and maintain job queues on HPC machines (High Performance Cluster) and analyze results ### Highlighted features + **Accurate and efficient**: DP-GEN is capable to sample more than tens of million structures and select only a few for first principles calculation. DP-GEN will finally obtain a uniformly accurate model. + **User-friendly and automatic**: Users may install and run DP-GEN easily. Once succusefully running, DP-GEN can dispatch and handle all jobs on HPCs, and thus there's no need for any personal effort. @@ -56,7 +61,6 @@ Options for TASK: * `test`: Auto-test for Deep Potential. * `db`: Collecting data from DP-GEN. - ## Download and Install One can download the source code of dpgen by ```bash @@ -264,6 +268,52 @@ Following is an example for `PARAM`, which generates data from a typical structu } ``` +Another example is `from_poscar` method. Here you need to specify the POSCAR file. + +``` +{ + "stages": [ + 1, + 2 + ], + "cell_type": "fcc", + "from_poscar": true, + "from_poscar_path": "POSCAR", + "super_cell": [ + 1, + 1, + 1 + ], + "layer_numb": 3, + "vacuum_max": 5, + "vacuum_resol": [0.5,2], + "mid_point": 2.0, + "millers": [ + [ + 1, + 0, + 0 + ] + ], + "elements": [ + "Al" + ], + "potcars": [ + "./POTCAR" + ], + "relax_incar" : "INCAR_metal_rlx_low", + "scale": [ + 1.0 + ], + "skip_relax": true, + "pert_numb": 5, + "pert_box": 0.03, + "pert_atom": 0.01, + "coll_ndata": 5000, + "_comment": "that's all" +} +``` + The following table gives explicit descriptions on keys in `PARAM`. The bold notation of key (such as **Elements**) means that it's a necessary key. @@ -275,7 +325,9 @@ The bold notation of key (such as **Elements**) means that it's a necessary key. | **cell_type** | String | "hcp" | Specifying which typical structure to be generated. **Options** include fcc, hcp, bcc, sc, diamond. | **latt** | Float | 4.479 | Lattice constant for single cell. | **layer_numb** | Integer | 3 | Number of equavilent layers of slab. +| **z__min** | Float | 9.0 | Thickness of slab without vacuum (Angstrom). If the `layer_numb` and `z_min` are all setted, the `z_min` value will be ignored. | **vacuum_max** | Float | 9 | Maximal thickness of vacuum (Angstrom). +| vacuum_min | Float | 3.0 | Minimal thickness of vacuum (Angstrom). Default value is 2 times atomic radius. | **vacuum_resol** | List of float | [0.5, 1 ] | Interval of thichness of vacuum. If size of `vacuum_resol` is 1, the interval is fixed to its value. If size of `vacuum_resol` is 2, the interval is `vacuum_resol[0]` before `mid_point`, otherwise `vacuum_resol[1]` after `mid_point`. | **millers** | List of list of Integer | [[1,0,0]] | Miller indices. | relax_incar | String | "....../INCAR" | Path of INCAR for relaxation in VASP. **Necessary** if `stages` include 1. @@ -466,6 +518,8 @@ The bold notation of key (such aas **type_map**) means that it's a necessary key | **sys_batch_size** | List of integer | [8, 8] | Each number is the batch_size for training of corresponding system in `sys_configs`. If set to `auto`, batch size will be 32 divided by number of atoms. | | *#Training* | **numb_models** | Integer | 4 (recommend) | Number of models to be trained in `00.train`. | +| training_iter0_model_path | list of string | ["/path/to/model0_ckpt/", ...] | The model used to init the first iter training. Number of element should be equal to `numb_models` | +| training_init_model | bool | False | Iteration > 0, the model parameters will be initilized from the model trained at the previous iteration. Iteration == 0, the model parameters will be initialized from `training_iter0_model_path`. | | **default_training_param** | Dict | {
...
"use_smooth": true,
"sel_a": [16, 4],
"rcut_smth": 0.5,
"rcut": 5,
"filter_neuron": [10, 20, 40],
...
} | Training parameters for `deepmd-kit` in `00.train`.
You can find instructions from here: (https://github.com/deepmodeling/deepmd-kit)..
We commonly let `stop_batch` = 200 * `decay_steps`. | | *#Exploration* | **model_devi_dt** | Float | 0.002 (recommend) | Timestep for MD | @@ -475,6 +529,8 @@ The bold notation of key (such aas **type_map**) means that it's a necessary key | **model_devi_e_trust_lo** | Float | 1e10 | Lower bound of energies for the selection. Recommend to set them a high number, since forces provide more precise information. Special cases such as energy minimization may need this. | | **model_devi_e_trust_hi** | Float | 1e10 | Upper bound of energies for the selection. | | **model_devi_clean_traj** | Boolean | true | Deciding whether to clean traj folders in MD since they are too large. | +| **model_devi_nopbc** | Boolean | False | Assume open boundary condition in MD simulations. | +| model_devi_activation_func | List of String | ["tanh", "tanh", "tanh", "tanh"] | Set activation functions for models, length of the list should be the same as `numb_models` | | **model_devi_jobs** | [
{
"sys_idx": [0],
"temps":
[100],
"press":
[1],
"trj_freq":
10,
"nsteps":
1000,
"ensembles":
"nvt"
},
...
] | List of dict | Settings for exploration in `01.model_devi`. Each dict in the list corresponds to one iteration. The index of `model_devi_jobs` exactly accord with index of iterations | | **model_devi_jobs["sys_idx"]** | List of integer | [0] | Systems to be selected as the initial structure of MD and be explored. The index corresponds exactly to the `sys_configs`. | | **model_devi_jobs["temps"]** | List of integer | [50, 300] | Temperature (**K**) in MD @@ -489,10 +545,13 @@ The bold notation of key (such aas **type_map**) means that it's a necessary key | **fp_style** | string | "vasp" | Software for First Principles. **Options** include “vasp”, “pwscf”, “siesta” and “gaussian” up to now. | | **fp_task_max** | Integer | 20 | Maximum of structures to be calculated in `02.fp` of each iteration. | | **fp_task_min** | Integer | 5 | Minimum of structures to calculate in `02.fp` of each iteration. | +| fp_accurate_threshold | Float | 0.9999 | If the accurate ratio is larger than this number, no fp calculation will be performed, i.e. fp_task_max = 0. | +| fp_accurate_soft_threshold | Float | 0.9999 | If the accurate ratio is between this number and `fp_accurate_threshold`, the fp_task_max linearly decays to zero. | | *fp_style == VASP* | **fp_pp_path** | String | "/sharedext4/.../ch4/" | Directory of psuedo-potential file to be used for 02.fp exists. | | **fp_pp_files** | List of string | ["POTCAR"] | Psuedo-potential file to be used for 02.fp. Note that the order of elements should correspond to the order in `type_map`. | -|**fp_incar** | String | "/sharedext4/../ch4/INCAR" | Input file for VASP. INCAR must specify KSPACING. +|**fp_incar** | String | "/sharedext4/../ch4/INCAR" | Input file for VASP. INCAR must specify KSPACING and KGAMMA. +|**fp_aniso_kspacing** | List of integer | [1.0,1.0,1.0] | Set anisotropic kspacing. Usually useful for 1-D or 2-D materials. Only support VASP. If it is setting the KSPACING key in INCAR will be ignored. |cvasp| Boolean | true | If `cvasp` is true, DP-GEN will use Custodian to help control VASP calculation. | *fp_style == Gaussian* | **use_clusters** | Boolean | false | If set to `true`, clusters will be taken instead of the whole system. This option does not work with DeePMD-kit 0.x. @@ -632,7 +691,7 @@ param.json in a dictionary. | conf_dir | path like string | "confs/Al/std-fcc" | the dir which contains vasp's POSCAR | | key_id | string| "DZIwdXCXg1fiXXXXXX" |the API key of Material project| | task_type | string | "vasp" | task type, one of deepmd vasp meam | -| task | string | "equi" | task, one of equi, eos, elastic, vacancy, interstitial, surf or all | +| task | string or list | "equi" | task, one or several tasks from { equi, eos, elastic, vacancy, interstitial, surf } or all stands for all tasks | | vasp_params| dict | seeing below | params relating to vasp INCAR| | lammps_params | dict| seeing below| params relating to lammps | diff --git a/dpgen/__init__.py b/dpgen/__init__.py index d77ec3898..b38875dee 100644 --- a/dpgen/__init__.py +++ b/dpgen/__init__.py @@ -45,3 +45,13 @@ def info(): except ImportError: print('%10s %10s Not Found' % (modui, '')) print() + + # reference + print("""Reference +------------ +Please cite: +Yuzhi Zhang, Haidi Wang, Weijie Chen, Jinzhe Zeng, Linfeng Zhang, Han Wang, and Weinan E, +DP-GEN: A concurrent learning platform for the generation of reliable deep learning +based potential energy models, Computer Physics Communications, 2020, 107206. +------------ +""") diff --git a/dpgen/auto_test/cmpt_01_eos.py b/dpgen/auto_test/cmpt_01_eos.py index f5670c422..385475059 100755 --- a/dpgen/auto_test/cmpt_01_eos.py +++ b/dpgen/auto_test/cmpt_01_eos.py @@ -12,7 +12,7 @@ def comput_lmp_eos(jdata,conf_dir, task_name) : conf_path = os.path.abspath(conf_path) conf_path = os.path.join(conf_path, task_name) vol_paths = glob.glob(os.path.join(conf_path, 'vol-*')) - vol_paths.sort() + vol_paths.sort(key=lambda k : float(k.split('-')[-1])) result = os.path.join(conf_path,'result') print('Vpa(A^3)\tEpA(eV)') with open(result,'w') as fp: @@ -38,7 +38,7 @@ def comput_vasp_eos(jdata, conf_dir) : vasp_str='vasp-k%.2f' % kspacing task_path = os.path.join(conf_path, vasp_str) vol_paths = glob.glob(os.path.join(task_path, 'vol-*')) - vol_paths.sort() + vol_paths.sort(key=lambda k : float(k.split('-')[-1])) result = os.path.join(task_path,'result') print('Vpa(A^3)\tEpA(eV)') with open(result,'w') as fp: diff --git a/dpgen/auto_test/gen_00_equi.py b/dpgen/auto_test/gen_00_equi.py index 59c1ddfdb..a2b5b00c9 100755 --- a/dpgen/auto_test/gen_00_equi.py +++ b/dpgen/auto_test/gen_00_equi.py @@ -1,11 +1,19 @@ #!/usr/bin/env python3 -import os, re, argparse, filecmp, json, glob +import os, re, argparse, filecmp, json, glob, shutil import subprocess as sp import numpy as np import dpgen.auto_test.lib.vasp as vasp import dpgen.auto_test.lib.lammps as lammps +from dpgen import dlog +from dpgen.generator.lib.vasp import incar_upper +from dpgen import ROOT_PATH +from pymatgen.io.vasp import Incar +from dpgen.generator.lib.vasp import incar_upper + +cvasp_file=os.path.join(ROOT_PATH,'generator/lib/cvasp.py') + global_task_name = '00.equi' ''' @@ -48,7 +56,14 @@ def make_vasp(jdata, conf_dir) : relax_incar_path = jdata['relax_incar'] assert(os.path.exists(relax_incar_path)) relax_incar_path = os.path.abspath(relax_incar_path) - fc = open(relax_incar_path).read() + incar = incar_upper(Incar.from_file(relax_incar_path)) + isif = 3 + if incar.get('ISIF') != isif: + dlog.info("%s:%s setting ISIF to %d" % (__file__, make_vasp.__name__, isif)) + incar['ISIF'] = isif + fc = incar.get_string() + kspacing = incar['KSPACING'] + kgamma = incar['KGAMMA'] vasp_path = os.path.join(equi_path, 'vasp-relax_incar' ) else : fp_params = jdata['vasp_params'] @@ -63,14 +78,24 @@ def make_vasp(jdata, conf_dir) : os.makedirs(vasp_path, exist_ok = True) os.chdir(vasp_path) - print(vasp_path) + # write incar with open('INCAR', 'w') as fp : fp.write(fc) + # gen poscar if os.path.exists('POSCAR') : os.remove('POSCAR') os.symlink(os.path.relpath(to_poscar), 'POSCAR') + + # gen kpoints + fc = vasp.make_kspacing_kpoints('POSCAR', kspacing, kgamma) + with open('KPOINTS', 'w') as fp: fp.write(fc) + + #copy cvasp + if ('cvasp' in jdata) and (jdata['cvasp'] == True): + shutil.copyfile(cvasp_file, 'cvasp.py') + # gen potcar with open('POTCAR', 'w') as outfile: for fname in potcar_list: @@ -84,6 +109,7 @@ def make_lammps (jdata, conf_dir,task_type) : type_map = fp_params['type_map'] model_dir = os.path.abspath(model_dir) model_name =fp_params['model_name'] + deepmd_version = fp_params.get("deepmd_version", "0.12") if not model_name and task_type =='deepmd': models = glob.glob(os.path.join(model_dir, '*pb')) model_name = [os.path.basename(ii) for ii in models] @@ -91,8 +117,9 @@ def make_lammps (jdata, conf_dir,task_type) : else: models = [os.path.join(model_dir,ii) for ii in model_name] - model_param = {'model_name' : fp_params['model_name'], - 'param_type': fp_params['model_param_type']} + model_param = {'model_name' : model_name, + 'param_type': fp_params['model_param_type'], + 'deepmd_version' : deepmd_version} ntypes = len(type_map) conf_path = os.path.abspath(conf_dir) @@ -121,7 +148,7 @@ def make_lammps (jdata, conf_dir,task_type) : fc = lammps.make_lammps_equi(os.path.basename(conf_file), ntypes, lammps.inter_deepmd, - model_name) + model_param) elif task_type=='meam': fc = lammps.make_lammps_equi(os.path.basename(conf_file), ntypes, diff --git a/dpgen/auto_test/gen_01_eos.py b/dpgen/auto_test/gen_01_eos.py index c129a4df3..ba9fae386 100755 --- a/dpgen/auto_test/gen_01_eos.py +++ b/dpgen/auto_test/gen_01_eos.py @@ -1,11 +1,18 @@ #!/usr/bin/env python3 -import os, re, argparse, filecmp, json, glob +import os, re, argparse, filecmp, json, glob, shutil import subprocess as sp import numpy as np import dpgen.auto_test.lib.vasp as vasp import dpgen.auto_test.lib.lammps as lammps + +from dpgen import dlog +from dpgen.generator.lib.vasp import incar_upper from pymatgen.core.structure import Structure +from pymatgen.io.vasp import Incar +from dpgen import ROOT_PATH + +cvasp_file=os.path.join(ROOT_PATH,'generator/lib/cvasp.py') global_equi_name = '00.equi' global_task_name = '01.eos' @@ -19,12 +26,26 @@ def make_vasp(jdata, conf_dir) : vol_start = jdata['vol_start'] vol_end = jdata['vol_end'] vol_step = jdata['vol_step'] - + eos_relax_cell_shape = jdata.get('eos_relax_cell_shape', True) conf_path = os.path.abspath(conf_dir) + conf_poscar = os.path.join(conf_path, 'POSCAR') + + if 'relax_incar' in jdata.keys(): + vasp_str='vasp-relax_incar' + else: + kspacing = jdata['vasp_params']['kspacing'] + vasp_str='vasp-k%.2f' % kspacing + + # get equi poscar + equi_path = re.sub('confs', global_equi_name, conf_path) + equi_path = os.path.join(equi_path, vasp_str) + equi_contcar = os.path.join(equi_path, 'CONTCAR') task_path = re.sub('confs', global_task_name, conf_path) - os.makedirs(task_path, exist_ok = True) + task_path = os.path.join(task_path, vasp_str) + os.makedirs(task_path, exist_ok = True) + # link poscar cwd = os.getcwd() - from_poscar = os.path.join(conf_path, 'POSCAR') + from_poscar = os.path.join(equi_contcar) to_poscar = os.path.join(task_path, 'POSCAR') if os.path.exists(to_poscar) : assert(filecmp.cmp(from_poscar, to_poscar)) @@ -56,8 +77,17 @@ def make_vasp(jdata, conf_dir) : relax_incar_path = jdata['relax_incar'] assert(os.path.exists(relax_incar_path)) relax_incar_path = os.path.abspath(relax_incar_path) - fc = open(relax_incar_path).read() - vasp_path = os.path.join(task_path, 'vasp-relax_incar' ) + incar = incar_upper(Incar.from_file(relax_incar_path)) + if eos_relax_cell_shape: + isif = 4 + else: + isif = 2 + if incar.get('ISIF') != isif: + dlog.info("%s:%s setting ISIF to %d" % (__file__, make_vasp.__name__, isif)) + incar['ISIF'] = isif + fc = incar.get_string() + kspacing = incar['KSPACING'] + kgamma = incar['KGAMMA'] else : fp_params = jdata['vasp_params'] ecut = fp_params['ecut'] @@ -67,11 +97,8 @@ def make_vasp(jdata, conf_dir) : kspacing = fp_params['kspacing'] kgamma = fp_params['kgamma'] fc = vasp.make_vasp_relax_incar(ecut, ediff, is_alloy, True, False, npar, kpar, kspacing, kgamma) - vasp_path = os.path.join(task_path, 'vasp-k%.2f' % kspacing) - os.makedirs(vasp_path, exist_ok = True) - os.chdir(vasp_path) - print(vasp_path) + os.chdir(task_path) with open('INCAR', 'w') as fp : fp.write(fc) @@ -82,30 +109,37 @@ def make_vasp(jdata, conf_dir) : outfile.write(infile.read()) # loop over volumes for vol in np.arange(vol_start, vol_end, vol_step) : - vol_path = os.path.join(vasp_path, 'vol-%.2f' % vol) + vol_path = os.path.join(task_path, 'vol-%.2f' % vol) os.makedirs(vol_path, exist_ok = True) os.chdir(vol_path) - print(vol_path) for ii in ['INCAR', 'POTCAR', 'POSCAR.orig', 'POSCAR'] : if os.path.exists(ii) : os.remove(ii) # link incar, potcar - os.symlink(os.path.relpath(os.path.join(vasp_path, 'INCAR')), 'INCAR') - os.symlink(os.path.relpath(os.path.join(vasp_path, 'POTCAR')), 'POTCAR') + os.symlink(os.path.relpath(os.path.join(task_path, 'INCAR')), 'INCAR') + os.symlink(os.path.relpath(os.path.join(task_path, 'POTCAR')), 'POTCAR') # gen poscar os.symlink(os.path.relpath(to_poscar), 'POSCAR.orig') scale = (vol / vol_to_poscar) ** (1./3.) # print(scale) vasp.poscar_scale('POSCAR.orig', 'POSCAR', scale) # print(vol_path, vasp.poscar_vol('POSCAR') / vasp.poscar_natoms('POSCAR')) + # gen kpoints + fc = vasp.make_kspacing_kpoints('POSCAR', kspacing, kgamma) + with open('KPOINTS', 'w') as fp: fp.write(fc) + #copy cvasp + if ('cvasp' in jdata) and (jdata['cvasp'] == True): + shutil.copyfile(cvasp_file, os.path.join(vol_path,'cvasp.py')) os.chdir(cwd) + def make_lammps (jdata, conf_dir,task_type) : fp_params = jdata['lammps_params'] model_dir = fp_params['model_dir'] type_map = fp_params['type_map'] model_dir = os.path.abspath(model_dir) model_name =fp_params['model_name'] + deepmd_version = fp_params.get("deepmd_version", "0.12") if not model_name and task_type =='deepmd': models = glob.glob(os.path.join(model_dir, '*pb')) model_name = [os.path.basename(ii) for ii in models] @@ -113,9 +147,9 @@ def make_lammps (jdata, conf_dir,task_type) : else: models = [os.path.join(model_dir,ii) for ii in model_name] - model_param = {'model_name' : fp_params['model_name'], - 'param_type': fp_params['model_param_type']} - + model_param = {'model_name' : model_name, + 'param_type': fp_params['model_param_type'], + 'deepmd_version' : deepmd_version} ntypes = len(type_map) vol_start = jdata['vol_start'] @@ -194,7 +228,7 @@ def make_lammps (jdata, conf_dir,task_type) : # make lammps input scale = (vol / vpa) ** (1./3.) if task_type=='deepmd': - fc = lammps.make_lammps_press_relax('conf.lmp', ntypes, scale,lammps.inter_deepmd, model_name) + fc = lammps.make_lammps_press_relax('conf.lmp', ntypes, scale,lammps.inter_deepmd, model_param) elif task_type =='meam': fc = lammps.make_lammps_press_relax('conf.lmp', ntypes, scale,lammps.inter_meam, model_param) with open(os.path.join(vol_path, 'lammps.in'), 'w') as fp : @@ -207,15 +241,16 @@ def make_lammps_fixv (jdata, conf_dir,task_type) : type_map = fp_params['type_map'] model_dir = os.path.abspath(model_dir) model_name =fp_params['model_name'] + deepmd_version = fp_params.get("deepmd_version", "0.12") if not model_name and task_type =='deepmd': models = glob.glob(os.path.join(model_dir, '*pb')) model_name = [os.path.basename(ii) for ii in models] else: models = [os.path.join(model_dir,ii) for ii in model_name] - model_param = {'model_name' : fp_params['model_name'], - 'param_type': fp_params['model_param_type']} - + model_param = {'model_name' : model_name, + 'param_type': fp_params['model_param_type'], + 'deepmd_version' : deepmd_version} ntypes = len(type_map) @@ -246,7 +281,7 @@ def make_lammps_fixv (jdata, conf_dir,task_type) : fc = lammps.make_lammps_equi('conf.lmp', ntypes, lammps.inter_deepmd, - model_name, + model_param, change_box = False) elif task_type=='meam': fc = lammps.make_lammps_equi('conf.lmp', diff --git a/dpgen/auto_test/gen_02_elastic.py b/dpgen/auto_test/gen_02_elastic.py index fa9ca9053..13b20a715 100755 --- a/dpgen/auto_test/gen_02_elastic.py +++ b/dpgen/auto_test/gen_02_elastic.py @@ -1,29 +1,38 @@ #!/usr/bin/env python3 -import os, re, argparse, filecmp, json, glob +import os, re, argparse, filecmp, json, glob, shutil import subprocess as sp import numpy as np import dpgen.auto_test.lib.vasp as vasp import dpgen.auto_test.lib.lammps as lammps + +from dpgen import dlog +from dpgen.generator.lib.vasp import incar_upper from pymatgen.core.structure import Structure from pymatgen.analysis.elasticity.strain import Deformation, DeformedStructureSet, Strain +from pymatgen.io.vasp import Incar +from dpgen import ROOT_PATH + +cvasp_file=os.path.join(ROOT_PATH,'generator/lib/cvasp.py') global_equi_name = '00.equi' global_task_name = '02.elastic' -def make_vasp(jdata, conf_dir, norm_def = 2e-3, shear_def = 5e-3) : - norm_def = jdata['norm_deform'] - shear_def = jdata['shear_deform'] +def make_vasp(jdata, conf_dir) : + default_norm_def = 2e-3 + default_shear_def = 5e-3 + norm_def = jdata.get('norm_deform', default_norm_def) + shear_def = jdata.get('shear_deform', default_shear_def) conf_path = os.path.abspath(conf_dir) conf_poscar = os.path.join(conf_path, 'POSCAR') - # get equi poscar if 'relax_incar' in jdata.keys(): vasp_str='vasp-relax_incar' else: kspacing = jdata['vasp_params']['kspacing'] vasp_str='vasp-k%.2f' % kspacing + # get equi poscar equi_path = re.sub('confs', global_equi_name, conf_path) equi_path = os.path.join(equi_path, vasp_str) equi_contcar = os.path.join(equi_path, 'CONTCAR') @@ -55,10 +64,14 @@ def make_vasp(jdata, conf_dir, norm_def = 2e-3, shear_def = 5e-3) : if 'relax_incar' in jdata.keys(): relax_incar_path = jdata['relax_incar'] assert(os.path.exists(relax_incar_path)) - relax_incar_path = os.path.abspath(relax_incar_path) - fc = open(relax_incar_path).read() - kspacing =float(re.findall((r"KSPACING(.+?)\n"),fc)[0].replace('=','')) - kgamma =('T' in re.findall((r"KGAMMA(.+?)\n"),fc)[0]) + relax_incar_path = os.path.abspath(relax_incar_path) + incar = incar_upper(Incar.from_file(relax_incar_path)) + if incar.get('ISIF') != 2: + dlog.info("%s:%s setting ISIF to 2" % (__file__, make_vasp.__name__)) + incar['ISIF'] = 2 + fc = incar.get_string() + kspacing = incar['KSPACING'] + kgamma = incar['KGAMMA'] else : fp_params = jdata['vasp_params'] ecut = fp_params['ecut'] @@ -107,7 +120,11 @@ def make_vasp(jdata, conf_dir, norm_def = 2e-3, shear_def = 5e-3) : os.symlink(os.path.relpath(os.path.join(task_path, 'INCAR')), 'INCAR') os.symlink(os.path.relpath(os.path.join(task_path, 'POTCAR')), 'POTCAR') os.symlink(os.path.relpath(os.path.join(task_path, 'KPOINTS')), 'KPOINTS') - cwd = os.getcwd() + #copy cvasp + if ('cvasp' in jdata) and (jdata['cvasp'] == True): + shutil.copyfile(cvasp_file, os.path.join(dfm_path,'cvasp.py')) + os.chdir(cwd) + def make_lammps(jdata, conf_dir,task_type) : fp_params = jdata['lammps_params'] @@ -115,6 +132,7 @@ def make_lammps(jdata, conf_dir,task_type) : type_map = fp_params['type_map'] model_dir = os.path.abspath(model_dir) model_name =fp_params['model_name'] + deepmd_version = fp_params.get("deepmd_version", "0.12") if not model_name and task_type =='deepmd': models = glob.glob(os.path.join(model_dir, '*pb')) model_name = [os.path.basename(ii) for ii in models] @@ -122,8 +140,9 @@ def make_lammps(jdata, conf_dir,task_type) : else: models = [os.path.join(model_dir,ii) for ii in model_name] - model_param = {'model_name' : fp_params['model_name'], - 'param_type': fp_params['model_param_type']} + model_param = {'model_name' : model_name, + 'param_type': fp_params['model_param_type'], + 'deepmd_version' : deepmd_version} ntypes = len(type_map) @@ -168,7 +187,7 @@ def make_lammps(jdata, conf_dir,task_type) : fc = lammps.make_lammps_elastic('conf.lmp', ntypes, lammps.inter_deepmd, - model_name) + model_param) elif task_type=='meam': fc = lammps.make_lammps_elastic('conf.lmp', ntypes, diff --git a/dpgen/auto_test/gen_03_vacancy.py b/dpgen/auto_test/gen_03_vacancy.py index 93b061ea9..f50591b45 100755 --- a/dpgen/auto_test/gen_03_vacancy.py +++ b/dpgen/auto_test/gen_03_vacancy.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import os, re, argparse, filecmp, json, glob +import os, re, argparse, filecmp, json, glob, shutil import subprocess as sp import numpy as np import dpgen.auto_test.lib.vasp as vasp @@ -9,6 +9,12 @@ from pymatgen.analysis.defects.core import Vacancy from pymatgen.analysis.defects.generators import VacancyGenerator +from dpgen import ROOT_PATH +from pymatgen.io.vasp import Incar +from dpgen.generator.lib.vasp import incar_upper +cvasp_file=os.path.join(ROOT_PATH,'generator/lib/cvasp.py') + + global_equi_name = '00.equi' global_task_name = '03.vacancy' @@ -48,7 +54,10 @@ def make_vasp(jdata, conf_dir, supercell = [1,1,1]) : relax_incar_path = jdata['relax_incar'] assert(os.path.exists(relax_incar_path)) relax_incar_path = os.path.abspath(relax_incar_path) - fc = open(relax_incar_path).read() + incar = incar_upper(Incar.from_file(relax_incar_path)) + fc = incar.get_string() + kspacing = incar['KSPACING'] + kgamma = incar['KGAMMA'] else : fp_params = jdata['vasp_params'] ecut = fp_params['ecut'] @@ -91,6 +100,14 @@ def make_vasp(jdata, conf_dir, supercell = [1,1,1]) : os.symlink(os.path.relpath(os.path.join(task_path, 'POTCAR')), 'POTCAR') # save supercell np.savetxt('supercell.out', supercell, fmt='%d') + + # write kp + fc = vasp.make_kspacing_kpoints('POSCAR', kspacing, kgamma) + with open('KPOINTS', 'w') as fp: fp.write(fc) + + #copy cvasp + if ('cvasp' in jdata) and (jdata['cvasp'] == True): + shutil.copyfile(cvasp_file, os.path.join(struct_path,'cvasp.py')) os.chdir(cwd) def make_lammps(jdata, conf_dir, task_type, supercell) : @@ -100,6 +117,7 @@ def make_lammps(jdata, conf_dir, task_type, supercell) : type_map = fp_params['type_map'] model_dir = os.path.abspath(model_dir) model_name =fp_params['model_name'] + deepmd_version = fp_params.get("deepmd_version", "0.12") if not model_name and task_type =='deepmd': models = glob.glob(os.path.join(model_dir, '*pb')) model_name = [os.path.basename(ii) for ii in models] @@ -107,8 +125,9 @@ def make_lammps(jdata, conf_dir, task_type, supercell) : else: models = [os.path.join(model_dir,ii) for ii in model_name] - model_param = {'model_name' : fp_params['model_name'], - 'param_type': fp_params['model_param_type']} + model_param = {'model_name' : model_name, + 'param_type': fp_params['model_param_type'], + 'deepmd_version' : deepmd_version} ntypes = len(type_map) @@ -154,7 +173,7 @@ def make_lammps(jdata, conf_dir, task_type, supercell) : ntypes, 1, lammps.inter_deepmd, - model_name) + model_param) elif task_type =='meam': fc = lammps.make_lammps_press_relax('conf.lmp', ntypes, diff --git a/dpgen/auto_test/gen_04_interstitial.py b/dpgen/auto_test/gen_04_interstitial.py index 376be45c6..f1c10c4c3 100755 --- a/dpgen/auto_test/gen_04_interstitial.py +++ b/dpgen/auto_test/gen_04_interstitial.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import os, re, argparse, filecmp, json, glob, warnings +import os, re, argparse, filecmp, json, glob, warnings, shutil import subprocess as sp import numpy as np import dpgen.auto_test.lib.vasp as vasp @@ -9,6 +9,12 @@ from pymatgen.analysis.defects.core import Interstitial from pymatgen.analysis.defects.generators import InterstitialGenerator +from dpgen import ROOT_PATH +from pymatgen.io.vasp import Incar +from dpgen.generator.lib.vasp import incar_upper +cvasp_file=os.path.join(ROOT_PATH,'generator/lib/cvasp.py') + + global_equi_name = '00.equi' global_task_name = '04.interstitial' @@ -53,7 +59,11 @@ def _make_vasp(jdata, conf_dir, supercell, insert_ele) : relax_incar_path = jdata['relax_incar'] assert(os.path.exists(relax_incar_path)) relax_incar_path = os.path.abspath(relax_incar_path) - fc = open(relax_incar_path).read() + incar = incar_upper(Incar.from_file(relax_incar_path)) + fc = incar.get_string() + kspacing = incar['KSPACING'] + kgamma = incar['KGAMMA'] + else : fp_params = jdata['vasp_params'] ecut = fp_params['ecut'] @@ -100,6 +110,14 @@ def _make_vasp(jdata, conf_dir, supercell, insert_ele) : os.symlink(os.path.relpath(os.path.join(task_path, 'INCAR')), 'INCAR') # save supercell np.savetxt('supercell.out', supercell, fmt='%d') + + # write kp + fc = vasp.make_kspacing_kpoints('POSCAR', kspacing, kgamma) + with open('KPOINTS', 'w') as fp: fp.write(fc) + + #copy cvasp + if ('cvasp' in jdata) and (jdata['cvasp'] == True): + shutil.copyfile(cvasp_file, os.path.join(struct_path,'cvasp.py')) os.chdir(cwd) @@ -114,6 +132,7 @@ def _make_reprod_traj(jdata, conf_dir, supercell, insert_ele, task_type) : type_map = fp_params['type_map'] model_dir = os.path.abspath(model_dir) model_name =fp_params['model_name'] + deepmd_version = fp_params.get("deepmd_version", "0.12") if not model_name and task_type=='deepmd': models = glob.glob(os.path.join(model_dir, '*pb')) model_name = [os.path.basename(ii) for ii in models] @@ -121,8 +140,9 @@ def _make_reprod_traj(jdata, conf_dir, supercell, insert_ele, task_type) : else: models = [os.path.join(model_dir,ii) for ii in model_name] - model_param = {'model_name' : fp_params['model_name'], - 'param_type': fp_params['model_param_type']} + model_param = {'model_name' : model_name, + 'param_type': fp_params['model_param_type'], + 'deepmd_version' : deepmd_version} ntypes = len(type_map) @@ -152,7 +172,7 @@ def _make_reprod_traj(jdata, conf_dir, supercell, insert_ele, task_type) : fc = lammps.make_lammps_eval('conf.lmp', ntypes, lammps.inter_deepmd, - model_name) + model_param) elif task_type =='meam': fc = lammps.make_lammps_eval('conf.lmp', ntypes, @@ -238,6 +258,7 @@ def _make_lammps(jdata, conf_dir, supercell, insert_ele, task_type) : type_map = fp_params['type_map'] model_dir = os.path.abspath(model_dir) model_name =fp_params['model_name'] + deepmd_version = fp_params.get("deepmd_version", "0.12") if not model_name and task_type=='deepmd': models = glob.glob(os.path.join(model_dir, '*pb')) model_name = [os.path.basename(ii) for ii in models] @@ -245,8 +266,9 @@ def _make_lammps(jdata, conf_dir, supercell, insert_ele, task_type) : else: models = [os.path.join(model_dir,ii) for ii in model_name] - model_param = {'model_name' : fp_params['model_name'], - 'param_type': fp_params['model_param_type']} + model_param = {'model_name' : model_name, + 'param_type': fp_params['model_param_type'], + 'deepmd_version' : deepmd_version} ntypes = len(type_map) @@ -291,7 +313,7 @@ def _make_lammps(jdata, conf_dir, supercell, insert_ele, task_type) : ntypes, 1, lammps.inter_deepmd, - model_name) + model_param) elif task_type =='meam': fc = lammps.make_lammps_press_relax('conf.lmp', ntypes, diff --git a/dpgen/auto_test/gen_05_surf.py b/dpgen/auto_test/gen_05_surf.py index 56a275630..ee6cd662d 100755 --- a/dpgen/auto_test/gen_05_surf.py +++ b/dpgen/auto_test/gen_05_surf.py @@ -1,12 +1,18 @@ #!/usr/bin/env python3 -import os, re, argparse, filecmp, json, glob +import os, re, argparse, filecmp, json, glob, shutil import subprocess as sp import numpy as np import dpgen.auto_test.lib.vasp as vasp import dpgen.auto_test.lib.lammps as lammps from pymatgen.core.surface import generate_all_slabs, Structure +from dpgen import ROOT_PATH +from pymatgen.io.vasp import Incar +from dpgen.generator.lib.vasp import incar_upper +cvasp_file=os.path.join(ROOT_PATH,'generator/lib/cvasp.py') + + global_equi_name = '00.equi' global_task_name = '05.surf' @@ -59,7 +65,10 @@ def make_vasp(jdata, conf_dir, max_miller = 2, relax_box = False, static = False scf_incar_path = jdata['scf_incar'] assert(os.path.exists(scf_incar_path)) scf_incar_path = os.path.abspath(scf_incar_path) - fc = open(scf_incar_path).read() + incar = incar_upper(Incar.from_file(scf_incar_path)) + fc = incar.get_string() + kspacing = incar['KSPACING'] + kgamma = incar['KGAMMA'] else : fp_params = jdata['vasp_params'] ecut = fp_params['ecut'] @@ -74,7 +83,10 @@ def make_vasp(jdata, conf_dir, max_miller = 2, relax_box = False, static = False relax_incar_path = jdata['relax_incar'] assert(os.path.exists(relax_incar_path)) relax_incar_path = os.path.abspath(relax_incar_path) - fc = open(relax_incar_path).read() + incar = incar_upper(Incar.from_file(relax_incar_path)) + fc = incar.get_string() + kspacing = incar['KSPACING'] + kgamma = incar['KGAMMA'] else : fp_params = jdata['vasp_params'] ecut = fp_params['ecut'] @@ -122,6 +134,14 @@ def make_vasp(jdata, conf_dir, max_miller = 2, relax_box = False, static = False # link incar, potcar, kpoints os.symlink(os.path.relpath(os.path.join(task_path, 'INCAR')), 'INCAR') os.symlink(os.path.relpath(os.path.join(task_path, 'POTCAR')), 'POTCAR') + + # write kp + fc = vasp.make_kspacing_kpoints('POSCAR', kspacing, kgamma) + with open('KPOINTS', 'w') as fp: fp.write(fc) + + #copy cvasp + if ('cvasp' in jdata) and (jdata['cvasp'] == True): + shutil.copyfile(cvasp_file, os.path.join(struct_path,'cvasp.py')) cwd = os.getcwd() def make_lammps(jdata, conf_dir, max_miller = 2, static = False, relax_box = False, task_type = 'wrong-task') : @@ -131,6 +151,7 @@ def make_lammps(jdata, conf_dir, max_miller = 2, static = False, relax_box = Fal type_map = fp_params['type_map'] model_dir = os.path.abspath(model_dir) model_name =fp_params['model_name'] + deepmd_version = fp_params.get("deepmd_version", "0.12") if not model_name and task_type=='deepmd': models = glob.glob(os.path.join(model_dir, '*pb')) model_name = [os.path.basename(ii) for ii in models] @@ -138,8 +159,9 @@ def make_lammps(jdata, conf_dir, max_miller = 2, static = False, relax_box = Fal else: models = [os.path.join(model_dir,ii) for ii in model_name] - model_param = {'model_name' : fp_params['model_name'], - 'param_type': fp_params['model_param_type']} + model_param = {'model_name' : model_name, + 'param_type': fp_params['model_param_type'], + 'deepmd_version' : deepmd_version} ntypes = len(type_map) @@ -184,12 +206,12 @@ def make_lammps(jdata, conf_dir, max_miller = 2, static = False, relax_box = Fal fc = lammps.make_lammps_eval('conf.lmp', ntypes, lammps.inter_deepmd, - model_name) + model_param) else : fc = lammps.make_lammps_equi('conf.lmp', ntypes, lammps.inter_deepmd, - model_name, + model_param, change_box = relax_box) elif task_type =='meam': if static : diff --git a/dpgen/auto_test/gen_06_phonon.py b/dpgen/auto_test/gen_06_phonon.py index 2b1e2af14..09e9ef9b3 100644 --- a/dpgen/auto_test/gen_06_phonon.py +++ b/dpgen/auto_test/gen_06_phonon.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import os, re, argparse, filecmp, json, glob +import os, re, argparse, filecmp, json, glob, shutil import subprocess as sp import numpy as np import dpgen.auto_test.lib.vasp as vasp @@ -8,6 +8,10 @@ from phonopy.structure.atoms import PhonopyAtoms import yaml +from dpgen import ROOT_PATH +from pymatgen.io.vasp import Incar +from dpgen.generator.lib.vasp import incar_upper +cvasp_file=os.path.join(ROOT_PATH,'generator/lib/cvasp.py') global_equi_name = '00.equi' @@ -117,9 +121,10 @@ def make_vasp(jdata, conf_dir) : user_incar_path = jdata['user_incar'] assert(os.path.exists(user_incar_path)) user_incar_path = os.path.abspath(user_incar_path) - fc = open(user_incar_path).read() - kspacing =float(re.findall((r"KSPACING(.+?)\n"),fc)[0].replace('=','')) - kgamma =('T' in re.findall((r"KGAMMA(.+?)\n"),fc)[0]) + incar = incar_upper(Incar.from_file(user_incar_path)) + fc = incar.get_string() + kspacing = incar['KSPACING'] + kgamma = incar['KGAMMA'] else : fp_params = jdata['vasp_params'] ecut = fp_params['ecut'] @@ -145,9 +150,17 @@ def make_vasp(jdata, conf_dir) : with open(fname) as infile: outfile.write(infile.read()) # gen kpoints - fc = vasp.make_kspacing_kpoints(task_poscar, kspacing, kgamma) - with open(os.path.join(task_path,'KPOINTS'), 'w') as fp: - fp.write(fc) +# fc = vasp.make_kspacing_kpoints(task_poscar, kspacing, kgamma) +# with open(os.path.join(task_path,'KPOINTS'), 'w') as fp: +# fp.write(fc) + + # write kp + fc = vasp.make_kspacing_kpoints('POSCAR', kspacing, kgamma) + with open('KPOINTS', 'w') as fp: fp.write(fc) + #copy cvasp + if ('cvasp' in jdata) and (jdata['cvasp'] == True): + shutil.copyfile(cvasp_file, os.path.join(task_path,'cvasp.py')) + # gen band.conf os.chdir(task_path) with open('band.conf','w') as fp: @@ -169,14 +182,16 @@ def make_lammps(jdata, conf_dir,task_type) : type_map = fp_params['type_map'] model_dir = os.path.abspath(model_dir) model_name =fp_params['model_name'] + deepmd_version = fp_params.get("deepmd_version", "0.12") if not model_name : models = glob.glob(os.path.join(model_dir, '*pb')) model_name = [os.path.basename(ii) for ii in models] else: models = [os.path.join(model_dir,ii) for ii in model_name] - model_param = {'model_name' : fp_params['model_name'], - 'param_type': fp_params['model_param_type']} + model_param = {'model_name' : model_name, + 'param_type': fp_params['model_param_type'], + 'deepmd_version' : deepmd_version} supercell_matrix=jdata['supercell_matrix'] band_path=jdata['band'] @@ -214,7 +229,7 @@ def make_lammps(jdata, conf_dir,task_type) : fc = lammps.make_lammps_phonon('conf.lmp', unitcell.masses, lammps.inter_deepmd, - model_name) + model_param) if task_type=='meam': fc = lammps.make_lammps_phonon('conf.lmp', unitcell.masses, diff --git a/dpgen/auto_test/gen_07_SScurve.py b/dpgen/auto_test/gen_07_SScurve.py index c97f3c733..557aaccf3 100644 --- a/dpgen/auto_test/gen_07_SScurve.py +++ b/dpgen/auto_test/gen_07_SScurve.py @@ -104,15 +104,16 @@ def make_lammps(jdata, conf_dir,task_type) : type_map = fp_params['type_map'] model_dir = os.path.abspath(model_dir) model_name =fp_params['model_name'] + deepmd_version = fp_params.get("deepmd_version", "0.12") if not model_name : models = glob.glob(os.path.join(model_dir, '*pb')) model_name = [os.path.basename(ii) for ii in models] else: models = [os.path.join(model_dir,ii) for ii in model_name] - model_param = {'model_name' : fp_params['model_name'], - 'param_type': fp_params['model_param_type']} - + model_param = {'model_name' : model_name, + 'param_type': fp_params['model_param_type'], + 'deepmd_version' : deepmd_version} ntypes = len(type_map) strain_start=jdata['strain_start'] strain_end=jdata['strain_end'] @@ -153,7 +154,7 @@ def make_lammps(jdata, conf_dir,task_type) : fc = lammps.make_lammps_elastic('conf.lmp', ntypes, lammps.inter_deepmd, - model_name) + model_param) elif task_type =='meam': fc = lammps.make_lammps_elastic('conf.lmp', ntypes, diff --git a/dpgen/auto_test/lib/lammps.py b/dpgen/auto_test/lib/lammps.py index b6a21f44a..aafc35e21 100644 --- a/dpgen/auto_test/lib/lammps.py +++ b/dpgen/auto_test/lib/lammps.py @@ -4,6 +4,7 @@ import dpdata import subprocess as sp import dpgen.auto_test.lib.util as util +from distutils.version import LooseVersion def cvt_lammps_conf(fin, fout, ofmt = 'lammps/data'): """ @@ -89,16 +90,26 @@ def _get_conf_natom(conf) : return int(ii.split()[0]) raise RuntimeError("cannot find line indicate atom types in ", conf) -def inter_deepmd(models) : - ret = "" - line = "pair_style deepmd " - if len(models) > 1 : - for ii in models : - line += ii + ' ' - line += ' 10 model_devi.out\npair_coeff\n' - else : - line += models[0] + '\npair_coeff\n' - ret += line +def inter_deepmd(param) : + models = param["model_name"] + deepmd_version = param["deepmd_version"] + ret = "pair_style deepmd " + model_list = "" + for ii in models: + model_list += ii + " " + if LooseVersion(deepmd_version) < LooseVersion('1'): + ## DeePMD-kit version == 0.x + if len(models) > 1 : + ret += '%s 10 model_devi.out\n' % model_list + else : + ret += models[0] + '\n' + else: + ## DeePMD-kit version >= 1 + if len(models) > 1: + ret += "%s out_freq 10 out_file model_devi.out\n" % model_list + else: + ret += models[0] + '\n' + ret += "pair_coeff\n" return ret def inter_meam(param) : diff --git a/dpgen/auto_test/lib/util.py b/dpgen/auto_test/lib/util.py index bbb8d438b..06fd27208 100644 --- a/dpgen/auto_test/lib/util.py +++ b/dpgen/auto_test/lib/util.py @@ -1,7 +1,7 @@ import numpy as np import requests import os,re -from dpgen.remote.RemoteJob import SSHSession +from dpgen import dlog from dpgen.auto_test.lib import vasp from dpgen.auto_test.lib import lammps from dpgen.auto_test.lib.utils import cmd_append_log @@ -60,6 +60,7 @@ def make_work_path(jdata,task,reprod_opt,static,user): if 'relax_incar' in jdata.keys(): task_type=task_type+'-reprod-relax_incar' else: + kspacing = jdata['vasp_params']['kspacing'] task_type=task_type+'-reprod-k%.2f'% (kspacing) work_path=os.path.join(task_path, task_type) @@ -73,7 +74,6 @@ def get_machine_info(mdata,task_type): group_size = mdata['fp_group_size'] resources = mdata['fp_resources'] machine=mdata['fp_machine'] - machine_type = mdata['fp_machine']['machine_type'] command = vasp_exec command = cmd_append_log(command, "log") elif task_type in lammps_task_type: @@ -81,11 +81,9 @@ def get_machine_info(mdata,task_type): group_size = mdata['model_devi_group_size'] resources = mdata['model_devi_resources'] machine=mdata['model_devi_machine'] - machine_type = mdata['model_devi_machine']['machine_type'] command = lmp_exec + " -i lammps.in" command = cmd_append_log(command, "model_devi.log") - ssh_sess = SSHSession(machine) - return machine, machine_type,ssh_sess,resources, command, group_size + return machine, resources, command, group_size def collect_task(all_task,task_type): diff --git a/dpgen/auto_test/lib/vasp.py b/dpgen/auto_test/lib/vasp.py index f2610de82..2e234b3c1 100644 --- a/dpgen/auto_test/lib/vasp.py +++ b/dpgen/auto_test/lib/vasp.py @@ -1,9 +1,11 @@ #!/usr/bin/python3 - +import os import warnings import numpy as np import dpgen.auto_test.lib.lammps as lammps import dpgen.auto_test.lib.util as util +from dpgen.generator.lib.vasp import incar_upper +from pymatgen.io.vasp import Incar,Kpoints,Potcar class OutcarItemError(Exception): pass @@ -102,6 +104,8 @@ def reciprocal_box(box) : return rbox def make_kspacing_kpoints(poscar, kspacing, kgamma) : + if type(kspacing) is not list: + kspacing = [kspacing, kspacing, kspacing] with open(poscar, 'r') as fp: lines = fp.read().split('\n') scale = float(lines[1]) @@ -111,7 +115,7 @@ def make_kspacing_kpoints(poscar, kspacing, kgamma) : box = np.array(box) box *= scale rbox = reciprocal_box(box) - kpoints = [(np.ceil(2 * np.pi * np.linalg.norm(ii) / kspacing).astype(int)) for ii in rbox] + kpoints = [max(1,(np.ceil(2 * np.pi * np.linalg.norm(ii) / ks).astype(int))) for ii,ks in zip(rbox,kspacing)] ret = make_vasp_kpoints(kpoints, kgamma) return ret @@ -247,7 +251,7 @@ def make_vasp_static_incar (ecut, ediff, ret += 'PREC=A\n' ret += 'ENCUT=%d\n' % ecut ret += '# ISYM=0\n' - ret += 'ALGO=fast\n' + ret += 'ALGO=normal\n' ret += 'EDIFF=%e\n' % ediff ret += 'EDIFFG=-0.01\n' ret += 'LREAL=A\n' @@ -288,7 +292,7 @@ def make_vasp_relax_incar (ecut, ediff, ret += 'PREC=A\n' ret += 'ENCUT=%d\n' % ecut ret += '# ISYM=0\n' - ret += 'ALGO=fast\n' + ret += 'ALGO=normal\n' ret += 'EDIFF=%e\n' % ediff ret += 'EDIFFG=-0.01\n' ret += 'LREAL=A\n' @@ -329,7 +333,7 @@ def make_vasp_phonon_incar (ecut, ediff, ret += 'PREC=A\n' ret += 'ENCUT=%d\n' % ecut ret += '# ISYM=0\n' - ret += 'ALGO=fast\n' + ret += 'ALGO=normal\n' ret += 'EDIFF=%e\n' % ediff ret += 'EDIFFG=-0.01\n' ret += 'LREAL=A\n' @@ -456,3 +460,38 @@ def make_vasp_kpoints (kpoints, kgamma = False) : ret = _make_vasp_kp_mp(kpoints) return ret + +def make_vasp_kpoints_from_incar(work_dir,jdata): + cwd=os.getcwd() + fp_aniso_kspacing = jdata.get('fp_aniso_kspacing') + os.chdir(work_dir) + # get kspacing and kgamma from incar + assert(os.path.exists('INCAR')) + with open('INCAR') as fp: + incar = fp.read() + standard_incar = incar_upper(Incar.from_string(incar)) + if fp_aniso_kspacing is None: + try: + kspacing = standard_incar['KSPACING'] + except KeyError: + raise RuntimeError ("KSPACING must be given in INCAR") + else : + kspacing = fp_aniso_kspacing + try: + gamma = standard_incar['KGAMMA'] + if isinstance(gamma,bool): + pass + else: + if gamma[0].upper()=="T": + gamma=True + else: + gamma=False + except KeyError: + raise RuntimeError ("KGAMMA must be given in INCAR") + # check poscar + assert(os.path.exists('POSCAR')) + # make kpoints + ret=make_kspacing_kpoints('POSCAR', kspacing, gamma) + kp=Kpoints.from_string(ret) + kp.write_file("KPOINTS") + os.chdir(cwd) diff --git a/dpgen/auto_test/run.py b/dpgen/auto_test/run.py index a44206909..9c5380893 100644 --- a/dpgen/auto_test/run.py +++ b/dpgen/auto_test/run.py @@ -35,15 +35,13 @@ from dpgen.auto_test.lib.utils import log_iter from dpgen.auto_test.lib.pwscf import make_pwscf_input from dpgen.auto_test.lib.siesta import make_siesta_input -from dpgen.remote.RemoteJob import SSHSession, JobStatus, SlurmJob, PBSJob, CloudMachineJob -from dpgen.remote.decide_machine import decide_fp_machine, decide_model_devi_machine -from dpgen.remote.group_jobs import * from dpgen.auto_test import gen_00_equi,cmpt_00_equi from dpgen.auto_test import gen_01_eos,cmpt_01_eos from dpgen.auto_test import gen_02_elastic,cmpt_02_elastic from dpgen.auto_test import gen_03_vacancy,cmpt_03_vacancy from dpgen.auto_test import gen_04_interstitial,cmpt_04_interstitial from dpgen.auto_test import gen_05_surf,cmpt_05_surf +from dpgen.remote.decide_machine import decide_fp_machine, decide_model_devi_machine #from dpgen.auto_test import gen_06_phonon,cmpt_06_phonon from dpgen.auto_test import gen_confs import requests @@ -74,8 +72,11 @@ def run_equi(task_type,jdata,mdata): if task_type=="vasp": mdata=decide_fp_machine(mdata) - forward_files = ['INCAR', 'POTCAR'] - backward_files = ['OUTCAR', 'autotest.out' , 'CONTCAR','OSZICAR'] + forward_files = ['INCAR', 'POTCAR', 'KPOINTS'] + if ('cvasp' in jdata) and (jdata['cvasp'] == True): + mdata['fp_resources']['cvasp'] = True + forward_files.append('cvasp.py') + backward_files = ['OUTCAR', task_type+'.out' , 'CONTCAR','OSZICAR'] common_files=['POSCAR'] #lammps @@ -83,7 +84,7 @@ def run_equi(task_type,jdata,mdata): mdata = decide_model_devi_machine(mdata) forward_files = ['conf.lmp', 'lammps.in'] - backward_files = ['dump.relax','log.lammps', 'autotest.out'] + backward_files = ['dump.relax','log.lammps', task_type+'.out'] fp_params = jdata['lammps_params'] model_dir = fp_params['model_dir'] @@ -104,8 +105,12 @@ def run_equi(task_type,jdata,mdata): run_tasks = util.collect_task(all_task,task_type) if len(run_tasks)==0: return - machine,machine_type,ssh_sess,resources,command,group_size=util.get_machine_info(mdata,task_type) - disp = make_dispatcher(machine) + machine,resources,command,group_size=util.get_machine_info(mdata,task_type) + disp = make_dispatcher(machine, resources, work_path, run_tasks, group_size) + #debug# + #print(' '.join(common_files)) + #print(' '.join(forward_files)) + #print(' '.join(backward_files)) disp.run_jobs(resources, command, work_path, @@ -114,8 +119,8 @@ def run_equi(task_type,jdata,mdata): common_files, forward_files, backward_files, - outlog='autotest.out', - errlog='autotest.err') + outlog=task_type+'.out', + errlog=task_type+'.err') def cmpt_equi(task_type,jdata,mdata): @@ -163,9 +168,12 @@ def run_eos(task_type,jdata,mdata): if task_type=="vasp": mdata=decide_fp_machine(mdata) - forward_files = ['INCAR', 'POSCAR','POTCAR'] - backward_files = ['OUTCAR', 'autotest.out' , 'OSZICAR'] + forward_files = ['INCAR', 'POSCAR','POTCAR','KPOINTS'] + backward_files = ['OUTCAR', task_type+'.out' , 'OSZICAR'] common_files=['INCAR','POTCAR'] + if ('cvasp' in jdata) and (jdata['cvasp'] == True): + mdata['fp_resources']['cvasp'] = True + forward_files.append('cvasp.py') #lammps elif task_type in lammps_task_type: @@ -181,7 +189,7 @@ def run_eos(task_type,jdata,mdata): else: models = [os.path.join(model_dir,ii) for ii in model_name] forward_files = ['conf.lmp', 'lammps.in']+model_name - backward_files = ['log.lammps', 'autotest.out'] + backward_files = ['log.lammps', task_type+'.out'] common_files=['lammps.in']+model_name if len(model_name)>1 and task_type == 'deepmd': @@ -192,8 +200,8 @@ def run_eos(task_type,jdata,mdata): run_tasks = util.collect_task(all_task,task_type) if len(run_tasks)==0: return - machine,machine_type,ssh_sess,resources,command,group_size=util.get_machine_info(mdata,task_type) - disp = make_dispatcher(machine) + machine,resources,command,group_size=util.get_machine_info(mdata,task_type) + disp = make_dispatcher(machine, resources, work_path, run_tasks, group_size) disp.run_jobs(resources, command, work_path, @@ -202,8 +210,8 @@ def run_eos(task_type,jdata,mdata): common_files, forward_files, backward_files, - outlog='autotest.out', - errlog='autotest.err') + outlog=task_type+'.out', + errlog=task_type+'.err') def cmpt_eos(task_type,jdata,mdata): conf_dir=jdata['conf_dir'] @@ -240,8 +248,11 @@ def run_elastic(task_type,jdata,mdata): mdata=decide_fp_machine(mdata) forward_files = ['INCAR', 'POSCAR','POTCAR','KPOINTS'] - backward_files = ['OUTCAR', 'autotest.out' , 'CONTCAR','OSZICAR'] + backward_files = ['OUTCAR', task_type+'.out' , 'CONTCAR','OSZICAR'] common_files=['INCAR','POTCAR','KPOINTS'] + if ('cvasp' in jdata) and (jdata['cvasp'] == True): + mdata['fp_resources']['cvasp'] = True + forward_files.append('cvasp.py') #lammps elif task_type in lammps_task_type: @@ -257,7 +268,7 @@ def run_elastic(task_type,jdata,mdata): else: models = [os.path.join(model_dir,ii) for ii in model_name] forward_files = ['conf.lmp', 'lammps.in','strain.out']+model_name - backward_files = ['log.lammps', 'autotest.out'] + backward_files = ['log.lammps', task_type+'.out'] common_files=['lammps.in']+model_name if len(model_name)>1 and task_type == 'deepmd': @@ -268,8 +279,8 @@ def run_elastic(task_type,jdata,mdata): run_tasks = util.collect_task(all_task,task_type) if len(run_tasks)==0: return - machine,machine_type,ssh_sess,resources,command,group_size=util.get_machine_info(mdata,task_type) - disp = make_dispatcher(machine) + machine,resources,command,group_size=util.get_machine_info(mdata,task_type) + disp = make_dispatcher(machine, resources, work_path, run_tasks, group_size) disp.run_jobs(resources, command, work_path, @@ -278,8 +289,8 @@ def run_elastic(task_type,jdata,mdata): common_files, forward_files, backward_files, - outlog='autotest.out', - errlog='autotest.err') + outlog=task_type+'.out', + errlog=task_type+'.err') def cmpt_elastic(task_type,jdata,mdata): conf_dir=jdata['conf_dir'] @@ -313,9 +324,12 @@ def run_vacancy(task_type,jdata,mdata): if task_type == "vasp": mdata=decide_fp_machine(mdata) - forward_files = ['INCAR', 'POSCAR','POTCAR'] - backward_files = ['OUTCAR', 'autotest.out' , 'OSZICAR'] + forward_files = ['INCAR', 'POSCAR','POTCAR','KPOINTS'] + backward_files = ['OUTCAR', task_type+'.out' , 'OSZICAR'] common_files=['INCAR','POTCAR'] + if ('cvasp' in jdata) and (jdata['cvasp'] == True): + mdata['fp_resources']['cvasp'] = True + forward_files.append('cvasp.py') #lammps elif task_type in lammps_task_type: @@ -332,7 +346,7 @@ def run_vacancy(task_type,jdata,mdata): models = [os.path.join(model_dir,ii) for ii in model_name] common_files = model_name forward_files = ['conf.lmp', 'lammps.in']+model_name - backward_files = ['log.lammps','autotest.out'] + backward_files = ['log.lammps',task_type+'.out'] common_files=['lammps.in']+model_name if len(model_name)>1 and task_type == 'deepmd': @@ -343,8 +357,8 @@ def run_vacancy(task_type,jdata,mdata): run_tasks = util.collect_task(all_task,task_type) if len(run_tasks)==0: return - machine,machine_type,ssh_sess,resources,command,group_size=util.get_machine_info(mdata,task_type) - disp = make_dispatcher(machine) + machine,resources,command,group_size=util.get_machine_info(mdata,task_type) + disp = make_dispatcher(machine, resources, work_path, run_tasks, group_size) disp.run_jobs(resources, command, work_path, @@ -353,8 +367,8 @@ def run_vacancy(task_type,jdata,mdata): common_files, forward_files, backward_files, - outlog='autotest.out', - errlog='autotest.err') + outlog=task_type+'.out', + errlog=task_type+'.err') def cmpt_vacancy(task_type,jdata,mdata): conf_dir=jdata['conf_dir'] @@ -397,9 +411,12 @@ def run_interstitial(task_type,jdata,mdata): if task_type == "vasp": mdata=decide_fp_machine(mdata) - forward_files = ['INCAR', 'POSCAR','POTCAR'] - backward_files = ['OUTCAR', 'autotest.out' , 'XDATCAR','OSZICAR'] + forward_files = ['INCAR', 'POSCAR','POTCAR',"KPOINTS"] + backward_files = ['OUTCAR', task_type+'.out' , 'XDATCAR','OSZICAR'] common_files=['INCAR'] + if ('cvasp' in jdata) and (jdata['cvasp'] == True): + mdata['fp_resources']['cvasp'] = True + forward_files.append('cvasp.py') #lammps elif task_type in lammps_task_type: @@ -431,7 +448,7 @@ def run_interstitial(task_type,jdata,mdata): else: models = [os.path.join(model_dir,ii) for ii in model_name] forward_files = ['conf.lmp', 'lammps.in']+model_name - backward_files = ['log.lammps', 'autotest.out'] + backward_files = ['log.lammps', task_type+'.out'] common_files=['lammps.in']+model_name if len(model_name)>1 and task_type == 'deepmd': @@ -440,15 +457,14 @@ def run_interstitial(task_type,jdata,mdata): else: raise RuntimeError ("unknow task %s, something wrong" % task_type) - machine,machine_type,ssh_sess,resources,command,group_size=util.get_machine_info(mdata,task_type) - disp = make_dispatcher(machine) + machine,resources,command,group_size=util.get_machine_info(mdata,task_type) if reprod_opt: for ii in work_path: run_tasks=[] for jj in run_tasks_: if ii in jj: run_tasks.append(os.path.basename(jj)) - + disp = make_dispatcher(machine, resources, work_path, run_tasks, group_size) disp.run_jobs(resources, command, ii, @@ -457,11 +473,12 @@ def run_interstitial(task_type,jdata,mdata): common_files, forward_files, backward_files, - outlog='autotest.out', - errlog='autotest.err') + outlog=task_type+'.out', + errlog=task_type+'.err') else: run_tasks = util.collect_task(all_task,task_type) if len(run_tasks)==0: return + disp = make_dispatcher(machine, resources, work_path, run_tasks, group_size) disp.run_jobs(resources, command, work_path, @@ -470,8 +487,8 @@ def run_interstitial(task_type,jdata,mdata): common_files, forward_files, backward_files, - outlog='autotest.out', - errlog='autotest.err') + outlog=task_type+'.out', + errlog=task_type+'.err') def cmpt_interstitial(task_type,jdata,mdata): conf_dir=jdata['conf_dir'] @@ -519,9 +536,12 @@ def run_surf(task_type,jdata,mdata): if task_type == "vasp": mdata=decide_fp_machine(mdata) - forward_files = ['INCAR', 'POSCAR','POTCAR'] - backward_files = ['OUTCAR', 'autotest.out' , 'OSZICAR'] + forward_files = ['INCAR', 'POSCAR','POTCAR','KPOINTS'] + backward_files = ['OUTCAR', task_type+'.out' , 'OSZICAR'] common_files=['INCAR','POTCAR'] + if ('cvasp' in jdata) and (jdata['cvasp'] == True): + mdata['fp_resources']['cvasp'] = True + forward_files.append('cvasp.py') #lammps elif task_type in lammps_task_type: @@ -537,7 +557,7 @@ def run_surf(task_type,jdata,mdata): else: models = [os.path.join(model_dir,ii) for ii in model_name] forward_files = ['conf.lmp', 'lammps.in']+model_name - backward_files = ['log.lammps','autotest.out'] + backward_files = ['log.lammps',task_type+'.out'] common_files=['lammps.in']+model_name if len(model_name)>1 and task_type == 'deepmd': @@ -548,8 +568,8 @@ def run_surf(task_type,jdata,mdata): run_tasks = util.collect_task(all_task,task_type) if len(run_tasks)==0: return - machine,machine_type,ssh_sess,resources,command,group_size=util.get_machine_info(mdata,task_type) - disp = make_dispatcher(machine) + machine,resources,command,group_size=util.get_machine_info(mdata,task_type) + disp = make_dispatcher(machine, resources, work_path, run_tasks, group_size) disp.run_jobs(resources, command, work_path, @@ -558,8 +578,8 @@ def run_surf(task_type,jdata,mdata): common_files, forward_files, backward_files, - outlog='autotest.out', - errlog='autotest.err') + outlog=task_type+'.out', + errlog=task_type+'.err') def cmpt_surf(task_type,jdata,mdata): conf_dir=jdata['conf_dir'] @@ -601,14 +621,17 @@ def run_phonon(task_type,jdata,mdata): #vasp if task_type == "vasp": mdata=decide_fp_machine(mdata) - machine,machine_type,ssh_sess,resources,command,group_size=util.get_machine_info(mdata,task_type) + machine,resources,command,group_size=util.get_machine_info(mdata,task_type) run_tasks = util.collect_task(all_task,task_type) - forward_files = ['INCAR', 'POTCAR','KPOINTS'] - backward_files = ['OUTCAR', 'autotest.out' , 'OSZICAR','vasprun.xml'] + forward_files = ['INCAR', 'POTCAR','KPOINTS','KPOINTS'] + backward_files = ['OUTCAR', task_type+'.out' , 'OSZICAR','vasprun.xml'] common_files=['POSCAR'] + if ('cvasp' in jdata) and (jdata['cvasp'] == True): + mdata['fp_resources']['cvasp'] = True + forward_files.append('cvasp.py') - disp = make_dispatcher(machine) + disp = make_dispatcher(machine, resources, work_path, run_tasks, group_size) disp.run_jobs(resources, command, work_path, @@ -617,13 +640,13 @@ def run_phonon(task_type,jdata,mdata): common_files, forward_files, backward_files, - outlog='autotest.out', - errlog='autotest.err') + outlog=task_type+'.out', + errlog=task_type+'.err') #lammps elif task_type in lammps_task_type: None else: - raise RuntimeError ("unknow task %s, something wrong" % task_type) + raise RuntimeError ("unknown task %s, something wrong" % task_type) def cmpt_phonon(task_type,jdata,mdata): conf_dir=jdata['conf_dir'] @@ -652,10 +675,29 @@ def run_task (json_file, machine_file) : ii = jdata['task_type'] jj=jdata['task'] - task_list=['equi','eos','elastic','vacancy','interstitial','surf','phonon','all'] + task_list=['equi','eos','elastic','vacancy','interstitial','surf','phonon'] + + if isinstance(jj,str): + if jj=='all': + all_task_list=task_list.copy() + else: + try: + assert jj in task_list + except AssertionError: + raise RuntimeError ("unknow task %s, something wrong" % jj) + all_task_list=[jj] + elif isinstance(jj,list): + try: + assert set(jj).issubset(set(task_list)) + except AssertionError: + raise RuntimeError ("unknow task %s, some tasks may not supported" % ' '.join(jj)) + all_task_list=jj.copy() + else: + raise RuntimeError ('unknow format for task, it must be a string or list') + task_type_list=['vasp']+lammps_task_type - if jj not in task_list : - raise RuntimeError ("unknow task %s, something wrong" % jj) + #if jj not in task_list : + # raise RuntimeError ("unknow task %s, something wrong" % jj) if ii not in task_type_list : raise RuntimeError ("unknow task type %s, something wrong" % ii) @@ -673,51 +715,52 @@ def run_task (json_file, machine_file) : run_equi (ii, jdata, mdata) log_iter ("cmpt_equi", ii,"equi") cmpt_equi (ii, jdata, mdata) - if jj == "eos" or jj=="all": - log_iter ("gen_eos", ii, "eos") - gen_eos (ii, jdata, mdata) - log_iter ("run_eos", ii, "eos") - run_eos (ii, jdata, mdata) - log_iter ("cmpt_eos", ii, "eos") - cmpt_eos (ii, jdata, mdata) - if jj=="elastic" or jj=="all": - log_iter ("gen_elastic", ii, "elastic") - gen_elastic (ii, jdata, mdata) - log_iter ("run_elastic", ii, "elastic") - run_elastic (ii, jdata, mdata) - log_iter ("cmpt_elastic", ii, "elastic") - cmpt_elastic (ii, jdata, mdata) - if jj=="vacancy" or jj=="all": - log_iter ("gen_vacancy", ii, "vacancy") - gen_vacancy (ii, jdata, mdata) - log_iter ("run_vacancy", ii, "vacancy") - run_vacancy (ii, jdata, mdata) - log_iter ("cmpt_vacancy", ii, "vacancy") - cmpt_vacancy (ii, jdata, mdata) - if jj=="interstitial" or jj=="all": - log_iter ("gen_interstitial", ii, "interstitial") - gen_interstitial (ii, jdata, mdata) - log_iter ("run_interstitial", ii, "interstitial") - run_interstitial (ii, jdata, mdata) - log_iter ("cmpt_interstitial", ii, "interstitial") - cmpt_interstitial (ii, jdata, mdata) - if jj=="surf" or jj=="all": - log_iter ("gen_surf", ii, "surf") - gen_surf (ii, jdata, mdata) - log_iter ("run_surf", ii, "surf") - run_surf (ii, jdata, mdata) - log_iter ("cmpt_surf", ii, "surf") - cmpt_surf (ii, jdata, mdata) - ''' - if jj=="phonon": - log_iter ("gen_phonon", ii, "phonon") - gen_phonon (ii, jdata, mdata) - log_iter ("run_phonon", ii, "phonon") - run_phonon (ii, jdata, mdata) - log_iter ("cmpt_phonon", ii, "phonon") - cmpt_phonon (ii, jdata, mdata) - ''' - record_iter (record, confs, ii, jj) + for jj in all_task_list: + if jj == "eos": + log_iter ("gen_eos", ii, "eos") + gen_eos (ii, jdata, mdata) + log_iter ("run_eos", ii, "eos") + run_eos (ii, jdata, mdata) + log_iter ("cmpt_eos", ii, "eos") + cmpt_eos (ii, jdata, mdata) + if jj=="elastic": + log_iter ("gen_elastic", ii, "elastic") + gen_elastic (ii, jdata, mdata) + log_iter ("run_elastic", ii, "elastic") + run_elastic (ii, jdata, mdata) + log_iter ("cmpt_elastic", ii, "elastic") + cmpt_elastic (ii, jdata, mdata) + if jj=="vacancy": + log_iter ("gen_vacancy", ii, "vacancy") + gen_vacancy (ii, jdata, mdata) + log_iter ("run_vacancy", ii, "vacancy") + run_vacancy (ii, jdata, mdata) + log_iter ("cmpt_vacancy", ii, "vacancy") + cmpt_vacancy (ii, jdata, mdata) + if jj=="interstitial": + log_iter ("gen_interstitial", ii, "interstitial") + gen_interstitial (ii, jdata, mdata) + log_iter ("run_interstitial", ii, "interstitial") + run_interstitial (ii, jdata, mdata) + log_iter ("cmpt_interstitial", ii, "interstitial") + cmpt_interstitial (ii, jdata, mdata) + if jj=="surf": + log_iter ("gen_surf", ii, "surf") + gen_surf (ii, jdata, mdata) + log_iter ("run_surf", ii, "surf") + run_surf (ii, jdata, mdata) + log_iter ("cmpt_surf", ii, "surf") + cmpt_surf (ii, jdata, mdata) + ''' + if jj=="phonon": + log_iter ("gen_phonon", ii, "phonon") + gen_phonon (ii, jdata, mdata) + log_iter ("run_phonon", ii, "phonon") + run_phonon (ii, jdata, mdata) + log_iter ("cmpt_phonon", ii, "phonon") + cmpt_phonon (ii, jdata, mdata) + ''' + record_iter (record, confs, ii, jj) def gen_test(args): logging.info ("start auto-testing") diff --git a/dpgen/collect/__init__.py b/dpgen/collect/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dpgen/collect/collect.py b/dpgen/collect/collect.py new file mode 100644 index 000000000..a8d5b4060 --- /dev/null +++ b/dpgen/collect/collect.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 + +import os,sys,json,glob,argparse,dpdata +import numpy as np +from dpgen.generator.run import data_system_fmt + +def collect_data(target_folder, param_file, output, + verbose = True, + shuffle = True, + merge = True) : + target_folder = os.path.abspath(target_folder) + output = os.path.abspath(output) + # goto input + cwd = os.getcwd() + os.chdir(target_folder) + jdata = json.load(open(param_file)) + sys_configs_prefix = jdata.get('sys_configs_prefix', '') + sys_configs = jdata.get('sys_configs', []) + if verbose : + max_str_len = max([len(str(ii)) for ii in sys_configs]) + max_form_len = 16 + ptr_fmt = '%%%ds %%%ds natoms %%6d nframes %%6d' % (max_str_len+5, max_form_len) + # init systems + init_data = [] + init_data_prefix = jdata.get('init_data_prefix', '') + init_data_sys = jdata.get('init_data_sys', []) + for ii in init_data_sys: + init_data.append(dpdata.LabeledSystem(os.path.join(init_data_prefix, ii), fmt='deepmd/npy')) + # collect systems from iter dirs + coll_data = {} + numb_sys = len(sys_configs) + model_devi_jobs = jdata.get('model_devi_jobs', {}) + numb_jobs = len(model_devi_jobs) + iters = ['iter.%06d' % ii for ii in range(numb_jobs)] + # loop over iters to collect data + for ii in range(len(iters)) : + iter_data = glob.glob(os.path.join(iters[ii], '02.fp', 'data.[0-9]*[0-9]')) + iter_data.sort() + for jj in iter_data : + sys = dpdata.LabeledSystem(jj, fmt = 'deepmd/npy') + if merge: + sys_str = sys.formula + else: + sys_str = (os.path.basename(jj).split('.')[-1]) + if sys_str in coll_data.keys(): + coll_data[sys_str].append(sys) + else: + coll_data[sys_str] = sys + # print information + if verbose: + for ii in range(len(init_data)): + print(ptr_fmt % (str(init_data_sys[ii]), + init_data[ii].formula, + init_data[ii].get_natoms(), + init_data[ii].get_nframes() )) + keys = list(coll_data.keys()) + keys.sort() + for ii in keys: + if merge: + sys_str = ii + else : + sys_str = str(sys_configs[int(ii)]) + print(ptr_fmt % (sys_str, + coll_data[ii].formula, + coll_data[ii].get_natoms(), + coll_data[ii].get_nframes() )) + # shuffle system data + if shuffle: + for kk in coll_data.keys(): + coll_data[kk].shuffle() + # create output dir + os.chdir(cwd) + os.makedirs(output, exist_ok = True) + # dump init data + for idx,ii in enumerate(init_data): + out_dir = 'init.' + (data_system_fmt % idx) + ii.to('deepmd/npy', os.path.join(output, out_dir)) + # dump iter data + for kk in coll_data.keys(): + out_dir = 'sys.%s' % kk + nframes = coll_data[kk].get_nframes() + coll_data[kk].to('deepmd/npy', os.path.join(output, out_dir), set_size = nframes) + +def gen_collect(args): + collect_data(args.JOB_DIR, args.parameter, args.OUTPUT, + verbose = args.verbose, + shuffle = args.shuffle, + merge = args.merge) + +def _main() : + parser = argparse.ArgumentParser(description='Collect data from DP-GEN iterations') + parser.add_argument("JOB_DIR", type=str, + help="the directory of the DP-GEN job") + parser.add_argument("OUTPUT", type=str, + help="the output directory of data") + parser.add_argument('-p',"--parameter", type=str, default = 'param.json', + help="the json file provides DP-GEN paramters, should be located in JOB_DIR") + parser.add_argument('-v',"--verbose", action = 'store_true', + help="print number of data in each system") + parser.add_argument('-m',"--merge", action = 'store_true', + help="merge the systems with the same chemical formula") + parser.add_argument('-s',"--shuffle", action = 'store_true', + help="shuffle the data systems") + args = parser.parse_args() + gen_collect(args) + +if __name__ == '__main__': + _main() + + diff --git a/dpgen/data/gen.py b/dpgen/data/gen.py index 73bb8e25f..ce1573a61 100644 --- a/dpgen/data/gen.py +++ b/dpgen/data/gen.py @@ -20,25 +20,33 @@ import dpgen.data.tools.bcc as bcc import dpgen.data.tools.diamond as diamond import dpgen.data.tools.sc as sc +from dpgen.generator.lib.vasp import incar_upper from pymatgen import Structure +from pymatgen.io.vasp import Incar from dpgen.remote.decide_machine import decide_fp_machine from dpgen import ROOT_PATH from dpgen.dispatcher.Dispatcher import Dispatcher, make_dispatcher -def create_path (path) : +def create_path (path,back=False) : if path[-1] != "/": path += '/' if os.path.isdir(path) : - dirname = os.path.dirname(path) - counter = 0 - while True : - bk_dirname = dirname + ".bk%03d" % counter - if not os.path.isdir(bk_dirname) : - shutil.move (dirname, bk_dirname) - break - counter += 1 + if back: + dirname = os.path.dirname(path) + counter = 0 + while True : + bk_dirname = dirname + ".bk%03d" % counter + if not os.path.isdir(bk_dirname) : + shutil.move (dirname, bk_dirname) + break + counter += 1 + os.makedirs (path) + return path + else: + return path + os.makedirs (path) return path @@ -238,7 +246,10 @@ def make_super_cell_poscar(jdata) : cwd = os.getcwd() to_file = os.path.abspath(to_file) os.chdir(path_work) - os.symlink(os.path.relpath(to_file), 'POSCAR') + try: + os.symlink(os.path.relpath(to_file), 'POSCAR') + except FileExistsError: + pass os.chdir(cwd) def make_combines (dim, natoms) : @@ -317,9 +328,15 @@ def make_vasp_relax (jdata, mdata) : for ss in sys_list: os.chdir(ss) ln_src = os.path.relpath(os.path.join(work_dir,'INCAR')) - os.symlink(ln_src, 'INCAR') + try: + os.symlink(ln_src, 'INCAR') + except FileExistsError: + pass ln_src = os.path.relpath(os.path.join(work_dir,'POTCAR')) - os.symlink(ln_src, 'POTCAR') + try: + os.symlink(ln_src, 'POTCAR') + except FileExistsError: + pass os.chdir(work_dir) os.chdir(cwd) @@ -452,8 +469,15 @@ def make_vasp_md(jdata) : shutil.copy2 (init_pos, 'POSCAR') file_incar = os.path.join(path_md, 'INCAR') file_potcar = os.path.join(path_md, 'POTCAR') - os.symlink(os.path.relpath(file_incar), 'INCAR') - os.symlink(os.path.relpath(file_potcar), 'POTCAR') + try: + os.symlink(os.path.relpath(file_incar), 'INCAR') + except FileExistsError: + pass + try: + os.symlink(os.path.relpath(file_potcar), 'POTCAR') + except FileExistsError: + pass + os.chdir(cwd) def coll_vasp_md(jdata) : @@ -532,7 +556,7 @@ def _vasp_check_fin (ii) : return False return True -def run_vasp_relax(jdata, mdata, dispatcher): +def run_vasp_relax(jdata, mdata): fp_command = mdata['fp_command'] fp_group_size = mdata['fp_group_size'] fp_resources = mdata['fp_resources'] @@ -557,7 +581,7 @@ def run_vasp_relax(jdata, mdata, dispatcher): # if not _vasp_check_fin(ii): # relax_run_tasks.append(ii) run_tasks = [os.path.basename(ii) for ii in relax_run_tasks] - + dispatcher = make_dispatcher(mdata['fp_machine'], mdata['fp_resources'], work_dir, run_tasks, fp_group_size) #dlog.info(run_tasks) dispatcher.run_jobs(fp_resources, [fp_command], @@ -568,7 +592,7 @@ def run_vasp_relax(jdata, mdata, dispatcher): forward_files, backward_files) -def run_vasp_md(jdata, mdata, dispatcher): +def run_vasp_md(jdata, mdata): fp_command = mdata['fp_command'] fp_group_size = mdata['fp_group_size'] fp_resources = mdata['fp_resources'] @@ -603,7 +627,7 @@ def run_vasp_md(jdata, mdata, dispatcher): run_tasks = [ii.replace(work_dir+"/", "") for ii in md_run_tasks] #dlog.info("md_work_dir", work_dir) #dlog.info("run_tasks",run_tasks) - + dispatcher = make_dispatcher(mdata['fp_machine'], mdata['fp_resources'], work_dir, run_tasks, fp_group_size) dispatcher.run_jobs(fp_resources, [fp_command], work_dir, @@ -631,7 +655,7 @@ def gen_init_bulk(args) : if args.MACHINE is not None: # Selecting a proper machine mdata = decide_fp_machine(mdata) - disp = make_dispatcher(mdata["fp_machine"]) + #disp = make_dispatcher(mdata["fp_machine"]) # Decide work path out_dir = out_dir_name(jdata) @@ -646,15 +670,11 @@ def gen_init_bulk(args) : try: md_incar = jdata['md_incar'] if os.path.isfile(md_incar): - with open(md_incar , "r") as fr: - md_incar_lines = fr.readlines() + standard_incar = incar_upper(Incar.from_file(md_incar)) nsw_flag = False - for incar_line in md_incar_lines: - line = incar_line.split() - if "NSW" in line: + if "NSW" in standard_incar: nsw_flag = True - nsw_steps = int(incar_line.split()[-1]) - break + nsw_steps = standard_incar['NSW'] #dlog.info("nsw_steps is", nsw_steps) #dlog.info("md_nstep_jdata is", md_nstep_jdata) if nsw_flag: @@ -664,7 +684,7 @@ def gen_init_bulk(args) : dlog.info("MD steps in md_incar is %d"%(nsw_steps)) dlog.info("DP-GEN will use settings in md_incar!") jdata['md_nstep'] = nsw_steps - except: + except KeyError: pass ## correct element name temp_elements = [] @@ -688,7 +708,7 @@ def gen_init_bulk(args) : place_element(jdata) if args.MACHINE is not None: make_vasp_relax(jdata, mdata) - run_vasp_relax(jdata, mdata, disp) + run_vasp_relax(jdata, mdata) else: make_vasp_relax(jdata, {"fp_resources":{}}) elif stage == 2 : @@ -699,7 +719,7 @@ def gen_init_bulk(args) : dlog.info("Current stage is 3, run a short md") make_vasp_md(jdata) if args.MACHINE is not None: - run_vasp_md(jdata, mdata, disp) + run_vasp_md(jdata, mdata) elif stage == 4 : dlog.info("Current stage is 4, collect data") coll_vasp_md(jdata) diff --git a/dpgen/data/surf.py b/dpgen/data/surf.py index 78c6f918c..45afc9a91 100644 --- a/dpgen/data/surf.py +++ b/dpgen/data/surf.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import time import os,json,shutil,re,glob,argparse import numpy as np import subprocess as sp @@ -9,17 +10,18 @@ import dpgen.data.tools.sc as sc import dpgen.data.tools.bcc as bcc from dpgen import dlog -import time from dpgen import ROOT_PATH from dpgen.remote.decide_machine import decide_fp_machine -from pymatgen.core.surface import SlabGenerator,generate_all_slabs, Structure -from pymatgen.io.vasp import Poscar from dpgen.dispatcher.Dispatcher import Dispatcher, make_dispatcher -#-----ASE------- +#-----PMG--------- +from pymatgen.io.vasp import Poscar +from pymatgen import Structure,Element from pymatgen.io.ase import AseAtomsAdaptor +#-----ASE------- from ase.io import read from ase.build import general_surface + def create_path (path) : path += '/' if os.path.isdir(path) : @@ -59,6 +61,8 @@ def replace (file_name, pattern, subst) : global_dirname_03 = '01.scale_pert' global_dirname_04 = '02.md' +max_layer_numb = 50 + def out_dir_name(jdata) : super_cell = jdata['super_cell'] @@ -174,7 +178,7 @@ def poscar_scale (poscar_in, poscar_out, scale) : with open(poscar_out, 'w') as fout: fout.write("".join(lines)) -def poscar_elong (poscar_in, poscar_out, elong) : +def poscar_elong (poscar_in, poscar_out, elong, shift_center=True) : with open(poscar_in, 'r') as fin : lines = list(fin) if lines[7][0].upper() != 'C' : @@ -185,8 +189,18 @@ def poscar_elong (poscar_in, poscar_out, elong) : elong_ratio = elong / boxzl boxz = boxz * (1. + elong_ratio) lines[4] = '%.16e %.16e %.16e\n' % (boxz[0],boxz[1],boxz[2]) - with open(poscar_out, 'w') as fout: - fout.write("".join(lines)) + if shift_center: + poscar_str="".join(lines) + st=Structure.from_str(poscar_str,fmt='poscar') + cart_coords=st.cart_coords + z_mean=cart_coords[:,2].mean() + z_shift=st.lattice.c/2-z_mean + cart_coords[:,2]=cart_coords[:,2]+z_shift + nst=Structure(st.lattice,st.species,coords=cart_coords,coords_are_cartesian=True) + nst.to('poscar',poscar_out) + else: + with open(poscar_out, 'w') as fout: + fout.write("".join(lines)) def make_unit_cell (jdata) : @@ -213,6 +227,12 @@ def make_super_cell_pymatgen (jdata) : make_unit_cell(jdata) out_dir = jdata['out_dir'] path_uc = os.path.join(out_dir, global_dirname_02) + + elements=[Element(ii) for ii in jdata['elements']] + if 'vacuum_min' in jdata: + vacuum_min=jdata['vacuum_min'] + else: + vacuum_min=max([float(ii.atomic_radius) for ii in elements]) from_poscar= jdata.get('from_poscar',False) @@ -232,8 +252,14 @@ def make_super_cell_pymatgen (jdata) : all_millers = jdata['millers'] path_sc = os.path.join(out_dir, global_dirname_02) - #z_min = jdata['z_min'] - layer_numb = jdata['layer_numb'] + + user_layer_numb = None # set default value + z_min = None + if 'layer_numb' in jdata: + user_layer_numb = jdata['layer_numb'] + else: + z_min = jdata['z_min'] + super_cell = jdata['super_cell'] cwd = os.getcwd() @@ -247,7 +273,16 @@ def make_super_cell_pymatgen (jdata) : path_cur_surf = create_path('surf-'+miller_str) os.chdir(path_cur_surf) #slabgen = SlabGenerator(ss, miller, z_min, 1e-3) - slab=general_surface.surface(ss,indices=miller,vacuum=1e-3,layers=layer_numb) + if user_layer_numb: + slab=general_surface.surface(ss,indices=miller,vacuum=vacuum_min,layers=user_layer_numb) + else: + # build slab according to z_min value + for layer_numb in range( 1,max_layer_numb+1): + slab=general_surface.surface(ss,indices=miller,vacuum=vacuum_min,layers=layer_numb) + if slab.cell.lengths()[-1] >= z_min: + break + if layer_numb == max_layer_numb: + raise RuntimeError("can't build the required slab") #all_slabs = slabgen.get_slabs() dlog.info(os.getcwd()) #dlog.info("Miller %s: The slab has %s termination, use the first one" %(str(miller), len(all_slabs))) diff --git a/dpgen/dispatcher/ALI.py b/dpgen/dispatcher/ALI.py index 5ceb5ad8d..f6e2c12ce 100644 --- a/dpgen/dispatcher/ALI.py +++ b/dpgen/dispatcher/ALI.py @@ -4,57 +4,387 @@ from aliyunsdkcore.acs_exception.exceptions import ServerException from aliyunsdkecs.request.v20140526.RunInstancesRequest import RunInstancesRequest from aliyunsdkecs.request.v20140526.DeleteInstancesRequest import DeleteInstancesRequest -import time, json, os, glob -from dpgen.dispatcher.Dispatcher import Dispatcher, _split_tasks +from aliyunsdkecs.request.v20140526.DescribeAutoProvisioningGroupInstancesRequest import DescribeAutoProvisioningGroupInstancesRequest +from aliyunsdkecs.request.v20140526.CreateAutoProvisioningGroupRequest import CreateAutoProvisioningGroupRequest +from aliyunsdkecs.request.v20140526.DeleteAutoProvisioningGroupRequest import DeleteAutoProvisioningGroupRequest +from aliyunsdkecs.request.v20140526.ModifyAutoProvisioningGroupRequest import ModifyAutoProvisioningGroupRequest +from aliyunsdkecs.request.v20140526.DeleteLaunchTemplateRequest import DeleteLaunchTemplateRequest +from aliyunsdkvpc.request.v20160428.DescribeVpcsRequest import DescribeVpcsRequest +from aliyunsdkecs.request.v20140526.DescribeLaunchTemplatesRequest import DescribeLaunchTemplatesRequest +from aliyunsdkecs.request.v20140526.CreateLaunchTemplateRequest import CreateLaunchTemplateRequest +from aliyunsdkecs.request.v20140526.DescribeImagesRequest import DescribeImagesRequest +from aliyunsdkecs.request.v20140526.DescribeSecurityGroupsRequest import DescribeSecurityGroupsRequest +import time, json, os, glob, string, random +from dpgen.dispatcher.Dispatcher import Dispatcher, _split_tasks, JobRecord +from dpgen.dispatcher.SSHContext import SSHSession from os.path import join +from dpgen import dlog +from hashlib import sha1 + +def manual_delete(stage): + with open('machine-ali.json') as fp1: + mdata = json.load(fp1) + adata = mdata[stage][0]['machine']['ali_auth'] + mdata_resources = mdata[stage][0]['resources'] + mdata_machine = mdata[stage][0]['machine'] + ali = ALI(adata, mdata_resources, mdata_machine, 0) + with open('apg_id.json', 'r') as fp2: + apg_id = json.load(fp2) + ali.apg_id = apg_id['apg_id'] + ali.delete_apg() + +def manual_create(stage, machine_number): + with open('machine-ali.json') as fp: + mdata = json.load(fp) + adata = mdata[stage][0]['machine']['ali_auth'] + mdata_resources = mdata[stage][0]['resources'] + mdata_machine = mdata[stage][0]['machine'] + ali = ALI(adata, mdata_resources, mdata_machine, machine_number) + ali.create_ess() + print(ali.ip_list) class ALI(): - def __init__(self, adata, mdata_resources, mdata_machine, nchunks, work_path): - self.ip_list = None - self.instance_list = None - self.dispatchers = None + def __init__(self, adata, mdata_resources, mdata_machine, nchunks): + self.ip_list = ["unalloc" for i in range(nchunks)] + self.instance_list = ["unalloc" for i in range(nchunks)] + self.dispatchers = [[None, "unalloc"] for i in range(nchunks)] + self.job_handlers = ["unalloc" for i in range(nchunks)] + self.task_chunks = None self.adata = adata + self.apg_id = None + self.template_id = None + self.vsw_id = None + self.regionID = adata["regionID"] + self.client = AcsClient(adata["AccessKey_ID"], adata["AccessKey_Secret"], self.regionID) self.mdata_resources = mdata_resources self.mdata_machine = mdata_machine self.nchunks = nchunks - self.work_path = work_path - - def init(self): - if self.check_restart(): + + def init(self, work_path, tasks, group_size): + if self.check_restart(work_path, tasks, group_size): pass else: - self.create_machine() - self.dispatchers = self.make_dispatchers() + self.create_ess() + self.make_dispatchers() + + def create_ess(self): + img_id = self.get_image_id(self.adata["img_name"]) + sg_id, vpc_id = self.get_sg_vpc_id() + self.template_id = self.create_template(img_id, sg_id, vpc_id) + self.vsw_id = self.get_vsw_id(vpc_id) + self.apg_id = self.create_apg() + dlog.info("begin to create ess, please wait two minutes") + time.sleep(120) + new_server_list = self.describe_apg_instances() + new_ip_list = self.get_ip(new_server_list) + for ii in range(len(new_server_list)): + profile = self.mdata_machine.copy() + profile['hostname'] = new_ip_list[ii] + profile['instance_id'] = new_server_list[ii] + if self.check_server(profile): + self.instance_list[ii] = new_server_list[ii] + self.ip_list[ii] = new_ip_list[ii] + + def delete_apg(self): + request = DeleteAutoProvisioningGroupRequest() + request.set_accept_format('json') + request.set_AutoProvisioningGroupId(self.apg_id) + request.set_TerminateInstances(True) + try: + response = self.client.do_action_with_exception(request) + except ServerException as e: + dlog.info(e) + except ClientException as e: + dlog.info(e) + + def create_apg(self): + request = CreateAutoProvisioningGroupRequest() + request.set_accept_format('json') + request.set_TotalTargetCapacity(str(self.nchunks)) + request.set_LaunchTemplateId(self.template_id) + request.set_AutoProvisioningGroupName(self.adata["instance_name"] + ''.join(random.choice(string.ascii_uppercase) for _ in range(20))) + request.set_AutoProvisioningGroupType("maintain") + request.set_SpotAllocationStrategy("lowest-price") + request.set_SpotInstanceInterruptionBehavior("terminate") + request.set_SpotInstancePoolsToUseCount(1) + request.set_ExcessCapacityTerminationPolicy("termination") + request.set_TerminateInstances(True) + request.set_PayAsYouGoTargetCapacity("0") + request.set_SpotTargetCapacity(str(self.nchunks)) + config = self.generate_config() + request.set_LaunchTemplateConfigs(config) + try: + response = self.client.do_action_with_exception(request) + response = json.loads(response) + with open('apg_id.json', 'w') as fp: + json.dump({'apg_id': response["AutoProvisioningGroupId"]}, fp, indent=4) + return response["AutoProvisioningGroupId"] + except ServerException as e: + dlog.info(e) + except ClientException as e: + dlog.info(e) + + def update_server_list(self): + instance_list = self.describe_apg_instances() + return list(set(instance_list) - set(self.instance_list)) - def check_restart(self): - dispatchers = [] + def describe_apg_instances(self): + request = DescribeAutoProvisioningGroupInstancesRequest() + request.set_accept_format('json') + request.set_AutoProvisioningGroupId(self.apg_id) + request.set_PageSize(100) + iteration = self.nchunks // 100 instance_list = [] - if len(glob.glob(os.path.join(self.work_path, 'jr.*.json'))) == self.nchunks: + for i in range(iteration + 1): + request.set_PageNumber(i+1) + response = self.client.do_action_with_exception(request) + response = json.loads(response) + for ins in response["Instances"]["Instance"]: + instance_list.append(ins["InstanceId"]) + return instance_list + + def generate_config(self): + machine_config = self.adata["machine_type_price"] + config = [] + for conf in machine_config: + for vsw in self.vsw_id: + tmp = { + "InstanceType": conf["machine_type"], + "MaxPrice": str(conf["price_limit"] * conf["numb"]), + "VSwitchId": vsw, + "WeightedCapacity": "1", + "Priority": str(conf["priority"]) + } + config.append(tmp) + return config + + def create_template(self, image_id, sg_id, vpc_id): + request = CreateLaunchTemplateRequest() + request.set_accept_format('json') + request.set_LaunchTemplateName(''.join(random.choice(string.ascii_uppercase) for _ in range(20))) + request.set_ImageId(image_id) + request.set_ImageOwnerAlias("self") + request.set_PasswordInherit(True) + request.set_InstanceType("ecs.c6.large") + request.set_InstanceName(self.adata["instance_name"]) + request.set_SecurityGroupId(sg_id) + request.set_VpcId(vpc_id) + request.set_SystemDiskCategory("cloud_efficiency") + request.set_SystemDiskSize(40) + request.set_IoOptimized("optimized") + request.set_InstanceChargeType("PostPaid") + request.set_NetworkType("vpc") + request.set_SpotStrategy("SpotWithPriceLimit") + request.set_SpotPriceLimit(100) + try: + response = self.client.do_action_with_exception(request) + response = json.loads(response) + return response["LaunchTemplateId"] + except ServerException as e: + dlog.info(e) + except ClientException as e: + dlog.info(e) + + def delete_template(self): + request = DeleteLaunchTemplateRequest() + request.set_accept_format('json') + request.set_LaunchTemplateId(self.template_id) + response = self.client.do_action_with_exception(request) + + def get_image_id(self, img_name): + request = DescribeImagesRequest() + request.set_accept_format('json') + request.set_ImageOwnerAlias("self") + response = self.client.do_action_with_exception(request) + response = json.loads(response) + for img in response["Images"]["Image"]: + if img["ImageName"] == img_name: + return img["ImageId"] + + def get_sg_vpc_id(self): + request = DescribeSecurityGroupsRequest() + request.set_accept_format('json') + response = self.client.do_action_with_exception(request) + response = json.loads(response) + for sg in response["SecurityGroups"]["SecurityGroup"]: + if sg["SecurityGroupName"] == "sg": + return sg["SecurityGroupId"], sg["VpcId"] + + def get_vsw_id(self, vpc_id): + request = DescribeVpcsRequest() + request.set_accept_format('json') + request.set_VpcId(vpc_id) + response = self.client.do_action_with_exception(request) + response = json.loads(response) + for vpc in response["Vpcs"]["Vpc"]: + if vpc["VpcId"] == vpc_id: + return vpc["VSwitchIds"]["VSwitchId"] + + def change_apg_capasity(self, capasity): + request = ModifyAutoProvisioningGroupRequest() + request.set_accept_format('json') + request.set_AutoProvisioningGroupId(self.apg_id) + request.set_TotalTargetCapacity(str(capasity)) + request.set_SpotTargetCapacity(str(capasity)) + request.set_PayAsYouGoTargetCapacity("0") + response = self.client.do_action_with_exception(request) + + def check_spot_callback(self, instance_id): + request = DescribeInstancesRequest() + request.set_accept_format('json') + request.set_InstanceIds([instance_id]) + status = False + try: + response = self.client.do_action_with_exception(request) + response = json.loads(response) + if len(response["Instances"]["Instance"]) == 1 and "Recycling" in response["Instances"]["Instance"][0]["OperationLocks"]["LockReason"]: + status = True + if instance_id not in self.describe_apg_instances(): + status = True + except: + pass + return status + + def check_restart(self, work_path, tasks, group_size): + if os.path.exists('apg_id.json'): + with open('apg_id.json') as fp: + apg = json.load(fp) + self.apg_id = apg["apg_id"] + self.task_chunks = _split_tasks(tasks, group_size) + task_chunks_str = ['+'.join(ii) for ii in self.task_chunks] + task_hashes = [sha1(ii.encode('utf-8')).hexdigest() for ii in task_chunks_str] + nchunks = len(self.task_chunks) + for ii in range(nchunks): + fn = 'jr.%.06d.json' % ii + if not os.path.exists(os.path.join(os.path.abspath(work_path), fn)): + pass + else: + job_record = JobRecord(work_path, self.task_chunks[ii], fname = fn) + cur_chunk = self.task_chunks[ii] + cur_hash = task_hashes[ii] + if not job_record.check_finished(cur_hash): + with open(os.path.join(work_path, fn)) as fp: + jr = json.load(fp) + ip = jr[cur_hash]['context'][3] + instance_id = jr[cur_hash]['context'][4] + profile = self.mdata_machine.copy() + profile['hostname'] = ip + profile['instance_id'] = instance_id + if self.check_server(profile): + disp = Dispatcher(profile, context_type='ssh', batch_type='shell', job_record='jr.%.06d.json' % ii) + self.dispatchers[ii] = [disp, "working"] + self.ip_list[ii] = ip + self.instance_list[ii] = instance_id + # spot callback + else: + os.remove(os.path.join(work_path, fn)) + else: + self.ip_list[ii] = "finished" + self.instance_list[ii] = "finished" + self.dispatchers[ii] = [None, "finished"] + return True + else: + self.task_chunks = _split_tasks(tasks, group_size) + return False + + def get_ip(self, instance_list): + request = DescribeInstancesRequest() + request.set_accept_format('json') + ip_list = [] + if len(instance_list) <= 10: + for i in range(len(instance_list)): + request.set_InstanceIds([instance_list[i]]) + response = self.client.do_action_with_exception(request) + response = json.loads(response) + ip_list.append(response["Instances"]["Instance"][0]["VpcAttributes"]["PrivateIpAddress"]['IpAddress'][0]) + else: + iteration = len(instance_list) // 10 + for i in range(iteration): + for j in range(10): + request.set_InstanceIds([instance_list[i*10+j]]) + response = self.client.do_action_with_exception(request) + response = json.loads(response) + ip_list.append(response["Instances"]["Instance"][0]["VpcAttributes"]["PrivateIpAddress"]['IpAddress'][0]) + if len(instance_list) - iteration * 10 != 0: + for j in range(len(instance_list) - iteration * 10): + request.set_InstanceIds([instance_list[iteration*10+j]]) + response = self.client.do_action_with_exception(request) + response = json.loads(response) + ip_list.append(response["Instances"]["Instance"][0]["VpcAttributes"]["PrivateIpAddress"]['IpAddress'][0]) + return ip_list + + def get_finished_job_num(self): + finished_num = 0 + for ii in range(len(self.dispatchers)): + if self.dispatchers[ii][1] == "finished": + finished_num += 1 + return finished_num + + def resubmission(self, machine_exception_num): + if self.adata["img_name"] == "kit": + new_ip_list = [] + try: + new_server_list = self.update_server_list() + new_ip_list = self.get_ip(new_server_list) + except: + pass for ii in range(self.nchunks): - with open(os.path.join(self.work_path, 'jr.%.06d.json' % ii)) as fp: - job_record = json.load(fp) - key = list(job_record.keys())[0] - ip, instance_id = job_record[key]['context'][-2], job_record[key]['context'][-1] - instance_list.append(instance_id) + if len(new_ip_list) == 0: + break + if self.dispatchers[ii][1] == "exception": + self.ip_list[ii] = new_ip_list.pop() + self.instance_list[ii] = new_server_list.pop() profile = self.mdata_machine.copy() - profile['hostname'] = ip - profile['instance_id'] = instance_id - disp = Dispatcher(profile, context_type='ssh', batch_type='shell', job_record='jr.%.06d.json' % ii) - max_check = 10 - cnt = 0 - while not disp.session._check_alive(): - cnt += 1 - if cnt == max_check: - break - if cnt != max_check: - dispatchers.append(disp) - restart = False - if len(dispatchers) == self.nchunks: - restart = True - self.dispatchers = dispatchers - self.instance_list = instance_list - return restart + profile["hostname"] = self.ip_list[ii] + profile["instance_id"] = self.instance_list[ii] + self.dispatchers[ii] = [Dispatcher(profile, context_type='ssh', batch_type='shell', job_record="jr.%.06d.json" % ii), "working"] + dlog.info(self.ip_list[ii]) + job_handler = self.dispatchers[ii][0].submit_jobs(resources, + command, + work_path, + self.task_chunks[ii], + group_size, + forward_common_files, + forward_task_files, + backward_task_files, + forward_task_deference, + outlog, + errlog) + self.job_handlers[ii] = job_handler + elif self.adata["img_name"] == "vasp": + if machine_exception_num / self.nchunks > 0.05: + self.change_apg_capasity(self.nchunks - self.get_finished_job_num() + machine_exception_num) + time.sleep(120) + new_ip_list = [] + try: + new_server_list = self.update_server_list() + new_ip_list = self.get_ip(new_server_list) + except: + pass + if len(new_ip_list) == machine_exception_num: + dlog.info("new submission of callback machine") + for ii in range(self.nchunks): + self.ip_list[ii] = new_ip_list.pop() + self.instance_list[ii] = new_server_list.pop() + profile = self.mdata_machine.copy() + profile["hostname"] = self.ip_list[ii] + profile["instance_id"] = self.instance_list[ii] + self.dispatchers[ii] = [Dispatcher(profile, context_type='ssh', batch_type='shell', job_record="jr.%.06d.json" % ii), "working"] + dlog.info(self.ip_list[ii]) + job_handler = self.dispatchers[ii][0].submit_jobs(resources, + command, + work_path, + self.task_chunks[ii], + group_size, + forward_common_files, + forward_task_files, + backward_task_files, + forward_task_deference, + outlog, + errlog) + self.job_handlers[ii] = job_handler + #@profile(precision=6) def run_jobs(self, resources, command, @@ -65,82 +395,144 @@ def run_jobs(self, forward_task_files, backward_task_files, forward_task_deference = True, + mark_failure = False, outlog = 'log', errlog = 'err'): - task_chunks = _split_tasks(tasks, group_size) - job_handlers = [] for ii in range(self.nchunks): - job_handler = self.dispatchers[ii].submit_jobs(resources, - command, - work_path, - task_chunks[ii], - group_size, - forward_common_files, - forward_task_files, - backward_task_files, - forward_task_deference, - outlog, - errlog) - job_handlers.append(job_handler) + if self.dispatchers[ii][1] == "working": + dlog.info(self.ip_list[ii]) + job_handler = self.dispatchers[ii][0].submit_jobs(resources, + command, + work_path, + self.task_chunks[ii], + group_size, + forward_common_files, + forward_task_files, + backward_task_files, + forward_task_deference, + outlog, + errlog) + self.job_handlers[ii] = job_handler + machine_exception_num = 0 while True: - cnt = 0 + new_ip_list = [] + try: + new_server_list = self.update_server_list() + new_ip_list = self.get_ip(new_server_list) + except: + pass + if machine_exception_num > 0: + self.resubmission(machine_exception_num) for ii in range(self.nchunks): - if self.dispatchers[ii].all_finished(job_handlers[ii]): - cnt += 1 - if cnt == self.nchunks: + if self.dispatchers[ii][1] == "working": + if self.check_spot_callback(self.instance_list[ii]): + machine_exception_num += 1 + dlog.info("machine %s callback" % self.instance_list[ii]) + os.remove(self.job_handlers[ii]["job_record"].fname) + self.job_handlers[ii] = "exception" + self.ip_list[ii] = "exception" + self.instance_list[ii] = "exception" + self.dispatchers[ii][1] = "exception" + if self.adata["img_name"] == "vasp": + self.change_apg_capasity(self.nchunks - self.get_finished_job_num() - machine_exception_num) + continue + if self.check_server(self.dispatchers[ii][0].remote_profile): + if self.dispatchers[ii][0].all_finished(self.job_handlers[ii], mark_failure, False): + self.delete(ii) + self.dispatchers[ii][1] = "finished" + self.ip_list[ii] = "finished" + self.instance_list[ii] = "finished" + self.change_apg_capasity(self.nchunks - self.get_finished_job_num()) + else: + dlog.info("ssh exception accured in %s" % self.ip_list[ii]) + elif self.dispatchers[ii][1] == "finished": + continue + elif self.dispatchers[ii][1] == "unalloc": + if new_ip_list: + profile = self.mdata_machine.copy() + profile["hostname"] = new_ip_list[0] + profile["instance_id"] = new_server_list[0] + if self.check_server(profile): + self.ip_list[ii] = new_ip_list.pop(0) + self.instance_list[ii] = new_server_list.pop(0) + self.dispatchers[ii] = [Dispatcher(profile, context_type='ssh', batch_type='shell', job_record='jr.%.06d.json' % ii), "working"] + dlog.info(self.ip_list[ii]) + job_handler = self.dispatchers[ii][0].submit_jobs(resources, + command, + work_path, + self.task_chunks[ii], + group_size, + forward_common_files, + forward_task_files, + backward_task_files, + forward_task_deference, + outlog, + errlog) + self.job_handlers[ii] = job_handler + if self.check_dispatcher_finished(): + os.remove('apg_id.json') + self.delete_template() + self.delete_apg() break else: time.sleep(10) - self.delete_machine() + +# status = ["unalloc", "working", "finished", "exception"] + def check_server(self, profile): + try: + session = SSHSession(profile) + session.close() + return True + except: + return False + + def check_dispatcher_finished(self): + count = 0 + flag = True + for ii in range(len(self.dispatchers)): + if self.dispatchers[ii][1] == "unalloc" or self.dispatchers[ii][1] == "working": flag = False + if self.dispatchers[ii][1] == "exception": count += 1 + if self.adata["img_name"] == "vasp" and count / self.nchunks > 0.05: flag = False + elif self.adata["img_name"] == "kit" and count > 0: flag = False + return flag + + def delete(self, ii): + request = DeleteInstancesRequest() + request.set_accept_format('json') + request.set_InstanceIds([self.instance_list[ii]]) + request.set_Force(True) + response = self.client.do_action_with_exception(request) def make_dispatchers(self): - dispatchers = [] for ii in range(self.nchunks): - profile = self.mdata_machine.copy() - profile['hostname'] = self.ip_list[ii] - profile['instance_id'] = self.instance_list[ii] - disp = Dispatcher(profile, context_type='ssh', batch_type='shell', job_record='jr.%.06d.json' % ii) - dispatchers.append(disp) - return dispatchers - - def create_machine(self): - AccessKey_ID = self.adata["AccessKey_ID"] - AccessKey_Secret = self.adata["AccessKey_Secret"] - strategy = self.adata["pay_strategy"] - pwd = self.adata["password"] - regionID = self.mdata_machine['regionID'] - template_name = '%s_%s_%s' % (self.mdata_resources['partition'], self.mdata_resources['numb_gpu'], strategy) - instance_name = self.adata["instance_name"] - client = AcsClient(AccessKey_ID,AccessKey_Secret, regionID) - request = RunInstancesRequest() - request.set_accept_format('json') - request.set_UniqueSuffix(True) - request.set_Password(pwd) - request.set_InstanceName(instance_name) - request.set_Amount(self.nchunks) - request.set_LaunchTemplateName(template_name) - response = client.do_action_with_exception(request) - response = json.loads(response) - self.instance_list = response["InstanceIdSets"]["InstanceIdSet"] - time.sleep(50) - request = DescribeInstancesRequest() - request.set_accept_format('json') - request.set_InstanceIds(self.instance_list) - response = client.do_action_with_exception(request) - response = json.loads(response) - ip = [] - for i in range(len(response["Instances"]["Instance"])): - ip.append(response["Instances"]["Instance"][i]["PublicIpAddress"]['IpAddress'][0]) - self.ip_list = ip - + if self.ip_list[ii] != "unalloc": + profile = self.mdata_machine.copy() + profile['hostname'] = self.ip_list[ii] + profile['instance_id'] = self.instance_list[ii] + if self.check_server(profile): + self.dispatchers[ii] = [Dispatcher(profile, context_type='ssh', batch_type='shell', job_record='jr.%.06d.json' % ii), "working"] + else: + continue + #self.dispatchers[ii] = [Dispatcher(profile, context_type='ssh', batch_type='shell', job_record='jr.%.06d.json' % ii), "working"] + def delete_machine(self): - AccessKey_ID = self.adata["AccessKey_ID"] - AccessKey_Secret = self.adata["AccessKey_Secret"] - regionID = self.mdata_machine['regionID'] - client = AcsClient(AccessKey_ID,AccessKey_Secret, regionID) request = DeleteInstancesRequest() request.set_accept_format('json') - request.set_InstanceIds(self.instance_list) - request.set_Force(True) - response = client.do_action_with_exception(request) - + if len(self.instance_list) <= 100: + request.set_InstanceIds(self.instance_list) + request.set_Force(True) + response = self.client.do_action_with_exception(request) + else: + iteration = len(self.instance_list) // 100 + for i in range(iteration): + request.set_InstanceIds(self.instance_list[i*100:(i+1)*100]) + request.set_Force(True) + response = self.client.do_action_with_exception(request) + if len(self.instance_list) - iteration * 100 != 0: + request.set_InstanceIds(self.instance_list[iteration*100:]) + request.set_Force(True) + response = self.client.do_action_with_exception(request) + self.instance_list = [] + self.ip_list = [] + os.remove('machine_record.json') + dlog.debug("Successfully free the machine!") diff --git a/dpgen/dispatcher/Batch.py b/dpgen/dispatcher/Batch.py index 0b6d16b71..02b09e68e 100644 --- a/dpgen/dispatcher/Batch.py +++ b/dpgen/dispatcher/Batch.py @@ -7,13 +7,16 @@ class Batch(object) : def __init__ (self, context, - uuid_names = False) : + uuid_names = True) : self.context = context + self.uuid_names = uuid_names if uuid_names: + self.upload_tag_name = '%s_tag_upload' % self.context.job_uuid self.finish_tag_name = '%s_tag_finished' % self.context.job_uuid self.sub_script_name = '%s.sub' % self.context.job_uuid self.job_id_name = '%s_job_id' % self.context.job_uuid else: + self.upload_tag_name = 'tag_upload' self.finish_tag_name = 'tag_finished' self.sub_script_name = 'run.sub' self.job_id_name = 'job_id' @@ -99,7 +102,6 @@ def submit(self, args = None, res = None, restart = False, - sleep = 0, outlog = 'log', errlog = 'err'): if restart: @@ -119,7 +121,11 @@ def submit(self, else: dlog.debug('new task') self.do_submit(job_dirs, cmd, args, res, outlog=outlog, errlog=errlog) - time.sleep(sleep) # For preventing the crash of the tasks while submitting + if res is None: + sleep = 0 + else: + sleep = res.get('submit_wait_time', 0) + time.sleep(sleep) # For preventing the crash of the tasks while submitting def check_finish_tag(self) : return self.context.check_file_exists(self.finish_tag_name) @@ -133,19 +139,17 @@ def _sub_script_inner(self, outlog = 'log', errlog = 'err') : ret = "" - try: - allow_failure = res['allow_failure'] - except: - allow_failure = False + allow_failure = res.get('allow_failure', False) for ii,jj in zip(job_dirs, args) : ret += 'cd %s\n' % ii - ret += 'test $? -ne 0 && exit\n\n' + ret += 'test $? -ne 0 && exit 1\n\n' if self.manual_cuda_devices <= 0: ret += 'if [ ! -f tag_%d_finished ] ;then\n' % idx ret += ' %s 1>> %s 2>> %s \n' % (self.sub_script_cmd(cmd, jj, res), outlog, errlog) if res['allow_failure'] is False: - ret += ' if test $? -ne 0; then exit; else touch tag_%d_finished; fi \n' % idx + ret += ' if test $? -ne 0; then exit 1; else touch tag_%d_finished; fi \n' % idx else : + ret += ' if test $? -ne 0; then touch tag_failure_%d; fi \n' % idx ret += ' touch tag_%d_finished \n' % idx ret += 'fi\n\n' else : @@ -154,7 +158,7 @@ def _sub_script_inner(self, ret += 'CUDA_VISIBLE_DEVICES=%d %s &\n\n' % ((self.cmd_cnt % self.manual_cuda_devices), tmp_cmd) self.cmd_cnt += 1 ret += 'cd %s\n' % self.context.remote_root - ret += 'test $? -ne 0 && exit\n' + ret += 'test $? -ne 0 && exit 1\n' if self.manual_cuda_devices > 0 and self.cmd_cnt % (self.manual_cuda_devices * self.manual_cuda_multiplicity) == 0: ret += '\nwait\n\n' ret += '\nwait\n\n' diff --git a/dpgen/dispatcher/Dispatcher.py b/dpgen/dispatcher/Dispatcher.py index 915d7e201..444f3431e 100644 --- a/dpgen/dispatcher/Dispatcher.py +++ b/dpgen/dispatcher/Dispatcher.py @@ -1,5 +1,4 @@ import os,sys,time,random,json,glob - from dpgen.dispatcher.LocalContext import LocalSession from dpgen.dispatcher.LocalContext import LocalContext from dpgen.dispatcher.LazyLocalContext import LazyLocalContext @@ -36,10 +35,11 @@ def __init__ (self, batch_type = 'slurm', job_record = 'jr.json'): self.remote_profile = remote_profile + if context_type == 'local': self.session = LocalSession(remote_profile) self.context = LocalContext - self.uuid_names = False + self.uuid_names = True elif context_type == 'lazy-local': self.session = None self.context = LazyLocalContext @@ -47,7 +47,7 @@ def __init__ (self, elif context_type == 'ssh': self.session = SSHSession(remote_profile) self.context = SSHContext - self.uuid_names = False + self.uuid_names = True else : raise RuntimeError('unknown context') if batch_type == 'slurm': @@ -74,6 +74,7 @@ def run_jobs(self, forward_task_files, backward_task_files, forward_task_deference = True, + mark_failure = False, outlog = 'log', errlog = 'err') : job_handler = self.submit_jobs(resources, @@ -87,8 +88,8 @@ def run_jobs(self, forward_task_deference, outlog, errlog) - while not self.all_finished(job_handler) : - time.sleep(10) + while not self.all_finished(job_handler, mark_failure) : + time.sleep(60) # delete path map file when job finish # _pmap.delete() @@ -134,13 +135,14 @@ def submit_jobs(self, batch = self.batch(context, uuid_names = self.uuid_names) rjob = {'context':context, 'batch':batch} # upload files - if not rjob['context'].check_file_exists('tag_upload'): + if not rjob['context'].check_file_exists(rjob['batch'].upload_tag_name): rjob['context'].upload('.', forward_common_files) rjob['context'].upload(cur_chunk, forward_task_files, dereference = forward_task_deference) - rjob['context'].write_file('tag_upload', '') + + rjob['context'].write_file(rjob['batch'].upload_tag_name, '') dlog.debug('uploaded files for %s' % task_chunks_str[ii]) # submit new or recover old submission if not submitted: @@ -155,10 +157,9 @@ def submit_jobs(self, job_list.append(rjob) ip = None instance_id = None - if "type" in self.remote_profile: - if self.remote_profile['type'] == 'ALI': - ip = self.remote_profile['hostname'] - instance_id = self.remote_profile['instance_id'] + if 'ali_auth' in self.remote_profile: + ip = self.remote_profile['hostname'] + instance_id = self.remote_profile['instance_id'] job_record.record_remote_context(cur_hash, context.local_root, context.remote_root, @@ -184,13 +185,16 @@ def submit_jobs(self, def all_finished(self, - job_handler): + job_handler, + mark_failure, + clean=True): task_chunks = job_handler['task_chunks'] task_chunks_str = ['+'.join(ii) for ii in task_chunks] task_hashes = [sha1(ii.encode('utf-8')).hexdigest() for ii in task_chunks_str] job_list = job_handler['job_list'] job_record = job_handler['job_record'] command = job_handler['command'] + tag_failure_list = ['tag_failure_%d' % ii for ii in range(len(command))] resources = job_handler['resources'] outlog = job_handler['outlog'] errlog = job_handler['errlog'] @@ -214,8 +218,13 @@ def all_finished(self, rjob['batch'].submit(task_chunks[idx], command, res = resources, outlog=outlog, errlog=errlog,restart=True) elif status == JobStatus.finished : dlog.info('job %s finished' % job_uuid) - rjob['context'].download(task_chunks[idx], backward_task_files) - rjob['context'].clean() + if mark_failure: + rjob['context'].download(task_chunks[idx], tag_failure_list, check_exists = True, mark_failure = False) + rjob['context'].download(task_chunks[idx], backward_task_files, check_exists = True) + else: + rjob['context'].download(task_chunks[idx], backward_task_files) + if clean: + rjob['context'].clean() job_record.record_finish(cur_hash) job_record.dump() job_record.dump() @@ -295,23 +304,28 @@ def _new_record(self): } -def make_dispatcher(mdata): - try: - hostname = mdata['hostname'] - context_type = 'ssh' - except: - context_type = 'local' - try: - batch_type = mdata['batch'] - except: - dlog.info('cannot find key "batch" in machine file, try to use deprecated key "machine_type"') - batch_type = mdata['machine_type'] - try: - lazy_local = mdata['lazy_local'] - except: - lazy_local = False - if lazy_local and context_type == 'local': - dlog.info('Dispatcher switches to the lazy local mode') - context_type = 'lazy-local' - disp = Dispatcher(mdata, context_type=context_type, batch_type=batch_type) - return disp +def make_dispatcher(mdata, mdata_resource=None, work_path=None, run_tasks=None, group_size=None): + if 'ali_auth' in mdata: + from dpgen.dispatcher.ALI import ALI + nchunks = len(_split_tasks(run_tasks, group_size)) + dispatcher = ALI(mdata['ali_auth'], mdata_resource, mdata, nchunks) + dispatcher.init(work_path, run_tasks, group_size) + return dispatcher + else: + hostname = mdata.get('hostname', None) + #use_uuid = mdata.get('use_uuid', False) + if hostname: + context_type = 'ssh' + else: + context_type = 'local' + try: + batch_type = mdata['batch'] + except: + dlog.info('cannot find key "batch" in machine file, try to use deprecated key "machine_type"') + batch_type = mdata['machine_type'] + lazy_local = (mdata.get('lazy-local', False)) or (mdata.get('lazy_local', False)) + if lazy_local and context_type == 'local': + dlog.info('Dispatcher switches to the lazy local mode') + context_type = 'lazy-local' + disp = Dispatcher(mdata, context_type=context_type, batch_type=batch_type) + return disp diff --git a/dpgen/dispatcher/DispatcherList.py b/dpgen/dispatcher/DispatcherList.py new file mode 100644 index 000000000..a62bd2107 --- /dev/null +++ b/dpgen/dispatcher/DispatcherList.py @@ -0,0 +1,153 @@ +from dpgen.dispatcher.Dispatcher import Dispatcher, _split_tasks, JobRecord + +class Entity(): + def __init__(self, ip, instance_id, job_handler=None): + self.ip = ip + self.instance_id = instance_id + self.job_handler = job_handler + +class DispatcherList(): + def __init__(self, mdata_machine, mdata_resources, nchunks, cloud_resources=None): + self.mdata_machine = mdata_machine + self.mdata_resources = mdata_resources + self.nchunks = nchunks + self.cloud_resources = cloud_resources + self.dispatcher_list = list({"dispatcher": None, + "dispatcher_status": "unallocated", + "entity": None} for ii in range(nchunks)) + # Base + def init(self): + for ii in range(self.nchunks): + self.create(ii) + + # Base + def run_jobs(self, + resources, + command, + work_path, + tasks, + group_size, + forward_common_files, + forward_task_files, + backward_task_files, + forward_task_deference = True, + mark_failure = False, + outlog = 'log', + errlog = 'err'): + task_chunks = _split_tasks(tasks, group_size) + while True: + if self.check_all_dispatchers_finished(): + break + ratio_failure = self.mdata_resources["ratio_failure"] + self.exception_handling(ratio_failure) + for ii in range(self.nchunks): + dispatcher_status = self.check_dispatcher_status(ii) + if dispatcher_status == "unsubmitted": + self.dispatcher_list[ii]["job_handler"] = self.dispatcher_list[ii]["dispatcher"].submit_jobs(resources, + command, + work_path, + task_chunks[ii], + group_size, + forward_common_files, + forward_task_files, + backward_task_files, + forward_task_deference, + outlog, + errlog) + self.dispatcher_list[ii]["dispatcher_status"] == "running" + elif dispatcher_status == "finished": + # to do + # no jobs in queue, delete current machine + # else allocate current machine to unalloc dispatcher + pass + elif dispatcher_status == "running": + pass + elif dispatcher_status == "unallocated": + # to do: if we can create machine, then make dispatcher + # else pass + pass + elif dispatcher_status == "terminated": + pass + + # Derivate + def create(self, ii): + '''if jr.json existed and job not finished, use jr.json to rebuild dispatcher + else create one machine, then make_dispatcher, change status from unallocated to unsubmitted''' + if not os.path.exists(os.path.join(os.path.abspath(work_path), "jr.%.06d.json" % ii)): + # to do: create machine, make dispatcher, change status from unallocated to unsubmitted + pass + else: + task_chunks = _split_tasks(tasks, group_size) + task_chunks_str = ['+'.join(ii) for ii in self.task_chunks] + task_hashes = [sha1(ii.encode('utf-8')).hexdigest() for ii in task_chunks_str] + job_record = JobRecord(work_path, task_chunks[ii], fname = "jr.%.06d.json" % ii) + cur_chunk = self.task_chunks[ii] + cur_hash = task_hashes[ii] + if not job_record.check_finished(cur_hash): + with open(os.path.join(work_path, fn)) as fp: + jr = json.load(fp) + self.dispatcher_list[ii]["entity"] = Entity(jr[cur_hash]['context'][3], jr[cur_hash]['context'][4]) + self.make_dispatcher(ii) + # Derivate + def delete(self): + '''delete one machine''' + pass + + # Base + def check_all_dispatchers_finished(self, ratio_failure=0): + exception_num = 0 + for ii in range(self.nchunks): + if self.dispatcher_list[ii]["dispatcher_status"] == "terminated": + exception_num += 1 + if exception_num / self.nchunks > ratio_failure: return False + else: return True + + # Base + def exception_handling(self, ratio_failure): + terminated_num = 0 + for ii in range(self.nchunks): + if self.dispatcher_list[ii]["dispatcher_status"] == "terminated": + terminated_num += 1 + if terminated_num / self.nchunks > ratio_failure: + self.resubmit() + + # Derivate + def resubmit(self): + '''create machines + make dispatcher + change status from terminated to unsubmitted''' + pass + + # Base + def make_dispatcher(self, ii): + '''use entity to distinguish machine, for example if ip isn't None, means we can make_dispatcher + change status from unallocated to unsubmitted''' + entity = self.dispatcher_list[ii]["entity"] + if entity.ip: + profile = self.mdata_machine.copy() + profile['hostname'] = entity.ip + profile['instance_id'] = entity.instance_id + self.dispatcher_list[ii]["dispatcher"] = Dispatcher(profile, context_type='ssh', batch_type='shell', job_record='jr.%.06d.json' % ii) + self.dispatcher_list[ii]["dispatcher_status"] = "unsubmitted" + + # Base + def check_dispatcher_status(self, ii): + '''catch running dispatcher exception + if no exception occured, check finished''' + if self.dispatcher_list[ii]["dispatcher_status"] == "running": + if not self.catch_dispatcher_exception(): + if self.dispatcher_list[ii]["dispatcher"].all_finished(): + self.dispatcher_list[ii]["dispatcher_status"] = "finished" + else: + self.dispatcher_list[ii]["dispatcher_status"] = "terminated" + return self.dispatcher_list[ii]["dispatcher_status"] + + # Derivate + def catch_dispatcher_exception(self): + pass + + + + + + diff --git a/dpgen/dispatcher/LSF.py b/dpgen/dispatcher/LSF.py index 62b46e99b..7c5d05854 100644 --- a/dpgen/dispatcher/LSF.py +++ b/dpgen/dispatcher/LSF.py @@ -34,9 +34,9 @@ def check_status(self): status_word = status_line.split()[2] # ref: https://www.ibm.com/support/knowledgecenter/en/SSETD4_9.1.2/lsf_command_ref/bjobs.1.html - if status_word in ["PEND", "WAIT"] : + if status_word in ["PEND", "WAIT", "PSUSP"] : return JobStatus.waiting - elif status_word in ["RUN"] : + elif status_word in ["RUN", "USUSP"] : return JobStatus.running elif status_word in ["DONE","EXIT"] : if self.check_finish_tag() : @@ -102,14 +102,19 @@ def sub_script_head(self, res): ret = '' ret += "#!/bin/bash -l\n#BSUB -e %J.err\n#BSUB -o %J.out\n" if res['numb_gpu'] == 0: - ret += '#BSUB -R span[ptile=%d]\n#BSUB -n %d\n' % ( - res['node_cpu'], res['numb_node'] * res['task_per_node']) + ret += '#BSUB -n %d\n#BSUB -R span[ptile=%d]\n' % ( + res['numb_node'] * res['task_per_node'], res['node_cpu']) else: if res['node_cpu']: ret += '#BSUB -R span[ptile=%d]\n' % res['node_cpu'] - # It is selected only for the situation that GPU is related to CPU node. - ret += '#BSUB -R "select[ngpus >0] rusage[ngpus_excl_p=1]"\n#BSUB -n %d\n' % ( - res['numb_gpu']) + if res.get('new_lsf_gpu', False): + # supportted in LSF >= 10.1.0 SP6 + # ref: https://www.ibm.com/support/knowledgecenter/en/SSWRJV_10.1.0/lsf_resource_sharing/use_gpu_res_reqs.html + ret += '#BSUB -n %d\n#BSUB -gpu "num=%d:mode=shared:j_exclusive=yes"\n' % ( + res['numb_gpu'], res['task_per_node']) + else: + ret += '#BSUB -n %d\n#BSUB -R "select[ngpus >0] rusage[ngpus_excl_p=%d]"\n' % ( + res['numb_gpu'], res['task_per_node']) if res['time_limit']: ret += '#BSUB -W %s\n' % (res['time_limit'].split(':')[ 0] + ':' + res['time_limit'].split(':')[1]) diff --git a/dpgen/dispatcher/LazyLocalContext.py b/dpgen/dispatcher/LazyLocalContext.py index e42ad1ac8..0f79a8161 100644 --- a/dpgen/dispatcher/LazyLocalContext.py +++ b/dpgen/dispatcher/LazyLocalContext.py @@ -47,8 +47,22 @@ def upload(self, def download(self, job_dirs, remote_down_files, + check_exists = False, + mark_failure = True, back_error=False) : - pass + for ii in job_dirs : + for jj in remote_down_files : + fname = os.path.join(self.local_root, ii, jj) + exists = os.path.exists(fname) + if not exists: + if check_exists: + if mark_failure: + with open(os.path.join(self.local_root, ii, 'tag_failure_download_%s' % jj), 'w') as fp: pass + else: + pass + else: + raise RuntimeError('do not find download file ' + fname) + def block_checkcall(self, cmd) : diff --git a/dpgen/dispatcher/LocalContext.py b/dpgen/dispatcher/LocalContext.py index 4549b307a..373f112c7 100644 --- a/dpgen/dispatcher/LocalContext.py +++ b/dpgen/dispatcher/LocalContext.py @@ -88,6 +88,8 @@ def upload(self, def download(self, job_dirs, remote_down_files, + check_exists = False, + mark_failure = True, back_error=False) : cwd = os.getcwd() for ii in job_dirs : @@ -103,7 +105,13 @@ def download(self, lfile = os.path.join(local_job, jj) if not os.path.realpath(rfile) == os.path.realpath(lfile) : if (not os.path.exists(rfile)) and (not os.path.exists(lfile)): - raise RuntimeError('do not find download file ' + rfile) + if check_exists : + if mark_failure: + with open(os.path.join(self.local_root, ii, 'tag_failure_download_%s' % jj), 'w') as fp: pass + else : + pass + else : + raise RuntimeError('do not find download file ' + rfile) elif (not os.path.exists(rfile)) and (os.path.exists(lfile)) : # already downloaded pass @@ -114,7 +122,7 @@ def download(self, # both exists, replace! dlog.info('find existing %s, replacing by %s' % (lfile, rfile)) if os.path.isdir(lfile): - shutil.rmtree(lfile) + shutil.rmtree(lfile, ignore_errors=True) elif os.path.isfile(lfile) or os.path.islink(lfile): os.remove(lfile) shutil.move(rfile, lfile) @@ -152,7 +160,7 @@ def block_call(self, cmd) : return code, None, stdout, stderr def clean(self) : - shutil.rmtree(self.remote_root) + shutil.rmtree(self.remote_root, ignore_errors=True) def write_file(self, fname, write_str): with open(os.path.join(self.remote_root, fname), 'w') as fp : diff --git a/dpgen/dispatcher/SSHContext.py b/dpgen/dispatcher/SSHContext.py index 185ade392..a6ac3c1d4 100644 --- a/dpgen/dispatcher/SSHContext.py +++ b/dpgen/dispatcher/SSHContext.py @@ -11,8 +11,8 @@ def __init__ (self, jdata) : # with open(remote_profile) as fp : # self.remote_profile = json.load(fp) self.remote_host = self.remote_profile['hostname'] - self.remote_port = self.remote_profile['port'] self.remote_uname = self.remote_profile['username'] + self.remote_port = self.remote_profile.get('port', 22) self.remote_password = None if 'password' in self.remote_profile : self.remote_password = self.remote_profile['password'] @@ -86,14 +86,17 @@ def __init__ (self, self.job_uuid = str(uuid.uuid4()) self.remote_root = os.path.join(ssh_session.get_session_root(), self.job_uuid) self.ssh_session = ssh_session - self.ssh = self.ssh_session.get_ssh_client() self.ssh_session.ensure_alive() try: - sftp = self.ssh.open_sftp() + sftp = self.ssh_session.ssh.open_sftp() sftp.mkdir(self.remote_root) sftp.close() except: pass + + @property + def ssh(self): + return self.ssh_session.get_ssh_client() def close(self): self.ssh_session.close() @@ -118,6 +121,8 @@ def upload(self, def download(self, job_dirs, remote_down_files, + check_exists = False, + mark_failure = True, back_error=False) : self.ssh_session.ensure_alive() cwd = os.getcwd() @@ -125,19 +130,37 @@ def download(self, file_list = [] for ii in job_dirs : for jj in remote_down_files : - file_list.append(os.path.join(ii,jj)) + file_name = os.path.join(ii,jj) + if check_exists: + if self.check_file_exists(file_name): + file_list.append(file_name) + elif mark_failure : + with open(os.path.join(self.local_root, ii, 'tag_failure_download_%s' % jj), 'w') as fp: pass + else: + pass + else: + file_list.append(file_name) if back_error: errors=glob(os.path.join(ii,'error*')) file_list.extend(errors) - self._get_files(file_list) + if len(file_list) > 0: + self._get_files(file_list) os.chdir(cwd) def block_checkcall(self, - cmd) : + cmd, + retry=0) : self.ssh_session.ensure_alive() stdin, stdout, stderr = self.ssh.exec_command(('cd %s ;' % self.remote_root) + cmd) exit_status = stdout.channel.recv_exit_status() if exit_status != 0: + if retry<3: + # sleep 60 s + dlog.warning("Get error code %d in calling %s through ssh with job: %s . message: %s" % + (exit_status, cmd, self.job_uuid, stderr.read().decode('utf-8'))) + dlog.warning("Sleep 60 s and retry the command...") + time.sleep(60) + return self.block_checkcall(cmd, retry=retry+1) raise RuntimeError("Get error code %d in calling %s through ssh with job: %s . message: %s" % (exit_status, cmd, self.job_uuid, stderr.read().decode('utf-8'))) return stdin, stdout, stderr @@ -190,7 +213,8 @@ def call(self, cmd): def check_finish(self, cmd_pipes): return cmd_pipes['stdout'].channel.exit_status_ready() - + + def get_return(self, cmd_pipes): if not self.check_finish(cmd_pipes): return None, None, None @@ -232,7 +256,10 @@ def _put_files(self, from_f = os.path.join(self.local_root, of) to_f = os.path.join(self.remote_root, of) sftp = self.ssh.open_sftp() - sftp.put(from_f, to_f) + try: + sftp.put(from_f, to_f) + except FileNotFoundError: + raise FileNotFoundError("from %s to %s Error!"%(from_f,to_f)) # remote extract self.block_checkcall('tar xf %s' % of) # clean up diff --git a/dpgen/dispatcher/Shell.py b/dpgen/dispatcher/Shell.py index 81bff523c..731ad52f5 100644 --- a/dpgen/dispatcher/Shell.py +++ b/dpgen/dispatcher/Shell.py @@ -1,6 +1,7 @@ import os,getpass,time from dpgen.dispatcher.Batch import Batch from dpgen.dispatcher.JobStatus import JobStatus +import datetime def _default_item(resources, key, value) : if key not in resources : @@ -10,14 +11,13 @@ def _default_item(resources, key, value) : class Shell(Batch) : def check_status(self) : - if not hasattr(self, 'proc'): - return JobStatus.unsubmitted - if not self.context.check_finish(self.proc) : - return JobStatus.running - elif (self.context.get_return(self.proc))[0] == 0 : + if self.check_finish_tag(): return JobStatus.finished - else : + elif self.check_running(): + return JobStatus.running + else: return JobStatus.terminated + ## warn: cannont distinguish terminated from unsubmitted. def do_submit(self, job_dirs, @@ -32,6 +32,16 @@ def do_submit(self, self.context.write_file(self.sub_script_name, script_str) self.proc = self.context.call('cd %s && exec bash %s' % (self.context.remote_root, self.sub_script_name)) + def check_running(self): + uuid_names = self.context.job_uuid + ## Check if the uuid.sub is running on remote machine + cnt = 0 + ret, stdin, stdout, stderr = self.context.block_call("ps aux | grep %s"%uuid_names) + response_list = stdout.read().decode('utf-8').split("\n") + for response in response_list: + if uuid_names + ".sub" in response: + return True + return False def default_resources(self, res_) : if res_ == None : @@ -91,13 +101,12 @@ def sub_script_cmd(self, _cmd = cmd.split('1>')[0].strip() if cvasp : if res['with_mpi']: - _cmd = 'python ../cvasp.py "mpirun -n %d %s %s" %s' % (res['task_per_node'], _cmd, arg, fp_max_errors) + _cmd = 'python cvasp.py "mpirun -n %d %s %s" %s' % (res['task_per_node'], _cmd, arg, fp_max_errors) else : - _cmd = 'python ../cvasp.py "%s %s" %s' % (_cmd, arg, fp_max_errors) + _cmd = 'python cvasp.py "%s %s" %s' % (_cmd, arg, fp_max_errors) else : if res['with_mpi']: _cmd = 'mpirun -n %d %s %s' % (res['task_per_node'], _cmd, arg) else : _cmd = '%s %s' % (_cmd, arg) return _cmd - diff --git a/dpgen/dispatcher/Slurm.py b/dpgen/dispatcher/Slurm.py index 5ae5493cd..9d409dbcc 100644 --- a/dpgen/dispatcher/Slurm.py +++ b/dpgen/dispatcher/Slurm.py @@ -75,18 +75,18 @@ def sub_script_head(self, res): ret = '' ret += "#!/bin/bash -l\n" ret += "#SBATCH -N %d\n" % res['numb_node'] - ret += "#SBATCH --ntasks-per-node %d\n" % res['task_per_node'] + ret += "#SBATCH --ntasks-per-node=%d\n" % res['task_per_node'] if res['cpus_per_task'] > 0 : - ret += "#SBATCH --cpus-per-task %d\n" % res['cpus_per_task'] + ret += "#SBATCH --cpus-per-task=%d\n" % res['cpus_per_task'] ret += "#SBATCH -t %s\n" % res['time_limit'] if res['mem_limit'] > 0 : - ret += "#SBATCH --mem %dG \n" % res['mem_limit'] + ret += "#SBATCH --mem=%dG \n" % res['mem_limit'] if len(res['account']) > 0 : - ret += "#SBATCH --account %s \n" % res['account'] + ret += "#SBATCH --account=%s \n" % res['account'] if len(res['partition']) > 0 : - ret += "#SBATCH --partition %s \n" % res['partition'] + ret += "#SBATCH --partition=%s \n" % res['partition'] if len(res['qos']) > 0 : - ret += "#SBATCH --qos %s \n" % res['qos'] + ret += "#SBATCH --qos=%s \n" % res['qos'] if res['numb_gpu'] > 0 : ret += "#SBATCH --gres=gpu:%d\n" % res['numb_gpu'] for ii in res['constraint_list'] : @@ -99,7 +99,7 @@ def sub_script_head(self, res): temp_exclude += ii temp_exclude += "," temp_exclude = temp_exclude[:-1] - ret += '#SBATCH --exclude %s \n' % temp_exclude + ret += '#SBATCH --exclude=%s \n' % temp_exclude ret += "\n" for ii in res['module_unload_list'] : ret += "module unload %s\n" % ii @@ -133,9 +133,9 @@ def sub_script_cmd(self, _cmd = cmd.split('1>')[0].strip() if cvasp : if res['with_mpi']: - _cmd = 'python ../cvasp.py "srun %s %s" %s' % (_cmd, arg, fp_max_errors) + _cmd = 'python cvasp.py "srun %s %s" %s' % (_cmd, arg, fp_max_errors) else : - _cmd = 'python ../cvasp.py "%s %s" %s' % (_cmd, arg, fp_max_errors) + _cmd = 'python cvasp.py "%s %s" %s' % (_cmd, arg, fp_max_errors) else : if res['with_mpi']: _cmd = 'srun %s %s' % (_cmd, arg) @@ -149,9 +149,9 @@ def _get_job_id(self) : else: return "" - def _check_status_inner(self, job_id): + def _check_status_inner(self, job_id, retry=0): ret, stdin, stdout, stderr\ - = self.context.block_call ("squeue --job " + job_id) + = self.context.block_call ('squeue -o "%.18i %.2t" -j ' + job_id) if (ret != 0) : err_str = stderr.read().decode('utf-8') if str("Invalid job id specified") in err_str : @@ -160,10 +160,19 @@ def _check_status_inner(self, job_id): else : return JobStatus.terminated else : + # retry 3 times + if retry < 3: + # rest 60s + time.sleep(60) + return self._check_status_inner(job_id, retry=retry+1) raise RuntimeError\ ("status command squeue fails to execute\nerror message:%s\nreturn code %d\n" % (err_str, ret)) status_line = stdout.read().decode('utf-8').split ('\n')[-2] - status_word = status_line.split ()[-4] + status_word = status_line.split ()[-1] + if not (len(status_line.split()) == 2 and status_word.isupper()): + raise RuntimeError("Error in getting job status, " + + f"status_line = {status_line}, " + + f"parsed status_word = {status_word}") if status_word in ["PD","CF","S"] : return JobStatus.waiting elif status_word in ["R"] : @@ -185,7 +194,7 @@ def _check_sub_limit(self, task_max, **kwarg) : username = getpass.getuser() stdin, stdout, stderr = self.context.block_checkcall('squeue -u %s -h' % username) nj = len(stdout.readlines()) - return nj < task_max + return nj >= task_max def _make_squeue(self,mdata1, res): ret = '' diff --git a/dpgen/generator/lib/gaussian.py b/dpgen/generator/lib/gaussian.py index 79a420e27..304622dfb 100644 --- a/dpgen/generator/lib/gaussian.py +++ b/dpgen/generator/lib/gaussian.py @@ -3,6 +3,7 @@ import uuid import itertools +import warnings import numpy as np import dpdata from scipy.sparse import csr_matrix @@ -113,30 +114,33 @@ def make_gaussian_input(sys_data, fp_params): keywords = [keywords] # assume default charge is zero and default spin multiplicity is 1 charge = fp_params.get('charge', 0) - mult_auto = False - frag = False - if 'multiplicity' in fp_params: - if type(fp_params['multiplicity']) == int: - multiplicity = fp_params['multiplicity'] - elif fp_params['multiplicity'] == 'auto': - mult_auto = True - elif fp_params['multiplicity'] == 'frag': - mult_auto = True - frag = True - else: - raise RuntimeError('The keyword "multiplicity" is illegal.') + use_fragment_guesses = False + multiplicity = fp_params.get('multiplicity', 'auto') + if type(multiplicity) == int: + multiplicity = fp_params['multiplicity'] + mult_auto = False + elif multiplicity == 'auto': + mult_auto = True else: - multiplicity = 1 + raise RuntimeError('The keyword "multiplicity" is illegal.') + + if fp_params.get("fragment_guesses", False): + # Initial guess generated from fragment guesses + # New feature of Gaussian 16 + use_fragment_guesses = True + if not mult_auto: + warnings.warn("Automatically set multiplicity to auto!") + mult_auto = True if mult_auto: frag_numb, frag_index = _crd2frag(symbols, coordinates) if frag_numb == 1: - frag = False + use_fragment_guesses = False mult_frags = [] for i in range(frag_numb): idx = frag_index == i mult_frags.append(detect_multiplicity(np.array(symbols)[idx])) - if frag: + if use_fragment_guesses: multiplicity = sum(mult_frags) - frag_numb + 1 chargekeywords_frag = "%d %d" % (charge, multiplicity) + \ ''.join([' %d %d' % (charge, mult_frag) @@ -148,7 +152,7 @@ def make_gaussian_input(sys_data, fp_params): np.count_nonzero(multi_frags == 3) * 2 buff = [] # keywords, e.g., force b3lyp/6-31g** - if frag: + if use_fragment_guesses: keywords[0] = '{} guess=fragment={}'.format( keywords[0], frag_numb) @@ -161,10 +165,10 @@ def make_gaussian_input(sys_data, fp_params): chargekeywords = '{} {}'.format(charge, multiplicity) buff = [*chkkeywords, nprockeywords, '#{}'.format( - keywords[0]), '', titlekeywords, '', (chargekeywords_frag if frag else chargekeywords)] + keywords[0]), '', titlekeywords, '', (chargekeywords_frag if use_fragment_guesses else chargekeywords)] for ii, (symbol, coordinate) in enumerate(zip(symbols, coordinates)): - if frag: + if use_fragment_guesses: buff.append("%s(Fragment=%d) %f %f %f" % (symbol, frag_index[ii] + 1, *coordinate)) else: diff --git a/dpgen/generator/lib/lammps.py b/dpgen/generator/lib/lammps.py index 367817a89..95dcb8568 100644 --- a/dpgen/generator/lib/lammps.py +++ b/dpgen/generator/lib/lammps.py @@ -31,6 +31,7 @@ def make_lammps_input(ensemble, ele_temp_f = None, ele_temp_a = None, max_seed = 1000000, + nopbc = False, deepmd_version = '0.1') : if (ele_temp_f is not None or ele_temp_a is not None) and LooseVersion(deepmd_version) < LooseVersion('1'): raise RuntimeError('the electron temperature is only supported by deepmd-kit >= 1.0.0, please upgrade your deepmd-kit') @@ -49,7 +50,10 @@ def make_lammps_input(ensemble, ret+= "variable TAU_P equal %f\n" % tau_p ret+= "\n" ret+= "units metal\n" - ret+= "boundary p p p\n" + if nopbc: + ret+= "boundary f f f\n" + else: + ret+= "boundary p p p\n" ret+= "atom_style atomic\n" ret+= "\n" ret+= "neighbor 1.0 bin\n" @@ -104,6 +108,8 @@ def make_lammps_input(ensemble, ret+= "\n" if ensemble.split('-')[0] == 'npt' : assert (pres is not None) + if nopbc: + raise RuntimeError('ensemble %s is conflicting with nopbc' % ensemble) if ensemble == "npt" or ensemble == "npt-i" or ensemble == "npt-iso" : ret+= "fix 1 all npt temp ${TEMP} ${TEMP} ${TAU_T} iso ${PRES} ${PRES} ${TAU_P}\n" elif ensemble == 'npt-a' or ensemble == 'npt-aniso' : @@ -116,6 +122,9 @@ def make_lammps_input(ensemble, ret+= "fix 1 all nve\n" else : raise RuntimeError("unknown emsemble " + ensemble) + if nopbc: + ret+= "velocity all zero linear\n" + ret+= "fix fm all momentum 1 linear 1 1 1\n" ret+= "\n" ret+= "timestep %f\n" % dt ret+= "run ${NSTEPS}\n" diff --git a/dpgen/generator/lib/pwmat.py b/dpgen/generator/lib/pwmat.py new file mode 100644 index 000000000..4011b82c7 --- /dev/null +++ b/dpgen/generator/lib/pwmat.py @@ -0,0 +1,177 @@ +#!/usr/bin/python3 + +import os +import numpy as np + +def _reciprocal_box(box) : + rbox = np.linalg.inv(box) + rbox = rbox.T + return rbox + +def _make_pwmat_kp_mp(kpoints) : + ret = "" + ret += "%d %d %d 0 0 0 " % (kpoints[0], kpoints[1], kpoints[2]) + return ret + +def _make_kspacing_kpoints(config, kspacing) : + with open(config, 'r') as fp: + lines = fp.read().split('\n') + box = [] + for idx, ii in enumerate(lines): + if 'lattice' in ii or 'Lattice' in ii or 'LATTICE' in ii: + for kk in range(idx+1,idx+1+3): + vector=[float(jj) for jj in lines[kk].split()[0:3]] + box.append(vector) + box = np.array(box) + rbox = _reciprocal_box(box) + kpoints = [(np.ceil(2 * np.pi * np.linalg.norm(ii) / kspacing).astype(int)) for ii in rbox] + ret = _make_pwmat_kp_mp(kpoints) + return ret + + +def make_pwmat_input_dict (node1, node2, atom_config, ecut, e_error, + rho_error, icmix = None, smearing = None, + sigma = None,kspacing = 0.5, flag_symm = None) : + input_dict = {} + input_dict['node1'] = node1 + input_dict['node2'] = node2 + input_dict['in.atom'] = atom_config + input_dict['ecut'] = ecut + input_dict['e_error'] = e_error + input_dict['rho_error'] = rho_error + if icmix is not None: + if sigma is not None: + if smearing is not None: + SCF_ITER0_1 = "6 4 3 0.0000 " + str(sigma) + " " + str(smearing) + SCF_ITER0_2 = "94 4 3 " + str(icmix) + " " + str(sigma) + " " + str(smearing) + else: + SCF_ITER0_1 = "6 4 3 0.0000 " + str(simga) + " 2" + SCF_ITER0_2 = "94 4 3 " + str(icmix) + " " + str(simga) + " 2" + + else: + if smearing is not None: + SCF_ITER0_1 = "6 4 3 0.0000 0.025 " + str(smearing) + SCF_ITER0_2 = "94 4 3 " + str(icmix) + " 0.025 " + str(smearing) + else: + SCF_ITER0_1 = "6 4 3 0.0000 0.025 2" + SCF_ITER0_2 = "94 4 3 " + str(icmix) + " 0.025 2" + else: + if sigma is not None: + if smearing is not None: + SCF_ITER0_1 = "6 4 3 0.0000 " + str(sigma) + " " + str(smearing) + SCF_ITER0_2 = "94 4 3 1.0000 " + str(sigma) + " " + str(smearing) + else: + SCF_ITER0_1 = "6 4 3 0.0000 " + str(sigma) + " 2" + SCF_ITER0_2 = "94 4 3 1.0000 " + str(sigma) + " 2" + else: + if smearing is not None: + SCF_ITER0_1 = "6 4 3 0.0000 0.025 " + str(smearing) + SCF_ITER0_2 = "94 4 3 1.0000 0.025 " + str(smearing) + else: + SCF_ITER0_1 = "6 4 3 0.0000 0.025 2" + SCF_ITER0_2 = "94 4 3 1.0000 0.025 2" + input_dict['scf_iter0_1'] = SCF_ITER0_1 + input_dict['scf_iter0_2'] = SCF_ITER0_2 + if flag_symm is not None : + MP_N123 = _make_kspacing_kpoints(atom_config, kspacing) + MP_N123 += str(flag_symm) + else: + MP_N123 = _make_kspacing_kpoints(atom_config, kspacing) + input_dict['mp_n123'] = MP_N123 + input_dict['out.wg'] = 'F' + input_dict['out.rho'] = 'F' + input_dict['out.mlmd'] = 'T\n' + return input_dict + +def _update_input_dict(input_dict_, user_dict) : + if user_dict is None: + return input_dict_ + input_dict = input_dict_ + for ii in user_dict : + input_dict[ci] = user_dict[ii] + return input_dict + +def write_input_dict(input_dict) : + lines = [] + for key in input_dict: + if (type(input_dict[key]) == bool): + if input_dict[key]: + rs = 'T' + else : + rs = 'F' + else : + rs = str(input_dict[key]) + lines.append('%s=%s' % (key, rs)) + return '\n'.join(lines) + + +def _make_smearing(fp_params) : + icmix = None + smearing = None + sigma = None + if 'icmix' in fp_params : + icmix = fp_params['icmix'] + if 'smearing' in fp_params : + smearing = fp_params['smearing'] + if 'sigma' in fp_params : + sigma = fp_params['sigma'] + if icmix == None: + if smearing == None: + if sigma == None: + return None, None, None + else: + return None, None, sigma + else: + if sigma == None: + return None, smearing, None + else: + return None, smearing, sigma + else: + if smearing == None: + if sigma == None: + return icmix, None, None + else: + return icmix, None, sigma + else: + if sigma == None: + return icmix, smearing, None + else: + return icmix, smearing, sigma +def _make_flag_symm(fp_params) : + flag_symm = None + if 'flag_symm' in fp_params : + flag_symm = fp_params['flag_symm'] + if flag_symm == 'NONE' : + flag_symm = None + elif str(flag_symm) not in [None, '0', '1', '2', '3'] : + raise RuntimeError ("unknow flag_symm type " + str(flag_symm)) + return flag_symm + +def make_pwmat_input_user_dict(fp_params) : + node1 = fp_params['node1'] + node2 = fp_params['node2'] + atom_config = fp_params['in.atom'] + ecut = fp_params['ecut'] + e_error = fp_params['e_error'] + rho_error = fp_params['rho_error'] + kspacing = fp_params['kspacing'] + if 'user_pwmat_params' in fp_params : + user_dict = fp_params['user_pwmat_params'] + else : + user_dict = None + icmix, smearing, sigma = _make_smearing(fp_params) + flag_symm = _make_flag_symm(fp_params) + input_dict = make_pwmat_input_dict(node1, node2, atom_config, ecut, e_error, + rho_error, icmix = icmix, smearing = smearing, + sigma = sigma, kspacing = kspacing, + flag_symm = flag_symm + ) + input_dict = _update_input_dict(input_dict, user_dict) + input = write_input_dict(input_dict) + return input + +def input_upper(dinput): + standard_input={} + for key,val in dinput.items(): + standard_input[key.upper()]=val + return Input(standard_input) diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index ceac94ca1..3b6c2a47d 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -43,6 +43,10 @@ from dpgen.generator.lib.vasp import incar_upper from dpgen.generator.lib.pwscf import make_pwscf_input #from dpgen.generator.lib.pwscf import cvt_1frame +from dpgen.generator.lib.pwmat import make_pwmat_input_dict +from dpgen.generator.lib.pwmat import write_input_dict +from dpgen.generator.lib.pwmat import make_pwmat_input_user_dict +from dpgen.generator.lib.pwmat import input_upper from dpgen.generator.lib.siesta import make_siesta_input from dpgen.generator.lib.gaussian import make_gaussian_input, take_cluster from dpgen.generator.lib.cp2k import make_cp2k_input, make_cp2k_xyz @@ -53,11 +57,6 @@ from dpgen.remote.group_jobs import group_local_jobs from dpgen.remote.decide_machine import decide_train_machine, decide_fp_machine, decide_model_devi_machine from dpgen.dispatcher.Dispatcher import Dispatcher, _split_tasks, make_dispatcher -try: - from dpgen.dispatcher.ALI import ALI -except ImportError as e: - dlog.info(e) - pass from dpgen.util import sepline from dpgen import ROOT_PATH from pymatgen.io.vasp import Incar,Kpoints,Potcar @@ -194,6 +193,15 @@ def make_train (iter_index, fp_task_min = jdata['fp_task_min'] model_devi_jobs = jdata['model_devi_jobs'] use_ele_temp = jdata.get('use_ele_temp', 0) + training_iter0_model = jdata.get('training_iter0_model_path', []) + training_init_model = jdata.get('training_init_model', False) + training_reuse_iter = jdata.get('training_reuse_iter') + training_reuse_old_ratio = jdata.get('training_reuse_old_ratio', 0.2) + training_reuse_stop_batch = jdata.get('training_reuse_stop_batch', 400000) + training_reuse_start_lr = jdata.get('training_reuse_start_lr', 1e-4) + training_reuse_start_pref_e = jdata.get('training_reuse_start_pref_e', 0.1) + training_reuse_start_pref_f = jdata.get('training_reuse_start_pref_f', 100) + model_devi_activation_func = jdata.get('model_devi_activation_func', None) if iter_index > 0 and _check_empty_iter(iter_index-1, fp_task_min) : log_task('prev data is empty, copy prev model') @@ -229,12 +237,18 @@ def make_train (iter_index, init_batch_size = [] if 'init_batch_size' in jdata: init_batch_size_ = list(jdata['init_batch_size']) + if len(init_data_sys_) > len(init_batch_size_): + warnings.warn("The batch sizes are not enough. Assume auto for those not spefified.") + init_batch_size.extend(["auto" for aa in range(len(init_data_sys_)-len(init_batch_size))]) else: init_batch_size_ = ["auto" for aa in range(len(jdata['init_data_sys']))] if 'sys_batch_size' in jdata: sys_batch_size = jdata['sys_batch_size'] else: sys_batch_size = ["auto" for aa in range(len(jdata['sys_configs']))] + + # make sure all init_data_sys has the batch size -- for the following `zip` + assert (len(init_data_sys_) <= len(init_batch_size_)) for ii, ss in zip(init_data_sys_, init_batch_size_) : if jdata.get('init_multi_systems', False): for single_sys in os.listdir(os.path.join(work_path, 'data.init', ii)): @@ -243,8 +257,11 @@ def make_train (iter_index, else: init_data_sys.append(os.path.join('..', 'data.init', ii)) init_batch_size.append(detect_batch_size(ss, os.path.join(work_path, 'data.init', ii))) + old_range = None if iter_index > 0 : for ii in range(iter_index) : + if ii == iter_index - 1: + old_range = len(init_data_sys) fp_path = os.path.join(make_iter_name(ii), fp_name) fp_data_sys = glob.glob(os.path.join(fp_path, "data.*")) for jj in fp_data_sys : @@ -262,9 +279,7 @@ def make_train (iter_index, init_data_sys.append(os.path.join('..', 'data.iters', jj, sys_single)) init_batch_size.append(detect_batch_size(sys_batch_size[sys_idx], os.path.join(jj, sys_single))) else: - tmp_box = np.loadtxt(os.path.join(jj, 'box.raw')) - tmp_box = np.reshape(tmp_box, [-1,9]) - nframes = tmp_box.shape[0] + nframes = dpdata.System(jj, 'deepmd/npy').get_nframes() if nframes < fp_task_min : log_task('nframes (%d) in data sys %s is too small, skip' % (nframes, jj)) continue @@ -274,8 +289,9 @@ def make_train (iter_index, jinput = jdata['default_training_param'] try: mdata["deepmd_version"] - except: + except KeyError: mdata = set_version(mdata) + # setup data systems if LooseVersion(mdata["deepmd_version"]) < LooseVersion('1'): # 0.x jinput['systems'] = init_data_sys @@ -286,6 +302,7 @@ def make_train (iter_index, # 1.x jinput['training']['systems'] = init_data_sys jinput['training']['batch_size'] = init_batch_size + jinput['model']['type_map'] = jdata['type_map'] # electron temperature if use_ele_temp == 0: pass @@ -297,14 +314,27 @@ def make_train (iter_index, jinput['model']['fitting_net'].pop('numb_fparam', None) else: raise RuntimeError('invalid setting for use_ele_temp ' + str(use_ele_temp)) + # set training reuse model + if training_reuse_iter is not None and iter_index >= training_reuse_iter: + jinput['training']['auto_prob_style'] \ + ="prob_sys_size; 0:%d:%f; %d:%d:%f" \ + %(old_range, training_reuse_old_ratio, old_range, len(init_data_sys), 1.-training_reuse_old_ratio) + if jinput['loss'].get('start_pref_e') is not None: + jinput['loss']['start_pref_e'] = training_reuse_start_pref_e + if jinput['loss'].get('start_pref_f') is not None: + jinput['loss']['start_pref_f'] = training_reuse_start_pref_f + jinput['learning_rate']['start_lr'] = training_reuse_start_lr + jinput['training']['stop_batch'] = training_reuse_stop_batch + for ii in range(numb_models) : task_path = os.path.join(work_path, train_task_fmt % ii) create_path(task_path) os.chdir(task_path) - for ii in init_data_sys : - if not os.path.isdir(ii) : - raise RuntimeError ("data sys %s does not exists, cwd is %s" % (ii, os.getcwd())) + for jj in init_data_sys : + if not os.path.isdir(jj) : + raise RuntimeError ("data sys %s does not exists, cwd is %s" % (jj, os.getcwd())) os.chdir(cwd) + # set random seed for each model if LooseVersion(mdata["deepmd_version"]) < LooseVersion('1'): # 0.x jinput['seed'] = random.randrange(sys.maxsize) % (2**32) @@ -313,6 +343,14 @@ def make_train (iter_index, jinput['model']['descriptor']['seed'] = random.randrange(sys.maxsize) % (2**32) jinput['model']['fitting_net']['seed'] = random.randrange(sys.maxsize) % (2**32) jinput['training']['seed'] = random.randrange(sys.maxsize) % (2**32) + # set model activation function + if model_devi_activation_func is not None: + if LooseVersion(mdata["deepmd_version"]) < LooseVersion('1'): + raise RuntimeError('model_devi_activation_func does not suppport deepmd version', mdata['deepmd_version']) + assert(type(model_devi_activation_func) is list and len(model_devi_activation_func) == numb_models) + jinput['model']['descriptor']['activation_function'] = model_devi_activation_func[ii] + jinput['model']['fitting_net']['activation_function'] = model_devi_activation_func[ii] + # dump the input.json with open(os.path.join(task_path, train_input_file), 'w') as outfile: json.dump(jinput, outfile, indent = 4) @@ -324,16 +362,38 @@ def make_train (iter_index, prev_task_path = os.path.join(prev_work_path, train_task_fmt%ii) old_model_files = glob.glob( os.path.join(prev_task_path, "model.ckpt*")) - task_path = os.path.join(work_path, train_task_fmt % ii) - task_old_path = os.path.join(task_path, 'old') - create_path(task_old_path) - cwd = os.getcwd() - for jj in old_model_files: - absjj = os.path.abspath(jj) - basejj = os.path.basename(jj) - os.chdir(task_old_path) - os.symlink(os.path.relpath(absjj), basejj) - os.chdir(cwd) + _link_old_models(work_path, old_model_files, ii) + else: + if type(training_iter0_model) == str: + training_iter0_model = [training_iter0_model] + iter0_models = [] + for ii in training_iter0_model: + model_is = glob.glob(ii) + model_is.sort() + iter0_models += [os.path.abspath(ii) for ii in model_is] + if training_init_model: + assert(numb_models == len(iter0_models)), "training_iter0_model should be provided, and the number of models should be equal to %d" % numb_models + for ii in range(len(iter0_models)): + old_model_files = glob.glob(os.path.join(iter0_models[ii], 'model.ckpt*')) + _link_old_models(work_path, old_model_files, ii) + + +def _link_old_models(work_path, old_model_files, ii): + """ + link the `ii`th old model given by `old_model_files` to + the `ii`th training task in `work_path` + """ + task_path = os.path.join(work_path, train_task_fmt % ii) + task_old_path = os.path.join(task_path, 'old') + create_path(task_old_path) + cwd = os.getcwd() + for jj in old_model_files: + absjj = os.path.abspath(jj) + basejj = os.path.basename(jj) + os.chdir(task_old_path) + os.symlink(os.path.relpath(absjj), basejj) + os.chdir(cwd) + def detect_batch_size(batch_size, system=None): if type(batch_size) == int: @@ -352,16 +412,19 @@ def run_train (iter_index, numb_models = jdata['numb_models'] # train_param = jdata['train_param'] train_input_file = default_train_input_file + training_reuse_iter = jdata.get('training_reuse_iter') + training_init_model = jdata.get('training_init_model', False) + if training_reuse_iter is not None and iter_index >= training_reuse_iter: + training_init_model = True try: mdata["deepmd_version"] - except: + except KeyError: mdata = set_version(mdata) if LooseVersion(mdata["deepmd_version"]) < LooseVersion('1'): # 0.x deepmd_path = mdata['deepmd_path'] else: # 1.x - python_path = mdata.get('python_path', None) train_command = mdata.get('train_command', 'dp') train_resources = mdata['train_resources'] @@ -385,22 +448,20 @@ def run_train (iter_index, commands.append(command) command = os.path.join(deepmd_path, 'bin/dp_frz') commands.append(command) - elif python_path: - # 1.x - command = '%s -m deepmd train %s' % (python_path, train_input_file) - commands.append(command) - command = '%s -m deepmd freeze' % python_path - commands.append(command) else: + # 1.x ## Commands are like `dp train` and `dp freeze` ## train_command should not be None assert(train_command) command = '%s train %s' % (train_command, train_input_file) + if training_init_model: + command = "{ if [ ! -f model.ckpt.index ]; then %s --init-model old/model.ckpt; else %s --restart model.ckpt; fi }" % (command, command) + else: + command = "{ if [ ! -f model.ckpt.index ]; then %s; else %s --restart model.ckpt; fi }" % (command, command) commands.append(command) command = '%s freeze' % train_command commands.append(command) - #_tasks = [os.path.basename(ii) for ii in all_task] # run_tasks = [] # for ii in all_task: @@ -413,7 +474,13 @@ def run_train (iter_index, run_tasks = [os.path.basename(ii) for ii in all_task] forward_files = [train_input_file] + if training_init_model: + forward_files += [os.path.join('old', 'model.ckpt.meta'), + os.path.join('old', 'model.ckpt.index'), + os.path.join('old', 'model.ckpt.data-00000-of-00001') + ] backward_files = ['frozen_model.pb', 'lcurve.out', 'train.log'] + backward_files+= ['model.ckpt.meta', 'model.ckpt.index', 'model.ckpt.data-00000-of-00001', 'checkpoint'] init_data_sys_ = jdata['init_data_sys'] init_data_sys = [] for ii in init_data_sys_ : @@ -429,21 +496,21 @@ def run_train (iter_index, if jdata.get('init_multi_systems', False): for single_sys in os.listdir(os.path.join(ii)): trans_comm_data += glob.glob(os.path.join(ii, single_sys, 'set.*')) - trans_comm_data += glob.glob(os.path.join(ii, single_sys, 'type.raw')) + trans_comm_data += glob.glob(os.path.join(ii, single_sys, 'type*.raw')) trans_comm_data += glob.glob(os.path.join(ii, single_sys, 'nopbc')) else: trans_comm_data += glob.glob(os.path.join(ii, 'set.*')) - trans_comm_data += glob.glob(os.path.join(ii, 'type.raw')) + trans_comm_data += glob.glob(os.path.join(ii, 'type*.raw')) trans_comm_data += glob.glob(os.path.join(ii, 'nopbc')) for ii in fp_data : if jdata.get('use_clusters', False): for single_sys in os.listdir(os.path.join(ii)): trans_comm_data += glob.glob(os.path.join(ii, single_sys, 'set.*')) - trans_comm_data += glob.glob(os.path.join(ii, single_sys, 'type.raw')) + trans_comm_data += glob.glob(os.path.join(ii, single_sys, 'type*.raw')) trans_comm_data += glob.glob(os.path.join(ii, single_sys, 'nopbc')) else: trans_comm_data += glob.glob(os.path.join(ii, 'set.*')) - trans_comm_data += glob.glob(os.path.join(ii, 'type.raw')) + trans_comm_data += glob.glob(os.path.join(ii, 'type*.raw')) trans_comm_data += glob.glob(os.path.join(ii, 'nopbc')) os.chdir(cwd) @@ -452,14 +519,7 @@ def run_train (iter_index, except: train_group_size = 1 - task_chunks = _split_tasks(run_tasks, train_group_size) - nchunks = len(task_chunks) - if "ali_auth" in mdata: - dispatcher = ALI(mdata['ali_auth'], mdata['train_resources'], mdata['train_machine'], nchunks, work_path) - dispatcher.init() - else: - dispatcher = make_dispatcher(mdata['train_machine']) - + dispatcher = make_dispatcher(mdata['train_machine'], mdata['train_resources'], work_path, run_tasks, train_group_size) dispatcher.run_jobs(mdata['train_resources'], commands, work_path, @@ -506,7 +566,7 @@ def parse_cur_job(cur_job) : if 'npt' in ensemble : temps = _get_param_alias(cur_job, ['Ts','temps']) press = _get_param_alias(cur_job, ['Ps','press']) - elif 'nvt' == ensemble : + elif 'nvt' == ensemble or 'nve' == ensemble: temps = _get_param_alias(cur_job, ['Ts','temps']) nsteps = _get_param_alias(cur_job, ['nsteps']) trj_freq = _get_param_alias(cur_job, ['t_freq', 'trj_freq','traj_freq']) @@ -664,6 +724,8 @@ def make_model_devi (iter_index, else: fmt = 'vasp/poscar' system = dpdata.System(os.path.join(conf_path, poscar_name), fmt = fmt, type_map = jdata['type_map']) + if jdata.get('model_devi_nopbc', False): + system.remove_pbc() system.to_lammps_lmp(os.path.join(conf_path, lmp_name)) conf_counter += 1 sys_counter += 1 @@ -672,6 +734,7 @@ def make_model_devi (iter_index, if "template" in cur_job: input_mode = "revise_template" use_plm = jdata.get('model_devi_plumed', False) + use_plm_path = jdata.get('model_devi_plumed_path', False) if input_mode == "native": _make_model_devi_native(iter_index, jdata, mdata, conf_systems) elif input_mode == "revise_template": @@ -692,6 +755,7 @@ def _make_model_devi_revmat(iter_index, jdata, mdata, conf_systems): raise RuntimeError("system index should be uniq") mass_map = jdata['mass_map'] use_plm = jdata.get('model_devi_plumed', False) + use_plm_path = jdata.get('model_devi_plumed_path', False) trj_freq = _get_param_alias(cur_job, ['t_freq', 'trj_freq','traj_freq']) rev_keys, rev_mat, num_lmp = parse_cur_job_revmat(cur_job, use_plm = use_plm) @@ -700,6 +764,9 @@ def _make_model_devi_revmat(iter_index, jdata, mdata, conf_systems): if use_plm: plm_templ = cur_job['template']['plm'] plm_templ = os.path.abspath(plm_templ) + if use_plm_path: + plm_path_templ = cur_job['template']['plm_path'] + plm_path_templ = os.path.abspath(plm_path_templ) iter_name = make_iter_name(iter_index) train_path = os.path.join(iter_name, train_name) @@ -711,7 +778,7 @@ def _make_model_devi_revmat(iter_index, jdata, mdata, conf_systems): work_path = os.path.join(iter_name, model_devi_name) try: mdata["deepmd_version"] - except: + except KeyError: mdata = set_version(mdata) deepmd_version = mdata['deepmd_version'] @@ -751,6 +818,8 @@ def _make_model_devi_revmat(iter_index, jdata, mdata, conf_systems): plm_lines = revise_by_keys(plm_lines, rev_keys[num_lmp:], rev_item[num_lmp:]) with open('input.plumed', 'w') as fp: fp.write(''.join(plm_lines)) + if use_plm_path: + shutil.copyfile(plm_path_templ, 'plmpath.pdb') # dump input of lammps with open('input.lammps', 'w') as fp: fp.write(''.join(lmp_lines)) @@ -788,6 +857,7 @@ def _make_model_devi_native(iter_index, jdata, mdata, conf_systems): if 'model_devi_taup' in jdata : model_devi_taup = jdata['model_devi_taup'] mass_map = jdata['mass_map'] + nopbc = jdata.get('model_devi_nopbc', False) iter_name = make_iter_name(iter_index) train_path = os.path.join(iter_name, train_name) @@ -840,7 +910,7 @@ def _make_model_devi_native(iter_index, jdata, mdata, conf_systems): os.chdir(task_path) try: mdata["deepmd_version"] - except: + except KeyError: mdata = set_version(mdata) deepmd_version = mdata['deepmd_version'] file_c = make_lammps_input(ensemble, @@ -859,6 +929,7 @@ def _make_model_devi_native(iter_index, jdata, mdata, conf_systems): pka_e = pka_e, ele_temp_f = te_f, ele_temp_a = te_a, + nopbc = nopbc, deepmd_version = deepmd_version) job = {} job["ensemble"] = ensemble @@ -887,6 +958,7 @@ def run_model_devi (iter_index, model_devi_group_size = mdata['model_devi_group_size'] model_devi_resources = mdata['model_devi_resources'] use_plm = jdata.get('model_devi_plumed', False) + use_plm_path = jdata.get('model_devi_plumed_path', False) iter_name = make_iter_name(iter_index) work_path = os.path.join(iter_name, model_devi_name) @@ -919,16 +991,13 @@ def run_model_devi (iter_index, backward_files = ['model_devi.out', 'model_devi.log', 'traj'] if use_plm: forward_files += ['input.plumed'] - backward_files += ['output.plumed'] + # backward_files += ['output.plumed'] + backward_files += ['output.plumed','COLVAR','dump.0.xyz'] + if use_plm_path: + forward_files += ['plmpath.pdb'] cwd = os.getcwd() - task_chunks = _split_tasks(run_tasks, model_devi_group_size) - nchunks = len(task_chunks) - if "ali_auth" in mdata: - dispatcher = ALI(mdata['ali_auth'], mdata['model_devi_resources'], mdata['model_devi_machine'], nchunks, work_path) - dispatcher.init() - else: - dispatcher = make_dispatcher(mdata['model_devi_machine']) + dispatcher = make_dispatcher(mdata['model_devi_machine'], mdata['model_devi_resources'], work_path, run_tasks, model_devi_group_size) dispatcher.run_jobs(mdata['model_devi_resources'], commands, work_path, @@ -946,6 +1015,40 @@ def post_model_devi (iter_index, mdata) : pass + +def _to_face_dist(box_): + box = np.reshape(box_, [3,3]) + vol = np.abs(np.linalg.det(box)) + dists = [] + for [ii,jj] in [[0, 1], [1, 2], [2, 0]]: + vv = np.cross(box[ii], box[jj]) + dists.append(vol / np.linalg.norm(vv)) + return np.array(dists) + +def check_bad_box(conf_name, + criteria, + fmt = 'lammps/dump'): + all_c = criteria.split(';') + sys = dpdata.System(conf_name, fmt) + assert(sys.get_nframes() == 1) + is_bad = False + for ii in all_c: + [key, value] = ii.split(':') + if key == 'length_ratio': + lengths = np.linalg.norm(sys['cells'][0], axis = 1) + ratio = np.max(lengths) / np.min(lengths) + if ratio > float(value): + is_bad = True + elif key == 'height_ratio': + lengths = np.linalg.norm(sys['cells'][0], axis = 1) + dists = _to_face_dist(sys['cells'][0]) + ratio = np.max(lengths) / np.min(dists) + if ratio > float(value): + is_bad = True + else: + raise RuntimeError('unknow key', key) + return is_bad + def _make_fp_vasp_inner (modd_path, work_path, model_devi_skip, @@ -979,6 +1082,8 @@ def _make_fp_vasp_inner (modd_path, cluster_cutoff = jdata['cluster_cutoff'] if jdata.get('use_clusters', False) else None # skip save *.out if detailed_report_make_fp is False, default is True detailed_report_make_fp = jdata.get("detailed_report_make_fp", True) + # skip bad box criteria + skip_bad_box = jdata.get('fp_skip_bad_box') for ss in system_index : fp_candidate = [] if detailed_report_make_fp: @@ -989,6 +1094,9 @@ def _make_fp_vasp_inner (modd_path, modd_system_task.sort() cc = 0 counter = Counter() + counter['candidate'] = 0 + counter['failed'] = 0 + counter['accurate'] = 0 for tt in modd_system_task : with warnings.catch_warnings(): warnings.simplefilter("ignore") @@ -1044,7 +1152,23 @@ def _make_fp_vasp_inner (modd_path, with open(os.path.join(work_path,'rest_failed.shuffled.%s.out'%ss), 'w') as fp: for ii in fp_rest_failed: fp.write(" ".join([str(nn) for nn in ii]) + "\n") - numb_task = min(fp_task_max, len(fp_candidate)) + + # set number of tasks + accurate_ratio = float(counter['accurate']) / float(fp_sum) + fp_accurate_threshold = jdata.get('fp_accurate_threshold', 1) + fp_accurate_soft_threshold = jdata.get('fp_accurate_soft_threshold', fp_accurate_threshold) + if accurate_ratio < fp_accurate_soft_threshold : + this_fp_task_max = fp_task_max + elif accurate_ratio >= fp_accurate_soft_threshold and accurate_ratio < fp_accurate_threshold: + this_fp_task_max = int(fp_task_max * (accurate_ratio - fp_accurate_threshold) / (fp_accurate_soft_threshold - fp_accurate_threshold)) + else: + this_fp_task_max = 0 + numb_task = min(this_fp_task_max, len(fp_candidate)) + if (numb_task < fp_task_min): + numb_task = 0 + dlog.info("system {0:s} accurate_ratio: {1:8.4f} thresholds: {2:6.4f} and {3:6.4f} eff. task min and max {4:4d} {5:4d} number of fp tasks: {6:6d}".format(ss, accurate_ratio, fp_accurate_soft_threshold, fp_accurate_threshold, fp_task_min, this_fp_task_max, numb_task)) + # make fp tasks + count_bad_box = 0 for cc in range(numb_task) : tt = fp_candidate[cc][0] ii = fp_candidate[cc][1] @@ -1052,6 +1176,11 @@ def _make_fp_vasp_inner (modd_path, conf_name = os.path.join(tt, "traj") conf_name = os.path.join(conf_name, str(ii) + '.lammpstrj') conf_name = os.path.abspath(conf_name) + if skip_bad_box is not None: + skip = check_bad_box(conf_name, skip_bad_box) + if skip: + count_bad_box += 1 + continue # link job.json job_name = os.path.join(tt, "job.json") @@ -1078,6 +1207,8 @@ def _make_fp_vasp_inner (modd_path, for pair in fp_link_files : os.symlink(pair[0], pair[1]) os.chdir(cwd) + if count_bad_box > 0: + dlog.info("system {0:s} skipped {1:6d} confs with bad box, {2:6d} remains".format(ss, count_bad_box, numb_task - count_bad_box)) if cluster_cutoff is None: cwd = os.getcwd() for ii in fp_tasks: @@ -1102,6 +1233,65 @@ def make_vasp_incar(jdata, filename): fp.write(incar) return incar +def make_pwmat_input(jdata, filename): + if 'fp_incar' in jdata.keys() : + fp_incar_path = jdata['fp_incar'] + assert(os.path.exists(fp_incar_path)) + fp_incar_path = os.path.abspath(fp_incar_path) + fr = open(fp_incar_path) + input = fr.read() + fr.close() + elif 'user_fp_params' in jdata.keys() : + fp_params = jdata['user_fp_params'] + node1 = fp_params['node1'] + node2 = fp_params['node2'] + atom_config = fp_params['in.atom'] + ecut = fp_params['ecut'] + e_error = fp_params['e_error'] + rho_error = fp_params['rho_error'] + kspacing = fp_params['kspacing'] + flag_symm = fp_params['flag_symm'] + os.system("command -v poscar2config.x | wc -l > 1.txt") + fc = open('1.txt') + flag_command = fc.read() + fc.close() + if int(flag_command) == 1 : + os.system('poscar2config.x < POSCAR > tmp.config') + else: + os.system('cp ../../../out_data_post_fp_pwmat/02.fp/task.000.000000/poscar2config.x ./') + os.system('./poscar2config.x < POSCAR > tmp.config') + os.system('rm -rf tmp.config') + input_dict = make_pwmat_input_dict(node1, node2, atom_config, ecut, e_error, + rho_error, icmix = None, smearing = None, + sigma = None, kspacing = kspacing, + flag_symm = flag_symm + ) + + input = write_input_dict(input_dict) + else: + input = make_pwmat_input_user_dict(jdata['fp_params']) + if 'IN.PSP' in input or 'in.psp' in input: + with open(filename, 'w') as fp: + fp.write(input) + fp.write('job=scf\n') + if 'OUT.MLMD' in input or 'out.mlmd' in input: + return input + else: + fp.write('OUT.MLMD = T') + return input + else: + with open(filename, 'w') as fp: + fp.write(input) + fp.write('job=scf\n') + fp_pp_files = jdata['fp_pp_files'] + for idx, ii in enumerate(fp_pp_files) : + fp.write('IN.PSP%d = %s\n' %(idx+1, ii)) + if 'OUT.MLMD' in input or 'out.mlmd' in input: + return input + else: + fp.write('OUT.MLMD = T') + return input + def make_vasp_incar_ele_temp(jdata, filename, ele_temp, nbands_esti = None): with open(filename) as fp: incar = fp.read() @@ -1116,7 +1306,7 @@ def make_vasp_incar_ele_temp(jdata, filename, ele_temp, nbands_esti = None): incar['NBANDS'] = nbands incar.write_file('INCAR') -def _make_fp_vasp_incar (iter_index, +def make_fp_vasp_incar (iter_index, jdata, nbands_esti = None) : iter_name = make_iter_name(iter_index) @@ -1138,9 +1328,44 @@ def _make_fp_vasp_incar (iter_index, nbands_esti = nbands_esti) os.chdir(cwd) -def _make_fp_vasp_kp (iter_index,jdata): +def _make_fp_pwmat_input (iter_index, + jdata) : iter_name = make_iter_name(iter_index) work_path = os.path.join(iter_name, fp_name) + fp_tasks = glob.glob(os.path.join(work_path, 'task.*')) + fp_tasks.sort() + if len(fp_tasks) == 0 : + return + cwd = os.getcwd() + for ii in fp_tasks: + os.chdir(ii) + make_pwmat_input(jdata, 'etot.input') + os.system("sed -i '1,2c 4 1' etot.input") + os.chdir(cwd) + +def make_fp_vasp_cp_cvasp(iter_index,jdata): + # Move cvasp interface to jdata + if ('cvasp' in jdata) and (jdata['cvasp'] == True): + pass + else: + return + iter_name = make_iter_name(iter_index) + work_path = os.path.join(iter_name, fp_name) + fp_tasks = glob.glob(os.path.join(work_path, 'task.*')) + fp_tasks.sort() + if len(fp_tasks) == 0 : + return + cwd = os.getcwd() + for ii in fp_tasks: + os.chdir(ii) + #copy cvasp.py + shutil.copyfile(cvasp_file, 'cvasp.py') + os.chdir(cwd) + +def make_fp_vasp_kp (iter_index,jdata): + iter_name = make_iter_name(iter_index) + work_path = os.path.join(iter_name, fp_name) + fp_aniso_kspacing = jdata.get('fp_aniso_kspacing') fp_tasks = glob.glob(os.path.join(work_path, 'task.*')) fp_tasks.sort() @@ -1154,10 +1379,13 @@ def _make_fp_vasp_kp (iter_index,jdata): with open('INCAR') as fp: incar = fp.read() standard_incar = incar_upper(Incar.from_string(incar)) - try: - kspacing = standard_incar['KSPACING'] - except: - raise RuntimeError ("KSPACING must be given in INCAR") + if fp_aniso_kspacing is None: + try: + kspacing = standard_incar['KSPACING'] + except KeyError: + raise RuntimeError ("KSPACING must be given in INCAR") + else : + kspacing = fp_aniso_kspacing try: gamma = standard_incar['KGAMMA'] if isinstance(gamma,bool): @@ -1167,7 +1395,7 @@ def _make_fp_vasp_kp (iter_index,jdata): gamma=True else: gamma=False - except: + except KeyError: raise RuntimeError ("KGAMMA must be given in INCAR") # check poscar assert(os.path.exists('POSCAR')) @@ -1253,10 +1481,6 @@ def _make_fp_vasp_configs(iter_index, work_path = os.path.join(iter_name, fp_name) create_path(work_path) - #copy cvasp.py - # Move cvasp interface to jdata - if ('cvasp' in jdata) and (jdata['cvasp'] == True): - shutil.copyfile(cvasp_file, os.path.join(work_path,'cvasp.py')) modd_path = os.path.join(iter_name, model_devi_name) task_min = -1 @@ -1293,9 +1517,11 @@ def make_fp_vasp (iter_index, # 1, create potcar sys_link_fp_vasp_pp(iter_index, jdata) # 2, create incar - _make_fp_vasp_incar(iter_index, jdata, nbands_esti = nbe) + make_fp_vasp_incar(iter_index, jdata, nbands_esti = nbe) # 3, create kpoints - _make_fp_vasp_kp(iter_index, jdata) + make_fp_vasp_kp(iter_index, jdata) + # 4, copy cvasp + make_fp_vasp_cp_cvasp(iter_index,jdata) def make_fp_pwscf(iter_index, @@ -1410,6 +1636,21 @@ def make_fp_cp2k (iter_index, # link pp files _link_fp_vasp_pp(iter_index, jdata) +def make_fp_pwmat (iter_index, + jdata) : + # make config + fp_tasks = _make_fp_vasp_configs(iter_index, jdata) + if len(fp_tasks) == 0 : + return + # abs path for fp_incar if it exists + if 'fp_incar' in jdata: + jdata['fp_incar'] = os.path.abspath(jdata['fp_incar']) + # order is critical! + # 1, link pp files + _link_fp_vasp_pp(iter_index, jdata) + # 2, create pwmat input + _make_fp_pwmat_input(iter_index, jdata) + def make_fp (iter_index, jdata, mdata) : @@ -1425,6 +1666,8 @@ def make_fp (iter_index, make_fp_gaussian(iter_index, jdata) elif fp_style == "cp2k" : make_fp_cp2k(iter_index, jdata) + elif fp_style == "pwmat" : + make_fp_pwmat(iter_index, jdata) else : raise RuntimeError ("unsupported fp style") @@ -1484,6 +1727,17 @@ def _cp2k_check_fin(ii): return False return True +def _pwmat_check_fin (ii) : + if os.path.isfile(os.path.join(ii, 'REPORT')) : + with open(os.path.join(ii, 'REPORT'), 'r') as fp : + content = fp.read() + count = content.count('time') + if count != 1 : + return False + else : + return False + return True + def run_fp_inner (iter_index, jdata, mdata, @@ -1495,6 +1749,7 @@ def run_fp_inner (iter_index, fp_command = mdata['fp_command'] fp_group_size = mdata['fp_group_size'] fp_resources = mdata['fp_resources'] + mark_failure = fp_resources.get('mark_failure', False) iter_name = make_iter_name(iter_index) work_path = os.path.join(iter_name, fp_name) @@ -1509,14 +1764,7 @@ def run_fp_inner (iter_index, # if not check_fin(ii) : # fp_run_tasks.append(ii) run_tasks = [os.path.basename(ii) for ii in fp_run_tasks] - cwd = os.getcwd() - task_chunks = _split_tasks(run_tasks, fp_group_size) - nchunks = len(task_chunks) - if "ali_auth" in mdata: - dispatcher = ALI(mdata['ali_auth'], mdata['fp_resources'], mdata['fp_machine'], nchunks, work_path) - dispatcher.init() - else: - dispatcher = make_dispatcher(mdata['fp_machine']) + dispatcher = make_dispatcher(mdata['fp_machine'], mdata['fp_resources'], work_path, run_tasks, fp_group_size) dispatcher.run_jobs(mdata['fp_resources'], [fp_command], work_path, @@ -1525,6 +1773,7 @@ def run_fp_inner (iter_index, forward_common_files, forward_files, backward_files, + mark_failure = mark_failure, outlog = log_file, errlog = log_file) @@ -1537,15 +1786,15 @@ def run_fp (iter_index, fp_pp_files = jdata['fp_pp_files'] if fp_style == "vasp" : - forward_files = ['POSCAR', 'INCAR', 'POTCAR'] + forward_files = ['POSCAR', 'INCAR', 'POTCAR','KPOINTS'] backward_files = ['OUTCAR','vasprun.xml'] # Move cvasp interface to jdata if ('cvasp' in jdata) and (jdata['cvasp'] == True): mdata['fp_resources']['cvasp'] = True if ('cvasp' in mdata["fp_resources"] ) and (mdata["fp_resources"]["cvasp"]==True): - #dlog.info("cvasp is on !") - forward_common_files=['cvasp.py'] - forward_files.append('KPOINTS') + dlog.info("cvasp is on !") + forward_files.append('cvasp.py') + forward_common_files=[] else: forward_common_files=[] run_fp_inner(iter_index, jdata, mdata, forward_files, backward_files, _vasp_check_fin, @@ -1566,10 +1815,37 @@ def run_fp (iter_index, forward_files = ['input.inp', 'coord.xyz'] backward_files = ['output'] run_fp_inner(iter_index, jdata, mdata, forward_files, backward_files, _cp2k_check_fin, log_file = 'output') + elif fp_style == "pwmat" : + forward_files = ['atom.config', 'etot.input'] + fp_pp_files + backward_files = ['REPORT', 'OUT.MLMD', 'output'] + run_fp_inner(iter_index, jdata, mdata, forward_files, backward_files, _pwmat_check_fin, log_file = 'output') else : raise RuntimeError ("unsupported fp style") +def post_fp_check_fail(iter_index, + jdata, + rfailed = None) : + ratio_failed = rfailed if rfailed else jdata.get('ratio_failed',0.05) + iter_name = make_iter_name(iter_index) + work_path = os.path.join(iter_name, fp_name) + fp_tasks = glob.glob(os.path.join(work_path, 'task.*')) + fp_tasks.sort() + if len(fp_tasks) == 0 : + return + # check fail according to tag_failure + fp_failed_tags = glob.glob(os.path.join(work_path, 'task.*', 'tag_failure*')) + fp_failed_tasks = [os.path.dirname(ii) for ii in fp_failed_tags] + fp_failed_tasks = list(set(fp_failed_tasks)) + + ntask = len(fp_tasks) + nfail = len(fp_failed_tasks) + rfail = float(nfail) / float(ntask) + dlog.info("failed tasks: %6d in %6d %6.2f %% " % (nfail, ntask, rfail * 100.)) + if rfail > ratio_failed: + raise RuntimeError("find too many unsuccessfully terminated jobs") + + def post_fp_vasp (iter_index, jdata, rfailed=None): @@ -1649,12 +1925,10 @@ def post_fp_vasp (iter_index, else: raise RuntimeError('invalid setting of use_ele_temp ' + str(use_ele_temp)) - dlog.info("failed frame number: %s "%icount) - dlog.info("total frame number: %s "%tcount) - reff=icount/tcount - dlog.info('ratio of failed frame: {:.2%}'.format(reff)) + rfail=float(icount)/float(tcount) + dlog.info("failed frame: %6d in %6d %6.2f %% " % (icount, tcount, rfail * 100.)) - if reff>ratio_failed: + if rfail>ratio_failed: raise RuntimeError("find too many unsuccessfully terminated jobs") @@ -1830,9 +2104,64 @@ def post_fp_cp2k (iter_index, all_sys.to_deepmd_npy(sys_data_path, set_size = len(sys_output)) +def post_fp_pwmat (iter_index, + jdata, + rfailed=None): + + ratio_failed = rfailed if rfailed else jdata.get('ratio_failed',0.05) + model_devi_jobs = jdata['model_devi_jobs'] + assert (iter_index < len(model_devi_jobs)) + + iter_name = make_iter_name(iter_index) + work_path = os.path.join(iter_name, fp_name) + fp_tasks = glob.glob(os.path.join(work_path, 'task.*')) + fp_tasks.sort() + if len(fp_tasks) == 0 : + return + + system_index = [] + for ii in fp_tasks : + system_index.append(os.path.basename(ii).split('.')[1]) + system_index.sort() + set_tmp = set(system_index) + system_index = list(set_tmp) + system_index.sort() + + cwd = os.getcwd() + + tcount=0 + icount=0 + for ss in system_index : + sys_output = glob.glob(os.path.join(work_path, "task.%s.*/OUT.MLMD"%ss)) + sys_output.sort() + tcount += len(sys_output) + all_sys = None + for oo in sys_output : + _sys = dpdata.LabeledSystem(oo, type_map = jdata['type_map']) + if len(_sys) == 1: + if all_sys is None: + all_sys = _sys + else: + all_sys.append(_sys) + else: + icount+=1 + if all_sys is not None: + sys_data_path = os.path.join(work_path, 'data.%s'%ss) + all_sys.to_deepmd_raw(sys_data_path) + all_sys.to_deepmd_npy(sys_data_path, set_size = len(sys_output)) + dlog.info("failed frame number: %s "%icount) + dlog.info("total frame number: %s "%tcount) + reff=icount/tcount + dlog.info('ratio of failed frame: {:.2%}'.format(reff)) + + if reff>ratio_failed: + raise RuntimeError("find too many unsuccessfully terminated jobs") + + def post_fp (iter_index, jdata) : fp_style = jdata['fp_style'] + post_fp_check_fail(iter_index, jdata) if fp_style == "vasp" : post_fp_vasp(iter_index, jdata) elif fp_style == "pwscf" : @@ -1843,6 +2172,8 @@ def post_fp (iter_index, post_fp_gaussian(iter_index, jdata) elif fp_style == 'cp2k' : post_fp_cp2k(iter_index, jdata) + elif fp_style == 'pwmat' : + post_fp_pwmat(iter_index, jdata) else : raise RuntimeError ("unsupported fp style") # clean traj diff --git a/dpgen/main.py b/dpgen/main.py index f5aff6583..7d180b596 100644 --- a/dpgen/main.py +++ b/dpgen/main.py @@ -10,6 +10,7 @@ from dpgen.data.gen import gen_init_bulk from dpgen.data.surf import gen_init_surf from dpgen.data.reaction import gen_init_reaction +from dpgen.collect.collect import gen_collect from dpgen.simplify.simplify import gen_simplify from dpgen.auto_test.run import gen_test from dpgen.database.run import db_run @@ -115,6 +116,24 @@ def main(): help="being loud") parser_rr.set_defaults(func=run_report) + # collect + parser_coll = subparsers.add_parser( + "collect", + help="Collect data.") + parser_coll.add_argument("JOB_DIR", type=str, + help="the directory of the DP-GEN job") + parser_coll.add_argument("OUTPUT", type=str, + help="the output directory of data") + parser_coll.add_argument('-p',"--parameter", type=str, default = 'param.json', + help="the json file provides DP-GEN paramters, should be located in JOB_DIR") + parser_coll.add_argument('-v',"--verbose", action = 'store_true', + help="print number of data in each system") + parser_coll.add_argument('-m',"--merge", action = 'store_true', + help="merge the systems with the same chemical formula") + parser_coll.add_argument('-s',"--shuffle", action = 'store_true', + help="shuffle the data systems") + parser_coll.set_defaults(func=gen_collect) + # simplify parser_run = subparsers.add_parser( "simplify", diff --git a/dpgen/remote/RemoteJob.py b/dpgen/remote/RemoteJob.py index 6997c583a..2450b59f5 100644 --- a/dpgen/remote/RemoteJob.py +++ b/dpgen/remote/RemoteJob.py @@ -755,7 +755,8 @@ def submit(self, while self.check_limit(task_max=resources['task_max']): time.sleep(60) self._submit(job_dirs, cmd, args, resources) - time.sleep(20) # For preventing the crash of the tasks while submitting. + if resources.get('wait_time', False): + time.sleep(resources['wait_time']) # For preventing the crash of the tasks while submitting. def _submit(self, job_dirs, @@ -807,11 +808,11 @@ def check_status(self) : status_word = status_line.split()[2] # ref: https://www.ibm.com/support/knowledgecenter/en/SSETD4_9.1.2/lsf_command_ref/bjobs.1.html - if status_word in ["PEND", "WAIT"] : + if status_word in ["PEND", "WAIT", "PSUSP"] : return JobStatus.waiting - elif status_word in ["RUN"] : + elif status_word in ["RUN", "USUSP"] : return JobStatus.running - elif status_word in ["DONE","EXIT"] : + elif status_word in ["DONE","EXIT"] : if self._check_finish_tag() : return JobStatus.finished else : @@ -845,14 +846,19 @@ def _make_script(self, ret = '' ret += "#!/bin/bash -l\n#BSUB -e %J.err\n#BSUB -o %J.out\n" if res['numb_gpu'] == 0: - ret += '#BSUB -R span[ptile=%d]\n#BSUB -n %d\n' % ( - res['node_cpu'], res['numb_node'] * res['task_per_node']) + ret += '#BSUB -n %d\n#BSUB -R span[ptile=%d]\n' % ( + res['numb_node'] * res['task_per_node'], res['node_cpu']) else: if res['node_cpu']: ret += '#BSUB -R span[ptile=%d]\n' % res['node_cpu'] - # It is selected only for the situation that GPU is related to CPU node. - ret += '#BSUB -R "select[ngpus >0] rusage[ngpus_excl_p=1]"\n#BSUB -n %d\n' % ( - res['numb_gpu']) + if 'new_lsf_gpu' in res and res['new_lsf_gpu'] == True: + # supportted in LSF >= 10.1.0 SP6 + # ref: https://www.ibm.com/support/knowledgecenter/en/SSWRJV_10.1.0/lsf_resource_sharing/use_gpu_res_reqs.html + ret += '#BSUB -n %d\n#BSUB -gpu "num=%d:mode=shared:j_exclusive=yes"\n' % ( + res['numb_gpu'], res['task_per_node']) + else: + ret += '#BSUB -n %d\n#BSUB -R "select[ngpus >0] rusage[ngpus_excl_p=%d]"\n' % ( + res['numb_gpu'], res['task_per_node']) if res['time_limit']: ret += '#BSUB -W %s\n' % (res['time_limit'].split(':')[ 0] + ':' + res['time_limit'].split(':')[1]) diff --git a/dpgen/simplify/simplify.py b/dpgen/simplify/simplify.py index ba99b0b6c..9856dc58a 100644 --- a/dpgen/simplify/simplify.py +++ b/dpgen/simplify/simplify.py @@ -15,7 +15,7 @@ import argparse import pickle import glob - +import fnmatch import dpdata import numpy as np @@ -24,7 +24,7 @@ from dpgen.util import sepline from dpgen.remote.decide_machine import decide_train_machine from dpgen.dispatcher.Dispatcher import Dispatcher, make_dispatcher -from dpgen.generator.run import make_train, run_train, post_train, run_fp, post_fp, fp_name, model_devi_name, train_name +from dpgen.generator.run import make_train, run_train, post_train, run_fp, post_fp, fp_name, model_devi_name, train_name, train_task_fmt, sys_link_fp_vasp_pp, make_fp_vasp_incar, make_fp_vasp_kp, make_fp_vasp_cp_cvasp, data_system_fmt, model_devi_task_fmt, fp_task_fmt # TODO: maybe the following functions can be moved to dpgen.util from dpgen.generator.lib.utils import log_iter, make_iter_name, create_path, record_iter from dpgen.remote.decide_machine import decide_train_machine, decide_fp_machine, decide_model_devi_machine @@ -35,6 +35,19 @@ rest_data_name = "data.rest" accurate_data_name = "data.accurate" detail_file_name_prefix = "details" +sys_name_fmt = 'sys.' + data_system_fmt +sys_name_pattern = 'sys.[0-9]*[0-9]' + +def expand_sys_str(root_dir): + matches = [] + for root, dirnames, filenames in os.walk(root_dir, followlinks=True): + for filename in fnmatch.filter(filenames, 'type.raw'): + matches.append(root) + matches.sort() + dirnames = [os.path.basename(ii) for ii in matches] + if (len(list(set(dirnames))) != len(matches)) : + raise RuntimeError('duplicated system name: it is highly recommend to place all systems in the same level of directory and has different names') + return matches def get_system_cls(jdata): @@ -43,29 +56,80 @@ def get_system_cls(jdata): return dpdata.System -def get_systems(path, jdata): +def get_multi_system(path, jdata): system = get_system_cls(jdata) systems = dpdata.MultiSystems( *[system(os.path.join(path, s), fmt='deepmd/npy') for s in os.listdir(path)]) return systems +def get_systems(path, jdata): + system_cls = get_system_cls(jdata) + system_paths = expand_sys_str(path) + systems = {} + for ii in system_paths: + systems[os.path.basename(ii)] = system_cls(ii, fmt='deepmd/npy') + return systems + + +def get_system_idx(path): + system_paths = expand_sys_str(path) + sys_idx_map = {} + for idx,ii in enumerate(system_paths): + sys_idx_map[os.path.basename(ii)] = idx + return sys_idx_map + + +def init_model(iter_index, jdata, mdata): + training_init_model = jdata.get('training_init_model', False) + if not training_init_model: + return + iter0_models = [] + training_iter0_model = jdata.get('training_iter0_model_path', []) + if type(training_iter0_model) == str: + training_iter0_model = [training_iter0_model] + for ii in training_iter0_model: + model_is = glob.glob(ii) + model_is.sort() + iter0_models += [os.path.abspath(ii) for ii in model_is] + numb_models = jdata['numb_models'] + assert(numb_models == len(iter0_models)), "training_iter0_model_path should be provided, and the number of models should be equal to %d" % numb_models + work_path = os.path.join(make_iter_name(iter_index), train_name) + create_path(work_path) + cwd = os.getcwd() + for ii in range(len(iter0_models)): + train_path = os.path.join(work_path, train_task_fmt % ii) + create_path(train_path) + os.chdir(train_path) + ckpt_files = glob.glob(os.path.join(iter0_models[ii], 'model.ckpt*')) + for jj in ckpt_files: + os.symlink(jj, os.path.basename(jj)) + os.chdir(cwd) + + def init_pick(iter_index, jdata, mdata): """pick up init data from dataset randomly""" pick_data = jdata['pick_data'] init_pick_number = jdata['init_pick_number'] + use_clusters = jdata.get('use_clusters', False) # use MultiSystems with System # TODO: support System and LabeledSystem # TODO: support other format - systems = get_systems(pick_data, jdata) + if use_clusters: + systems = get_multi_system(pick_data, jdata) + else: + systems = get_systems(pick_data, jdata) # label the system labels = [] - for key, system in systems.systems.items(): + if use_clusters: + items = systems.systems.items() + else: + items = systems.items() + for key, system in items: labels.extend([(key, j) for j in range(len(system))]) # random pick iter_name = make_iter_name(iter_index) - create_path(iter_name) work_path = os.path.join(iter_name, model_devi_name) create_path(work_path) idx = np.arange(len(labels)) @@ -74,27 +138,56 @@ def init_pick(iter_index, jdata, mdata): rest_idx = idx[init_pick_number:] # dump the init data - picked_systems = dpdata.MultiSystems() - for j in pick_idx: - sys_name, sys_id = labels[j] - picked_systems.append(systems[sys_name][sys_id]) sys_data_path = os.path.join(work_path, picked_data_name) - - picked_systems.to_deepmd_raw(sys_data_path) - picked_systems.to_deepmd_npy(sys_data_path, set_size=init_pick_number) + _init_dump_selected_frames(systems, labels, pick_idx, sys_data_path, jdata) # dump the rest data - rest_systems = dpdata.MultiSystems() - for j in rest_idx: - sys_name, sys_id = labels[j] - rest_systems.append(systems[sys_name][sys_id]) sys_data_path = os.path.join(work_path, rest_data_name) - rest_systems.to_deepmd_raw(sys_data_path) - rest_systems.to_deepmd_npy(sys_data_path, set_size=rest_idx.size) + _init_dump_selected_frames(systems, labels, rest_idx, sys_data_path, jdata) + + +def _add_system(systems, key, system): + if key in systems.keys(): + systems[key].append(system) + else: + systems[key] = system + return systems + + +def _init_dump_selected_frames(systems, labels, selc_idx, sys_data_path, jdata): + pick_data = jdata['pick_data'] + use_clusters = jdata.get('use_clusters', False) + if use_clusters: + selc_systems = dpdata.MultiSystems() + for j in selc_idx: + sys_name, sys_id = labels[j] + selc_systems.append(systems[sys_name][sys_id]) + selc_systems.to_deepmd_raw(sys_data_path) + selc_systems.to_deepmd_npy(sys_data_path, set_size=selc_idx.size) + else: + selc_systems = {} + for j in selc_idx: + sys_name, sys_id = labels[j] + selc_systems = _add_system(selc_systems, sys_name, systems[sys_name][sys_id]) + sys_idx_map = get_system_idx(pick_data) + for kk in selc_systems.keys(): + sub_path = os.path.join(sys_data_path, sys_name_fmt % sys_idx_map[kk]) + selc_systems[kk].to_deepmd_raw(sub_path) + selc_systems[kk].to_deepmd_npy(sub_path, set_size=selc_idx.size) + with open(os.path.join(sys_data_path, 'sys_idx_map.json'), 'w') as fp: + json.dump(sys_idx_map, fp, indent=4) + +def _dump_system_dict(systems, path): + for kk in systems: + sub_path = os.path.join(path, sys_name_fmt % (int(kk))) + systems[kk].to_deepmd_raw(sub_path) + systems[kk].to_deepmd_npy(sub_path, set_size=systems[kk].get_nframes()) def make_model_devi(iter_index, jdata, mdata): """calculate the model deviation of the rest idx""" + pick_data = jdata['pick_data'] + use_clusters = jdata.get('use_clusters', False) iter_name = make_iter_name(iter_index) work_path = os.path.join(iter_name, model_devi_name) create_path(work_path) @@ -110,16 +203,29 @@ def make_model_devi(iter_index, jdata, mdata): rest_data_path = os.path.join(last_iter_name, model_devi_name, rest_data_name) if not os.path.exists(rest_data_path): return False - for jj, subsystem in enumerate(os.listdir(rest_data_path)): - task_name = "task.%03d.%06d" % (0, jj) - task_path = os.path.join(work_path, task_name) - create_path(task_path) - os.symlink(os.path.abspath(os.path.join(rest_data_path, subsystem)), - os.path.abspath(os.path.join(task_path, rest_data_name))) + if use_clusters: + for jj, subsystem in enumerate(os.listdir(rest_data_path)): + task_name = "task." + model_devi_task_fmt % (0, jj) + task_path = os.path.join(work_path, task_name) + create_path(task_path) + os.symlink(os.path.abspath(os.path.join(rest_data_path, subsystem)), + os.path.abspath(os.path.join(task_path, rest_data_name))) + else: + rest_data_path = os.path.abspath(rest_data_path) + sys_path = glob.glob(os.path.join(rest_data_path, sys_name_pattern)) + cwd = os.getcwd() + for ii in sys_path: + task_name = "task." + model_devi_task_fmt % (int(os.path.basename(ii).split('.')[1]), 0) + task_path = os.path.join(work_path, task_name) + create_path(task_path) + os.chdir(task_path) + os.symlink(os.path.relpath(ii), rest_data_name) + os.chdir(cwd) + os.chdir(cwd) return True -def run_model_devi(iter_index, jdata, mdata, dispatcher): +def run_model_devi(iter_index, jdata, mdata): """submit dp test tasks""" iter_name = make_iter_name(iter_index) work_path = os.path.join(iter_name, model_devi_name) @@ -163,6 +269,7 @@ def run_model_devi(iter_index, jdata, mdata, dispatcher): forward_files = [rest_data_name] backward_files = sum([[pf+".e.out", pf+".f.out", pf+".v.out"] for pf in detail_file_names], []) + dispatcher = make_dispatcher(mdata['model_devi_machine'], mdata['model_devi_resources'], work_path, run_tasks, model_devi_group_size) dispatcher.run_jobs(mdata['model_devi_resources'], commands, work_path, @@ -177,20 +284,31 @@ def run_model_devi(iter_index, jdata, mdata, dispatcher): def post_model_devi(iter_index, jdata, mdata): """calculate the model deviation""" + use_clusters = jdata.get('use_clusters', False) iter_name = make_iter_name(iter_index) work_path = os.path.join(iter_name, model_devi_name) tasks = glob.glob(os.path.join(work_path, "task.*")) + tasks.sort() e_trust_lo = jdata['e_trust_lo'] e_trust_hi = jdata['e_trust_hi'] f_trust_lo = jdata['f_trust_lo'] f_trust_hi = jdata['f_trust_hi'] - sys_accurate = dpdata.MultiSystems() - sys_candinate = dpdata.MultiSystems() - sys_failed = dpdata.MultiSystems() + if use_clusters: + sys_accurate = dpdata.MultiSystems() + sys_candinate = dpdata.MultiSystems() + sys_failed = dpdata.MultiSystems() + else: + sys_accurate = {} + sys_candinate = {} + sys_failed = {} + all_names = set() for task in tasks: + if not use_clusters: + sys_name = os.path.basename(task).split('.')[1] + all_names.add(sys_name) # e.out details_e = glob.glob(os.path.join(task, "{}.*.e.out".format(detail_file_name_prefix))) e_all = np.array([np.loadtxt(detail_e, ndmin=2)[:, 1] for detail_e in details_e]) @@ -211,86 +329,233 @@ def post_model_devi(iter_index, jdata, mdata): system_cls = get_system_cls(jdata) for subsys, e_devi, f_devi in zip(system_cls(os.path.join(task, rest_data_name), fmt='deepmd/npy'), e_std, f_std): if (e_devi < e_trust_hi and e_devi >= e_trust_lo) or (f_devi < f_trust_hi and f_devi >= f_trust_lo) : - sys_candinate.append(subsys) + if use_clusters: + sys_candinate.append(subsys) + else: + sys_candinate = _add_system(sys_candinate, sys_name, subsys) elif (e_devi >= e_trust_hi ) or (f_devi >= f_trust_hi ): - sys_failed.append(subsys) + if use_clusters: + sys_failed.append(subsys) + else: + sys_failed = _add_system(sys_failed, sys_name, subsys) elif (e_devi < e_trust_lo and f_devi < f_trust_lo ): - sys_accurate.append(subsys) - counter = {"candidate": sys_candinate.get_nframes(), "accurate": sys_accurate.get_nframes(), "failed": sys_failed.get_nframes()} - fp_sum = sum(counter.values()) - for cc_key, cc_value in counter.items(): - dlog.info("{0:9s} : {1:6d} in {2:6d} {3:6.2f} %".format(cc_key, cc_value, fp_sum, cc_value/fp_sum*100)) + if use_clusters: + sys_accurate.append(subsys) + else: + sys_accurate = _add_system(sys_accurate, sys_name, subsys) + else: + raise RuntimeError('reach a place that should NOT be reached...') + if use_clusters: + counter = {"candidate": sys_candinate.get_nframes(), "accurate": sys_accurate.get_nframes(), "failed": sys_failed.get_nframes()} + fp_sum = sum(counter.values()) + for cc_key, cc_value in counter.items(): + dlog.info("{0:9s} : {1:6d} in {2:6d} {3:6.2f} %".format(cc_key, cc_value, fp_sum, cc_value/fp_sum*100)) + else: + all_names = list(all_names) + all_names.sort() + counter = {"candidate": 0, "accurate": 0, "failed": 0} + for kk in all_names: + sys_counter = {"candidate": 0, "accurate": 0, "failed": 0} + if kk in sys_candinate.keys(): + sys_counter['candidate'] += sys_candinate[kk].get_nframes() + if kk in sys_accurate.keys(): + sys_counter['accurate'] += sys_accurate[kk].get_nframes() + if kk in sys_failed.keys(): + sys_counter['failed'] += sys_failed[kk].get_nframes() + fp_sum = sum(sys_counter.values()) + for cc_key, cc_value in sys_counter.items(): + if fp_sum != 0: + dlog.info("sys{0:s} {1:9s} : {2:6d} in {3:6d} {4:6.2f} %".format(kk, cc_key, cc_value, fp_sum, cc_value/fp_sum*100)) + else: + dlog.info("sys{0:s} {1:9s} : {2:6d} in {3:6d} {4:6.2f} %".format(kk, cc_key, cc_value, fp_sum, 0*100)) + for ii in ['candidate', 'accurate', 'failed']: + counter[ii] += sys_counter[ii] + if counter['candidate'] == 0 and counter['failed'] > 0: + raise RuntimeError('no candidate but still have failed cases, stop. You may want to refine the training or to increase the trust level hi') + # label the candidate system labels = [] - for key, system in sys_candinate.systems.items(): + if use_clusters: + items = sys_candinate.systems.items() + else: + items = sys_candinate.items() + for key, system in items: labels.extend([(key, j) for j in range(len(system))]) # candinate: pick up randomly iter_pick_number = jdata['iter_pick_number'] idx = np.arange(counter['candidate']) + assert(len(idx) == len(labels)) np.random.shuffle(idx) pick_idx = idx[:iter_pick_number] rest_idx = idx[iter_pick_number:] + dlog.info("total candidate {0:6d} picked {1:6d} ({2:6.2f} %) rest {3:6d} ({4:6.2f} % )".format\ + (counter['candidate'], len(pick_idx), float(len(pick_idx))/counter['candidate']*100., len(rest_idx), float(len(rest_idx))/counter['candidate']*100.)) # dump the picked candinate data - picked_systems = dpdata.MultiSystems() - for j in pick_idx: - sys_name, sys_id = labels[j] - picked_systems.append(sys_candinate[sys_name][sys_id]) - sys_data_path = os.path.join(work_path, picked_data_name) - - picked_systems.to_deepmd_raw(sys_data_path) - picked_systems.to_deepmd_npy(sys_data_path, set_size=iter_pick_number) + if use_clusters: + picked_systems = dpdata.MultiSystems() + for j in pick_idx: + sys_name, sys_id = labels[j] + picked_systems.append(sys_candinate[sys_name][sys_id]) + sys_data_path = os.path.join(work_path, picked_data_name) + picked_systems.to_deepmd_raw(sys_data_path) + picked_systems.to_deepmd_npy(sys_data_path, set_size=iter_pick_number) + else: + selc_systems = {} + for j in pick_idx: + sys_name, sys_id = labels[j] + selc_systems = _add_system(selc_systems, sys_name, sys_candinate[sys_name][sys_id]) + sys_data_path = os.path.join(work_path, picked_data_name) + _dump_system_dict(selc_systems, sys_data_path) # dump the rest data (not picked candinate data and failed data) - rest_systems = dpdata.MultiSystems() - for j in rest_idx: - sys_name, sys_id = labels[j] - rest_systems.append(sys_candinate[sys_name][sys_id]) - rest_systems += sys_failed - sys_data_path = os.path.join(work_path, rest_data_name) - rest_systems.to_deepmd_raw(sys_data_path) - rest_systems.to_deepmd_npy(sys_data_path, set_size=rest_idx.size) + if use_clusters: + rest_systems = dpdata.MultiSystems() + for j in rest_idx: + sys_name, sys_id = labels[j] + rest_systems.append(sys_candinate[sys_name][sys_id]) + rest_systems += sys_failed + sys_data_path = os.path.join(work_path, rest_data_name) + rest_systems.to_deepmd_raw(sys_data_path) + rest_systems.to_deepmd_npy(sys_data_path, set_size=rest_idx.size) + else: + selc_systems = {} + for j in rest_idx: + sys_name, sys_id = labels[j] + selc_systems = _add_system(selc_systems, sys_name, sys_candinate[sys_name][sys_id]) + for kk in sys_failed.keys(): + selc_systems = _add_system(selc_systems, kk, sys_failed[kk]) + sys_data_path = os.path.join(work_path, rest_data_name) + _dump_system_dict(selc_systems, sys_data_path) # dump the accurate data -- to another directory - sys_data_path = os.path.join(work_path, accurate_data_name) - sys_accurate.to_deepmd_raw(sys_data_path) - sys_accurate.to_deepmd_npy(sys_data_path, set_size=sys_accurate.get_nframes()) + if use_clusters: + sys_data_path = os.path.join(work_path, accurate_data_name) + sys_accurate.to_deepmd_raw(sys_data_path) + sys_accurate.to_deepmd_npy(sys_data_path, set_size=sys_accurate.get_nframes()) + else: + sys_data_path = os.path.join(work_path, accurate_data_name) + _dump_system_dict(sys_accurate, sys_data_path) -def make_fp(iter_index, jdata, mdata): +def make_fp_labeled(iter_index, jdata): + dlog.info("already labeled, skip make_fp and link data directly") + pick_data = jdata['pick_data'] + use_clusters = jdata.get('use_clusters', False) iter_name = make_iter_name(iter_index) work_path = os.path.join(iter_name, fp_name) create_path(work_path) picked_data_path = os.path.join(iter_name, model_devi_name, picked_data_name) - if jdata.get("labeled", False): - dlog.info("already labeled, skip make_fp and link data directly") + if use_clusters: os.symlink(os.path.abspath(picked_data_path), os.path.abspath( - os.path.join(work_path, "task.%03d" % 0))) + os.path.join(work_path, "task." + data_system_fmt % 0))) os.symlink(os.path.abspath(picked_data_path), os.path.abspath( - os.path.join(work_path, "data.%03d" % 0))) - return - systems = get_systems(picked_data_path, jdata) - fp_style = jdata['fp_style'] + os.path.join(work_path, "data." + data_system_fmt % 0))) + else: + picked_data_path = os.path.abspath(picked_data_path) + sys_path = glob.glob(os.path.join(picked_data_path, sys_name_pattern)) + cwd = os.getcwd() + os.chdir(work_path) + for ii in sys_path: + sys_idx = os.path.basename(ii).split('.')[1] + data_dir = 'data.' + data_system_fmt % int(sys_idx) + task_dir = 'task.' + data_system_fmt % int(sys_idx) + os.symlink(os.path.relpath(ii), data_dir) + os.symlink(os.path.relpath(ii), task_dir) + os.chdir(cwd) + + +def make_fp_configs(iter_index, jdata): + pick_data = jdata['pick_data'] + use_clusters = jdata.get('use_clusters', False) + iter_name = make_iter_name(iter_index) + work_path = os.path.join(iter_name, fp_name) + create_path(work_path) + picked_data_path = os.path.join(iter_name, model_devi_name, picked_data_name) + if use_clusters: + systems = get_multi_system(picked_data_path, jdata) + jj = 0 + for system in systems: + for subsys in system: + task_name = "task." + fp_task_fmt % (0, jj) + task_path = os.path.join(work_path, task_name) + create_path(task_path) + subsys.to('vasp/poscar', os.path.join(task_path, 'POSCAR')) + jj += 1 + else: + picked_data_path = os.path.abspath(picked_data_path) + sys_path = glob.glob(os.path.join(picked_data_path, sys_name_pattern)) + for ii in sys_path: + tmp_sys = dpdata.System(ii, fmt = 'deepmd/npy') + sys_idx = os.path.basename(ii).split('.')[1] + jj = 0 + for ss in tmp_sys: + task_name = "task." + fp_task_fmt % (int(sys_idx), jj) + task_path = os.path.join(work_path, task_name) + create_path(task_path) + ss.to('vasp/poscar', os.path.join(task_path, 'POSCAR')) + job = {} + with open(os.path.join(task_path, 'job.json'), 'w') as fp: + json.dump(job, fp, indent=4) + jj += 1 + + +def make_fp_gaussian(iter_index, jdata): + work_path = os.path.join(make_iter_name(iter_index), fp_name) + fp_tasks = glob.glob(os.path.join(work_path, 'task.*')) + cwd = os.getcwd() if 'user_fp_params' in jdata.keys() : fp_params = jdata['user_fp_params'] else: fp_params = jdata['fp_params'] - jj = 0 - for system in systems: - for subsys in system: - sys_data = subsys.data - task_name = "task.%03d.%06d" % (0, jj) - task_path = os.path.join(work_path, task_name) - create_path(task_path) - if fp_style == "gaussian" : - ret = make_gaussian_input(sys_data, fp_params) - with open(os.path.join(task_path, 'input'), 'w') as fp: - fp.write(ret) - else : - # TODO: support other formats - raise RuntimeError ("unsupported fp style") - jj += 1 + cwd = os.getcwd() + for ii in fp_tasks: + os.chdir(ii) + sys_data = dpdata.System('POSCAR').data + ret = make_gaussian_input(sys_data, fp_params) + with open('input', 'w') as fp: + fp.write(ret) + os.chdir(cwd) + + +def make_fp_vasp(iter_index, jdata): + # abs path for fp_incar if it exists + if 'fp_incar' in jdata: + jdata['fp_incar'] = os.path.abspath(jdata['fp_incar']) + # get nbands esti if it exists + if 'fp_nbands_esti_data' in jdata: + nbe = NBandsEsti(jdata['fp_nbands_esti_data']) + else: + nbe = None + # order is critical! + # 1, create potcar + sys_link_fp_vasp_pp(iter_index, jdata) + # 2, create incar + make_fp_vasp_incar(iter_index, jdata, nbands_esti = nbe) + # 3, create kpoints + make_fp_vasp_kp(iter_index, jdata) + # 4, copy cvasp + make_fp_vasp_cp_cvasp(iter_index,jdata) + + +def make_fp_calculation(iter_index, jdata): + fp_style = jdata['fp_style'] + if fp_style == 'vasp': + make_fp_vasp(iter_index, jdata) + elif fp_style == 'gaussian': + make_fp_gaussian(iter_index, jdata) + else : + raise RuntimeError('unsupported fp_style ' + fp_style) + + +def make_fp(iter_index, jdata, mdata): + labeled = jdata.get("labeled", False) + if labeled: + make_fp_labeled(iter_index, jdata) + else: + make_fp_configs(iter_index, jdata) + make_fp_calculation(iter_index, jdata) def run_iter(param_file, machine_file): @@ -365,6 +630,7 @@ def run_iter(param_file, machine_file): if ii == 0 and jj < 6: if jj == 0: log_iter("init_pick", ii, jj) + init_model(ii, jdata, mdata) init_pick(ii, jdata, mdata) dlog.info("first iter, skip step 1-5") elif jj == 0: @@ -373,7 +639,7 @@ def run_iter(param_file, machine_file): elif jj == 1: log_iter("run_train", ii, jj) mdata = decide_train_machine(mdata) - disp = make_dispatcher(mdata['train_machine']) + #disp = make_dispatcher(mdata['train_machine']) run_train(ii, jdata, mdata) elif jj == 2: log_iter("post_train", ii, jj) @@ -386,8 +652,8 @@ def run_iter(param_file, machine_file): elif jj == 4: log_iter("run_model_devi", ii, jj) mdata = decide_model_devi_machine(mdata) - disp = make_dispatcher(mdata['model_devi_machine']) - run_model_devi(ii, jdata, mdata, disp) + #disp = make_dispatcher(mdata['model_devi_machine']) + run_model_devi(ii, jdata, mdata) elif jj == 5: log_iter("post_model_devi", ii, jj) post_model_devi(ii, jdata, mdata) @@ -400,7 +666,7 @@ def run_iter(param_file, machine_file): dlog.info("already have labeled data, skip run_fp") else: mdata = decide_fp_machine(mdata) - disp = make_dispatcher(mdata['fp_machine']) + #disp = make_dispatcher(mdata['fp_machine']) run_fp(ii, jdata, mdata) elif jj == 8: log_iter("post_fp", ii, jj) diff --git a/examples/machine/DeePMD-kit-1.0/machine-ali.json b/examples/machine/DeePMD-kit-1.0/machine-ali.json new file mode 100644 index 000000000..e539ee74e --- /dev/null +++ b/examples/machine/DeePMD-kit-1.0/machine-ali.json @@ -0,0 +1,115 @@ +{ + "train": [ + { + "machine": { + "batch": "shell", + "hostname": "", + "password": "975481DING!", + "port": 22, + "username": "root", + "work_path": "/root/dpgen_work", + "ali_auth": { + "AccessKey_ID":"", + "AccessKey_Secret":"", + "regionID": "cn-shenzhen", + "img_name": "kit", + "machine_type_price": [ + {"machine_type": "ecs.gn6v-c8g1.2xlarge", "price_limit": 20.00, "numb": 1, "priority": 0}, + {"machine_type": "ecs.gn5-c4g1.xlarge", "price_limit": 20.00, "numb": 1, "priority": 1} + ], + "instance_name": "CH4", + "pay_strategy": "spot" + } + }, + "resources": { + "numb_gpu": 1, + "numb_node": 1, + "task_per_node": 12, + "partition": "gpu", + "exclude_list": [], + "mem_limit": 32, + "source_list": [], + "module_list": [], + "time_limit": "23:0:0" + }, + "command": "/root/deepmd-kit/bin/dp", + "group_size": 2 + } + ], + + "model_devi": [ + { + "machine": { + "batch": "shell", + "hostname": "", + "password": "975481DING!", + "port": 22, + "username": "root", + "work_path": "/root/dpgen_work", + "ali_auth": { + "AccessKey_ID":"", + "AccessKey_Secret":"", + "regionID": "cn-shenzhen", + "img_name": "kit", + "machine_type_price": [ + {"machine_type": "ecs.gn6v-c8g1.2xlarge", "price_limit": 20.00, "numb": 1, "priority": 0}, + {"machine_type": "ecs.gn5-c4g1.xlarge", "price_limit": 20.00, "numb": 1, "priority": 1} + ], + "instance_name": "CH4", + "pay_strategy": "spot" + } + }, + "resources": { + "numb_gpu": 1, + "task_per_node": 4, + "partition": "gpu", + "exclude_list": [], + "mem_limit": 11, + "source_list": [], + "module_list": [], + "time_limit": "23:0:0" + }, + "command": "/root/deepmd-kit/bin/lmp", + "group_size": 2 + } + ], + + "fp": [ + { + "machine": { + "batch": "shell", + "hostname": "", + "password": "975481DING!", + "port": 22, + "username": "root", + "work_path": "/root/dpgen_work", + "ali_auth": { + "AccessKey_ID":"", + "AccessKey_Secret":"", + "regionID": "cn-shenzhen", + "img_name": "vasp", + "machine_type_price": [ + {"machine_type": "ecs.c6.4xlarge", "price_limit": 0.2, "numb": 16, "priority": 0}, + {"machine_type": "ecs.g6.4xlarge", "price_limit": 0.2, "numb": 16, "priority": 1} + ], + "instance_name": "CH4", + "pay_strategy": "spot" + } + }, + "resources": { + "numb_gpu": 0, + "task_per_node": 16, + "with_mpi": "false", + "source_list": ["/opt/intel/parallel_studio_xe_2018/psxevars.sh"], + "module_list": [], + "partition": "cpu", + "envs" : {"PATH" : "/root/deepmd-pkg/vasp.5.4.4/bin:$PATH"} + }, + "command": "mpirun -n 16 /root/deepmd-pkg/vasp.5.4.4/bin/vasp_std", + "group_size": 1 + } + ] +} + + + diff --git a/examples/run/dp-lammps-pwmat/machine-slurm-pwmat-single.json b/examples/run/dp-lammps-pwmat/machine-slurm-pwmat-single.json new file mode 100644 index 000000000..9138f0cbf --- /dev/null +++ b/examples/run/dp-lammps-pwmat/machine-slurm-pwmat-single.json @@ -0,0 +1,80 @@ +{ + "train": [ + { + "machine": { + "machine_type": "slurm", + "hostname": "mstation", + "port": 22, + "username": "test", + "password": "PWmat2019", + "work_path": "/home/test/software/dpgen/examples/run/dp-lammps-pwmat/work_train" + }, + "resources": { + "numb_node": 1, + "numb_gpu": 4, + "task_per_node": 4, + "partition": "control", + "exclude_list": [], + "source_list": [ + "/home/test/software/dpgen/examples/run/dp-lammps-pwmat/train.env" + ], + "module_list": [], + "time_limit": "23:0:0" + }, + "deepmd_path" : "/home/test/anaconda2/envs/python3/" + } + ], + "model_devi": [ + { + "machine": { + "machine_type": "slurm", + "hostname": "mstation", + "port": 22, + "username": "test", + "password": "PWmat2019", + "work_path": "/home/test/software/dpgen/examples/run/dp-lammps-pwmat/work_model" + }, + "resources": { + "numb_node": 1, + "numb_gpu": 4, + "task_per_node": 4, + "partition": "control", + "exclude_list": [], + "source_list": [ + "/home/test/software/dpgen/examples/run/dp-lammps-pwmat/train.env" + ], + "module_list": [], + "time_limit": "23:0:0" + }, + "command": "srun --mpi=pmi2 lmp_mpi", + "group_size": 10 + } + ], + "fp": [ + { + "machine": { + "machine_type": "slurm", + "hostname": "mstation", + "port": 22, + "username": "test", + "password": "PWmat2019", + "work_path": "/home/test/software/dpgen/examples/run/dp-lammps-pwmat/work_fp" + }, + "resources": { + "task_per_node": 4, + "numb_gpu": 4, + "exclude_list": [], + "with_mpi": false, + "source_list": [], + "module_list": [ + "cuda/8.0" + ], + "time_limit": "120:0:0", + "partition": "control", + "_comment": "that's All" + }, + "command": "mpirun -np 4 PWmat", + "group_size": 5 + } + ] +} diff --git a/examples/run/dp-lammps-pwmat/param_CH4.json b/examples/run/dp-lammps-pwmat/param_CH4.json new file mode 100644 index 000000000..cda28a5ec --- /dev/null +++ b/examples/run/dp-lammps-pwmat/param_CH4.json @@ -0,0 +1,99 @@ +{ + "type_map": ["H","C"], + "mass_map": [1, 12], + + "init_data_prefix": "/home/test/software/dpgen/examples/run/dp-lammps-pwmat/", + + "init_data_sys": [ + "ch4/00.data" + ], + "init_batch_size": [ + 8 + ], + "sys_configs_prefix": + "/home/test/software/dpgen/examples/run/dp-lammps-pwmat/", + "sys_configs": [ + ["/home/test/software/dpgen/examples/run/dp-lammps-pwmat/scale-1.000/00000*/POSCAR"], + ["/home/test/software/dpgen/examples/run/dp-lammps-pwmat/scale-1.000/00001*/POSCAR"] + ], + + "sys_batch_size": [ + 8, 8, 8, 8 + ], + + + "_comment": " 00.train ", + "numb_models": 4, + "default_training_param" : { + "_comment": " model parameters", + "use_smooth": true, + "sel_a": [16,4], + "rcut_smth": 0.50, + "rcut": 5, + "filter_neuron": [10, 20, 40], + "filter_resnet_dt": false, + "n_axis_neuron": 12, + "n_neuron": [120,120,120], + "resnet_dt": true, + "coord_norm": true, + "type_fitting_net": false, + + "_comment": " traing controls", + "systems": ["./00.data/"], + "set_prefix": "set", + "stop_batch": 2000, + "batch_size": 1, + "start_lr": 0.001, + "decay_steps": 100, + "decay_rate": 0.95, + "seed": 0, + + "start_pref_e": 0.02, + "limit_pref_e": 2, + "start_pref_f": 1000, + "limit_pref_f": 1, + "start_pref_v": 0.0, + "limit_pref_v": 0.0, + + "_comment": " display and restart", + "_comment": " frequencies counted in batch", + "disp_file": "lcurve.out", + "disp_freq": 1000, + "numb_test": 4, + "save_freq": 1000, + "save_ckpt": "model.ckpt", + "load_ckpt": "model.ckpt", + "disp_training": true, + "time_training": true, + "profiling": false, + "profiling_file": "timeline.json", + + "_comment": "that's all" + }, + + "_comment": " 01.model_devi ", + "_comment": "model_devi_skip: the first x of the recorded frames", + "model_devi_dt": 0.002, + "model_devi_skip": 0, + "model_devi_f_trust_lo": 0.05, + "model_devi_f_trust_hi": 0.15, + "model_devi_e_trust_lo": 1e10, + "model_devi_e_trust_hi": 1e10, + "model_devi_clean_traj": true, + "model_devi_jobs": [ + {"sys_idx": [0], + "temps": [ 300], "press": [0.0], "trj_freq": 10, "nsteps": 300, "ensemble": "nvt", "_idx": "00"}, + {"sys_idx": [1], + "temps": [ 300], "press": [0.0], "trj_freq": 10, "nsteps": 3000, "ensemble": "nvt", "_idx": "01"} + ], + + "_comment": " 02.fp ", + "fp_style": "pwmat", + "shuffle_poscar": false, + "fp_task_max": 20, + "fp_task_min": 8, + "fp_pp_path": ".", + "fp_pp_files": ["C.SG15.PBE.UPF", "H.SG15.PBE.UPF"], + "fp_incar" : "etot.input", + "_comment": " that's all " +} diff --git a/examples/run/dp-lammps-vasp/CH4/POT_C b/examples/run/dp-lammps-vasp/CH4/POT_C new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/examples/run/dp-lammps-vasp/CH4/POT_C @@ -0,0 +1 @@ + diff --git a/examples/run/dp-lammps-vasp/CH4/POT_H b/examples/run/dp-lammps-vasp/CH4/POT_H new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/examples/run/dp-lammps-vasp/CH4/POT_H @@ -0,0 +1 @@ + diff --git a/examples/run/dp-lammps-vasp/CH4/param_CH4.json b/examples/run/dp-lammps-vasp/CH4/param_CH4.json index 2ec6216d5..f69b4652d 100644 --- a/examples/run/dp-lammps-vasp/CH4/param_CH4.json +++ b/examples/run/dp-lammps-vasp/CH4/param_CH4.json @@ -88,12 +88,13 @@ ], "_comment": " 02.fp ", + "cvasp": false, "fp_style": "vasp", "shuffle_poscar": false, "fp_task_max": 20, "fp_task_min": 5, "fp_pp_path": "/gpfs/share/home/1600017784/yuzhi/methane/", - "fp_pp_files": ["POTCAR"], + "fp_pp_files": ["POT_H","POT_C"], "fp_incar" : "/gpfs/share/home/1600017784/yuzhi/methane/INCAR_methane", "_comment": " that's all " } diff --git a/setup.py b/setup.py index 71286b666..1016662a9 100755 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ with open(path.join('dpgen', '_date.py'), 'w') as fp : fp.write('date = \'%s\'' % today) -install_requires=['numpy>=1.14.3', 'dpdata>=0.1.12', 'pymatgen>=2019.1.13', 'ase', 'monty>2.0.0', 'paramiko', 'custodian'] +install_requires=['numpy>=1.14.3', 'dpdata>=0.1.14', 'pymatgen>=2019.1.13', 'ase', 'monty>2.0.0', 'paramiko', 'custodian'] setuptools.setup( name=NAME, @@ -42,6 +42,7 @@ 'dpgen/database', 'dpgen/tools', 'dpgen/simplify', + 'dpgen/collect', ], # data_files = [('dpgen/tools/', ['dpgen/tools/update_time.sh', ])], # package_data={'example':['*.json']}, diff --git a/tests/auto_test/00.equi/si/mp-149/vasp-relax_incar/CONTCAR b/tests/auto_test/00.equi/si/mp-149/vasp-relax_incar/CONTCAR new file mode 100644 index 000000000..4fe904343 --- /dev/null +++ b/tests/auto_test/00.equi/si/mp-149/vasp-relax_incar/CONTCAR @@ -0,0 +1,25 @@ +Si8 + 1.00000000000000 + 5.4687279999999996 0.0000000000000000 0.0000000000000000 + 0.0000000000000000 5.4687279999999996 0.0000000000000000 + 0.0000000000000000 0.0000000000000000 5.4687279999999996 + Si + 8 +Direct + 0.2500000000000000 0.2500000000000000 0.2500000000000000 + 0.5000000000000000 0.5000000000000000 0.0000000000000000 + 0.2500000000000000 0.7500000000000000 0.7500000000000000 + 0.5000000000000000 0.0000000000000000 0.5000000000000000 + 0.7500000000000000 0.2500000000000000 0.7500000000000000 + 0.0000000000000000 0.5000000000000000 0.5000000000000000 + 0.7500000000000000 0.7500000000000000 0.2500000000000000 + 0.0000000000000000 0.0000000000000000 0.0000000000000000 + + 0.00000000E+00 0.00000000E+00 0.00000000E+00 + 0.00000000E+00 0.00000000E+00 0.00000000E+00 + 0.00000000E+00 0.00000000E+00 0.00000000E+00 + 0.00000000E+00 0.00000000E+00 0.00000000E+00 + 0.00000000E+00 0.00000000E+00 0.00000000E+00 + 0.00000000E+00 0.00000000E+00 0.00000000E+00 + 0.00000000E+00 0.00000000E+00 0.00000000E+00 + 0.00000000E+00 0.00000000E+00 0.00000000E+00 diff --git a/tests/auto_test/00.equi/si/mp-149/vasp-relax_incar/OUTCAR b/tests/auto_test/00.equi/si/mp-149/vasp-relax_incar/OUTCAR new file mode 100644 index 000000000..0bbd2d456 --- /dev/null +++ b/tests/auto_test/00.equi/si/mp-149/vasp-relax_incar/OUTCAR @@ -0,0 +1,204 @@ + + + INCAR: + POTCAR: PAW_PBE Si 05Jan2001 + + POTCAR: PAW_PBE Si 05Jan2001 + VRHFIN =Si: s2p2 + LEXCH = PE + EATOM = 103.0669 eV, 7.5752 Ry + + TITEL = PAW_PBE Si 05Jan2001 + LULTRA = F use ultrasoft PP ? + IUNSCR = 1 unscreen: 0-lin 1-nonlin 2-no + RPACOR = 1.500 partial core radius + POMASS = 28.085; ZVAL = 4.000 mass and valenz + RCORE = 1.900 outmost cutoff radius + RWIGS = 2.480; RWIGS = 1.312 wigner-seitz radius (au A) + ENMAX = 245.345; ENMIN = 184.009 eV + ICORE = 2 local potential + LCOR = T correct aug charges + LPAW = T paw PP + EAUG = 322.069 + DEXC = 0.000 + RMAX = 1.950 core radius for proj-oper + RAUG = 1.300 factor for augmentation sphere + RDEP = 1.993 radius for radial grids + RDEPT = 1.837 core radius for aug-charge + + Atomic configuration + 6 entries + n l j E occ. + 1 0 0.50 -1785.8828 2.0000 + 2 0 0.50 -139.4969 2.0000 + 2 1 1.50 -95.5546 6.0000 + 3 0 0.50 -10.8127 2.0000 + 3 1 0.50 -4.0811 2.0000 + 3 2 1.50 -4.0817 0.0000 + Description + l E TYP RCUT TYP RCUT + 0 -10.8127223 23 1.900 + 0 -7.6451159 23 1.900 + 1 -4.0811372 23 1.900 + 1 2.4879257 23 1.900 + 2 -4.0817478 7 1.900 + local pseudopotential read in + partial core-charges read in + partial kinetic energy density read in + atomic valenz-charges read in + non local Contribution for L= 0 read in + real space projection operators read in + non local Contribution for L= 0 read in + real space projection operators read in + non local Contribution for L= 1 read in + real space projection operators read in + non local Contribution for L= 1 read in + real space projection operators read in + PAW grid and wavefunctions read in + + number of l-projection operators is LMAX = 4 + number of lm-projection operators is LMMAX = 8 + + PAW_PBE Si 05Jan2001 : + energy of atom 1 EATOM= -103.0669 + kinetic energy error for atom= 0.0003 (will be added to EATOM!!) + + + POSCAR: Si8 + positions in direct lattice + velocities in cartesian coordinates + exchange correlation table for LEXCH = 8 + RHO(1)= 0.500 N(1) = 2000 + RHO(2)= 100.500 N(2) = 4000 + + + +------------------------ aborting loop because EDIFF is reached ---------------------------------------- + + + CHARGE: cpu time 0.2963: real time 0.2963 + FORLOC: cpu time 0.0049: real time 0.0049 + FORNL : cpu time 0.9680: real time 0.9681 + STRESS: cpu time 3.5438: real time 3.5442 + FORCOR: cpu time 0.0514: real time 0.0514 + FORHAR: cpu time 0.0141: real time 0.0141 + MIXING: cpu time 0.0020: real time 0.0020 + OFIELD: cpu time 0.0000: real time 0.0000 + + FORCE on cell =-STRESS in cart. coord. units (eV): + Direction XX YY ZZ XY YZ ZX + -------------------------------------------------------------------------------------- + Alpha Z 13.17272 13.17272 13.17272 + Ewald -302.59373 -302.59373 -302.59373 0.00000 0.00000 -0.00000 + Hartree 20.22818 20.22818 20.22818 -0.00000 0.00000 -0.00001 + E(xc) -100.98430 -100.98430 -100.98430 0.00000 -0.00000 0.00000 + Local -118.00021 -118.00021 -118.00020 0.00001 -0.00000 0.00002 + n-local 309.78388 309.78388 309.78388 -0.00000 0.00000 -0.00000 + augment -46.59684 -46.59684 -46.59684 -0.00000 -0.00000 -0.00000 + Kinetic 224.99250 224.99250 224.99250 0.00000 0.00000 0.00001 + Fock 0.00000 0.00000 0.00000 0.00000 0.00000 0.00000 + ------------------------------------------------------------------------------------- + Total 0.00220 0.00220 0.00220 0.00001 0.00000 0.00002 + in kB 0.02151 0.02155 0.02156 0.00008 0.00001 0.00019 + external pressure = 0.02 kB Pullay stress = 0.00 kB + + + VOLUME and BASIS-vectors are now : + ----------------------------------------------------------------------------- + energy-cutoff : 650.00 + volume of cell : 163.55 + direct lattice vectors reciprocal lattice vectors + 5.468728000 0.000000000 0.000000000 0.182857878 0.000000000 0.000000000 + 0.000000000 5.468728000 0.000000000 0.000000000 0.182857878 0.000000000 + 0.000000000 0.000000000 5.468728000 0.000000000 0.000000000 0.182857878 + + length of vectors + 5.468728000 5.468728000 5.468728000 0.182857878 0.182857878 0.182857878 + + + FORCES acting on ions + electron-ion (+dipol) ewald-force non-local-force convergence-correction + ----------------------------------------------------------------------------------------------- + -.236E-04 -.111E-04 -.458E-04 0.249E-13 0.107E-13 -.355E-14 -.871E-05 -.478E-05 0.920E-05 0.337E-04 0.158E-04 0.387E-04 + -.255E-04 -.104E-04 -.182E-04 0.180E-13 0.384E-14 -.710E-14 0.241E-05 0.915E-05 -.846E-06 0.256E-04 0.474E-05 0.222E-04 + -.805E-04 0.107E-05 0.374E-04 0.107E-13 0.355E-14 0.000E+00 0.146E-04 0.648E-05 -.650E-05 0.733E-04 -.793E-05 -.346E-04 + -.760E-05 -.183E-04 -.565E-05 0.341E-14 -.357E-14 -.362E-14 -.135E-05 0.548E-05 0.133E-04 0.888E-05 0.154E-04 -.541E-05 + 0.555E-04 0.406E-05 0.204E-04 0.355E-14 0.355E-14 0.355E-14 -.750E-05 -.781E-05 0.356E-05 -.512E-04 0.202E-05 -.277E-04 + -.321E-05 0.309E-04 0.493E-04 -.178E-13 -.350E-14 -.718E-16 0.836E-05 -.964E-05 -.109E-04 -.534E-05 -.228E-04 -.407E-04 + 0.481E-04 0.281E-04 -.852E-05 -.107E-13 -.355E-14 0.711E-14 -.257E-05 -.775E-05 -.798E-05 -.479E-04 -.223E-04 0.139E-04 + 0.234E-04 -.320E-04 -.298E-04 -.320E-13 -.106E-13 0.356E-14 -.448E-05 0.995E-05 0.233E-05 -.207E-04 0.245E-04 0.287E-04 + ----------------------------------------------------------------------------------------------- + -.134E-04 -.763E-05 -.867E-06 0.124E-15 0.361E-15 -.131E-15 0.759E-06 0.107E-05 0.221E-05 0.165E-04 0.934E-05 -.503E-05 + + + POSITION TOTAL-FORCE (eV/Angst) + ----------------------------------------------------------------------------------- + 1.36718 1.36718 1.36718 0.000001 -0.000000 0.000003 + 2.73436 2.73436 0.00000 0.000002 0.000003 0.000004 + 1.36718 4.10155 4.10155 0.000007 -0.000001 -0.000003 + 2.73436 0.00000 2.73436 -0.000001 0.000002 0.000003 + 4.10155 1.36718 4.10155 -0.000004 -0.000002 -0.000003 + 0.00000 2.73436 2.73436 -0.000001 -0.000002 -0.000002 + 4.10155 4.10155 1.36718 -0.000003 -0.000002 -0.000002 + 0.00000 0.00000 0.00000 -0.000002 0.000002 0.000002 + ----------------------------------------------------------------------------------- + total drift: 0.000004 0.000003 -0.000004 + + +-------------------------------------------------------------------------------------------------------- + + + + FREE ENERGIE OF THE ION-ELECTRON SYSTEM (eV) + --------------------------------------------------- + free energy TOTEN = -43.40017315 eV + + energy without entropy= -43.40017315 energy(sigma->0) = -43.40017315 + + + +-------------------------------------------------------------------------------------------------------- + + + POTLOK: cpu time 0.0496: real time 0.0496 + + +-------------------------------------------------------------------------------------------------------- + + + + +-------------------------------------------------------------------------------------------------------- + + + + reached required accuracy - stopping structural energy minimisation + LOOP+: cpu time 73.7767: real time 73.8176 + 4ORBIT: cpu time 0.0000: real time 0.0000 + + total amount of memory used by VASP MPI-rank0 119933. kBytes +======================================================================= + + base : 30000. kBytes + nonl-proj : 43479. kBytes + fftplans : 4240. kBytes + grid : 9842. kBytes + one-center: 24. kBytes + wavefun : 32348. kBytes + + + + General timing and accounting informations for this job: + ======================================================== + + Total CPU time used (sec): 75.718 + User time (sec): 75.184 + System time (sec): 0.534 + Elapsed time (sec): 75.841 + + Maximum memory used (kb): 186724. + Average memory used (kb): 0. + + Minor page faults: 71438 + Major page faults: 0 + Voluntary context switches: 409 diff --git a/tests/auto_test/confs/si/mp-149/POSCAR b/tests/auto_test/confs/si/mp-149/POSCAR new file mode 100644 index 000000000..4fe904343 --- /dev/null +++ b/tests/auto_test/confs/si/mp-149/POSCAR @@ -0,0 +1,25 @@ +Si8 + 1.00000000000000 + 5.4687279999999996 0.0000000000000000 0.0000000000000000 + 0.0000000000000000 5.4687279999999996 0.0000000000000000 + 0.0000000000000000 0.0000000000000000 5.4687279999999996 + Si + 8 +Direct + 0.2500000000000000 0.2500000000000000 0.2500000000000000 + 0.5000000000000000 0.5000000000000000 0.0000000000000000 + 0.2500000000000000 0.7500000000000000 0.7500000000000000 + 0.5000000000000000 0.0000000000000000 0.5000000000000000 + 0.7500000000000000 0.2500000000000000 0.7500000000000000 + 0.0000000000000000 0.5000000000000000 0.5000000000000000 + 0.7500000000000000 0.7500000000000000 0.2500000000000000 + 0.0000000000000000 0.0000000000000000 0.0000000000000000 + + 0.00000000E+00 0.00000000E+00 0.00000000E+00 + 0.00000000E+00 0.00000000E+00 0.00000000E+00 + 0.00000000E+00 0.00000000E+00 0.00000000E+00 + 0.00000000E+00 0.00000000E+00 0.00000000E+00 + 0.00000000E+00 0.00000000E+00 0.00000000E+00 + 0.00000000E+00 0.00000000E+00 0.00000000E+00 + 0.00000000E+00 0.00000000E+00 0.00000000E+00 + 0.00000000E+00 0.00000000E+00 0.00000000E+00 diff --git a/tests/auto_test/test_elastic.py b/tests/auto_test/test_elastic.py new file mode 100644 index 000000000..53a8d57a2 --- /dev/null +++ b/tests/auto_test/test_elastic.py @@ -0,0 +1,52 @@ +import os,sys,json,glob,shutil +import dpdata +import numpy as np +import unittest + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +__package__ = 'auto_test' +from .context import make_kspacing_kpoints +from .context import setUpModule + +from pymatgen.io.vasp import Incar +from dpgen.auto_test.gen_02_elastic import make_vasp + +class Test02(unittest.TestCase): + def tearDown(self): + if os.path.exists('02.elastic'): + shutil.rmtree('02.elastic') + + def test_make_vasp (self): + jdata = { + 'relax_incar' : 'vasp_input/INCAR.rlx', + 'potcar_map': {'Si': 'vasp_input/POTCAR' }, + } + make_vasp(jdata, 'confs/si/mp-149') + + target_path = '02.elastic/si/mp-149/vasp-relax_incar' + equi_path = '00.equi/si/mp-149/vasp-relax_incar' + dfm_dirs = glob.glob(os.path.join(target_path, 'dfm*')) + + # check root INCAR + incar0 = Incar.from_file(os.path.join('vasp_input', 'INCAR.rlx')) + incar1 = Incar.from_file(os.path.join(target_path, 'INCAR')) + self.assertFalse(incar0 == incar1) + incar0['ISIF'] = 2 + self.assertTrue(incar0 == incar1) + # check root POTCAR + with open(os.path.join('vasp_input', 'POTCAR')) as fp: + pot0 = fp.read() + with open(os.path.join(target_path, 'POTCAR')) as fp: + pot1 = fp.read() + self.assertEqual(pot0, pot1) + # check root POSCAR + self.assertEqual(os.path.realpath(os.path.join(target_path, 'POSCAR')), + os.path.realpath(os.path.join(equi_path, 'CONTCAR'))) + # check subdir + for ii in dfm_dirs: + self.assertEqual(os.path.realpath(os.path.join(ii, 'INCAR')), + os.path.realpath(os.path.join(target_path, 'INCAR'))) + self.assertEqual(os.path.realpath(os.path.join(ii, 'KPOINTS')), + os.path.realpath(os.path.join(target_path, 'KPOINTS'))) + self.assertEqual(os.path.realpath(os.path.join(ii, 'POTCAR')), + os.path.realpath(os.path.join(target_path, 'POTCAR'))) diff --git a/tests/auto_test/test_eos.py b/tests/auto_test/test_eos.py new file mode 100644 index 000000000..1cddb92c7 --- /dev/null +++ b/tests/auto_test/test_eos.py @@ -0,0 +1,81 @@ +import os,sys,json,glob,shutil +import dpdata +import numpy as np +import unittest + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +__package__ = 'auto_test' +from .context import make_kspacing_kpoints +from .context import setUpModule + +from pymatgen.io.vasp import Incar +from dpgen.auto_test.gen_01_eos import make_vasp + +class Test01(unittest.TestCase): + def tearDown(self): + if os.path.exists('01.eos'): + shutil.rmtree('01.eos') + + def test_make_vasp_rlx_cell_shape (self): + jdata = { + 'relax_incar' : 'vasp_input/INCAR.rlx', + 'potcar_map': {'Si': 'vasp_input/POTCAR' }, + 'vol_start': 15, + 'vol_end': 25, + 'vol_step': 1.0, + 'eos_relax_cell_shape': True, + } + make_vasp(jdata, 'confs/si/mp-149') + + target_path = '01.eos/si/mp-149/vasp-relax_incar' + equi_path = '00.equi/si/mp-149/vasp-relax_incar' + dfm_dirs = glob.glob(os.path.join(target_path, 'vol*')) + + # check root INCAR + incar0 = Incar.from_file(os.path.join('vasp_input', 'INCAR.rlx')) + incar1 = Incar.from_file(os.path.join(target_path, 'INCAR')) + self.assertFalse(incar0 == incar1) + incar0['ISIF'] = 4 + self.assertTrue(incar0 == incar1) + # check root POTCAR + with open(os.path.join('vasp_input', 'POTCAR')) as fp: + pot0 = fp.read() + with open(os.path.join(target_path, 'POTCAR')) as fp: + pot1 = fp.read() + self.assertEqual(pot0, pot1) + # check subdir + for ii in dfm_dirs: + self.assertTrue(os.path.isfile(os.path.join(ii, 'KPOINTS'))) + self.assertEqual(os.path.realpath(os.path.join(ii, 'POSCAR.orig')), + os.path.realpath(os.path.join(equi_path, 'CONTCAR'))) + self.assertEqual(os.path.realpath(os.path.join(ii, 'INCAR')), + os.path.realpath(os.path.join(target_path, 'INCAR'))) + self.assertEqual(os.path.realpath(os.path.join(ii, 'POTCAR')), + os.path.realpath(os.path.join(target_path, 'POTCAR'))) + sys = dpdata.System(os.path.join(ii, 'POSCAR')) + vol = float(ii.split('/')[-1].split('-')[1]) + natoms = sys.get_natoms() + self.assertAlmostEqual(vol, np.linalg.det(sys['cells'][0]) / natoms) + + + def test_make_vasp_norlx_cell_shape (self): + jdata = { + 'relax_incar' : 'vasp_input/INCAR.rlx', + 'potcar_map': {'Si': 'vasp_input/POTCAR' }, + 'vol_start': 15, + 'vol_end': 25, + 'vol_step': 1.0, + 'eos_relax_cell_shape': False, + } + make_vasp(jdata, 'confs/si/mp-149') + + target_path = '01.eos/si/mp-149/vasp-relax_incar' + equi_path = '00.equi/si/mp-149/vasp-relax_incar' + dfm_dirs = glob.glob(os.path.join(target_path, 'vol*')) + + # check root INCAR + incar0 = Incar.from_file(os.path.join('vasp_input', 'INCAR.rlx')) + incar1 = Incar.from_file(os.path.join(target_path, 'INCAR')) + self.assertFalse(incar0 == incar1) + incar0['ISIF'] = 2 + self.assertTrue(incar0 == incar1) diff --git a/tests/auto_test/vasp_input/INCAR b/tests/auto_test/vasp_input/INCAR new file mode 100644 index 000000000..1e2a28323 --- /dev/null +++ b/tests/auto_test/vasp_input/INCAR @@ -0,0 +1,30 @@ +#Parameters +SYSTEM = dpgen +PREC = A +ISART = 0 +ICHARG = 2 +#Electronic Relaxation +ENCUT = 650 +NELM = 100 +NELMIN = 6 +NELMDL = -5 +EDIFF = 1e-06 +LREAL = False +ALGO = Fast # or normal +#Ionic relaxation +IBRION = -1 +ISIF = 2 +ISYM = 0 +NSW = 0 +ISMEAR = 0 +SIGMA = 0.1 +# Write flags +LWAVE = False +LCHARG = False +#parallel related +#KPAR = 4 +#NPAR = 1 +KSPACING = 0.1 +KGAMMA = False + +PSTRESS = 0.0 diff --git a/tests/auto_test/vasp_input/INCAR.md b/tests/auto_test/vasp_input/INCAR.md new file mode 100644 index 000000000..b7b00d64b --- /dev/null +++ b/tests/auto_test/vasp_input/INCAR.md @@ -0,0 +1,38 @@ +#Parameters +SYSTEM = dpgen +PREC = A +ISART = 0 +ICHARG = 2 +#Electronic Relaxation +ENCUT = 650 +NELM = 100 +NELMIN = 6 +NELMDL = -5 +EDIFF = 1e-06 +LREAL = False +ALGO = Fast # or normal +#Ionic relaxation +IBRION = 0 +ISIF = 2 +#EDIFFG = -0.01 # useless for MD +ISYM = 0 +NSW = 10 +ISMEAR = 0 +SIGMA = 0.1 +# MD related +SMASS = 0 +POTIM = 2 +TEBEG = 100 +TEEND = 100 +NBLOCK = 1 +KBLOCK = 100 +# Write flags +LWAVE = False +LCHARG = False +#parallel related +#KPAR = 4 +#NPAR = 1 +KSPACING = 0.1 +KGAMMA = False + +PSTRESS = 0.0 diff --git a/tests/auto_test/vasp_input/INCAR.rlx b/tests/auto_test/vasp_input/INCAR.rlx new file mode 100644 index 000000000..b871170ff --- /dev/null +++ b/tests/auto_test/vasp_input/INCAR.rlx @@ -0,0 +1,32 @@ +#Parameters +SYSTEM = dpgen +PREC = A +ISART = 0 +ICHARG = 2 +#Electronic Relaxation +ENCUT = 650 +NELM = 100 +NELMIN = 6 +NELMDL = -5 +EDIFF = 1e-06 +LREAL = False +ALGO = Fast # or normal +#Ionic relaxation +IBRION = 2 +POTIM = 0.2 +ISIF = 3 +EDIFFG = -0.01 +ISYM = 0 +NSW = 100 +ISMEAR = 0 +SIGMA = 0.1 +# Write flags +LWAVE = False +LCHARG = False +#parallel related +#KPAR = 4 +#NPAR = 1 +KSPACING = 0.2 +KGAMMA = False + +PSTRESS = 0.0 diff --git a/tests/auto_test/vasp_input/POTCAR b/tests/auto_test/vasp_input/POTCAR new file mode 100644 index 000000000..7bfb856bd --- /dev/null +++ b/tests/auto_test/vasp_input/POTCAR @@ -0,0 +1 @@ +bf8505dd-84b6-4fcc-a32c-3e1768b8a119 diff --git a/tests/data/surf_poscar.json b/tests/data/surf_poscar.json index b5727ff83..3399b920f 100644 --- a/tests/data/surf_poscar.json +++ b/tests/data/surf_poscar.json @@ -12,14 +12,11 @@ 1 ], "layer_numb": 3, - "vacuum_max": 9, - "vacuum_resol": [ - 0.5, - 1 - ], - "mid_point": 4.0, - "head_ratio": 0.6, - "vacuum_numb": 20, + "vacuum_max": 5, + "vacuum_resol": [0.5,2], + "mid_point": 2.0, + "_head_ratio": 0.6, + "_vacuum_numb": 20, "millers": [ [ 1, diff --git a/tests/data/test_gen_surf.py b/tests/data/test_gen_surf.py index 5392ccb74..63e9ed7a4 100644 --- a/tests/data/test_gen_surf.py +++ b/tests/data/test_gen_surf.py @@ -1,6 +1,6 @@ import os,sys,json,glob,shutil import unittest -from pymatgen import Structure +from pymatgen import Structure,Element sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) __package__ = 'data' @@ -24,6 +24,7 @@ def setUp(self): make_vasp_relax(jdata) make_scale(jdata) pert_scaled(jdata) + self.jdata=jdata def tearDown(self): shutil.rmtree(self.root_dir) @@ -38,7 +39,8 @@ def test(self): surf=poscar.split('/')[-3] st1=Structure.from_file(surf+'.POSCAR') st2=Structure.from_file(poscar) - self.assertEqual(st1,st2) + vacuum_size=float(Element(self.jdata['elements'][0]).atomic_radius*2) + self.assertTrue(st1.lattice.c+vacuum_size-st2.lattice.c<0.01) for surf in self.surfs: elongs=glob.glob("surf.al.fcc.01x01x01/01.scale_pert/"+surf+"/sys-*/scale-1.000/el*") diff --git a/tests/data/test_gen_surf_poscar.py b/tests/data/test_gen_surf_poscar.py index 5c49b0578..1ed44eac3 100644 --- a/tests/data/test_gen_surf_poscar.py +++ b/tests/data/test_gen_surf_poscar.py @@ -1,6 +1,6 @@ import os,sys,json,glob,shutil import unittest -from pymatgen import Structure +from pymatgen import Structure,Element sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) __package__ = 'data' @@ -10,9 +10,8 @@ class TestGenSurfPOSCAR(unittest.TestCase): def setUp(self): self.surfs=["surf-100"] - self.elongs=["elong-0.500", "elong-1.000", "elong-1.500", "elong-2.000", "elong-2.500",\ - "elong-3.000", "elong-3.500", "elong-4.000", "elong-5.000", "elong-6.000",\ - "elong-7.000", "elong-8.000" ] + self.elongs=["elong-0.500", "elong-1.000", "elong-1.500", + "elong-2.000", "elong-4.000"] with open (param_file, 'r') as fp : jdata = json.load (fp) out_dir = out_dir_name(jdata) @@ -24,6 +23,7 @@ def setUp(self): make_vasp_relax(jdata) make_scale(jdata) pert_scaled(jdata) + self.jdata=jdata def tearDown(self): shutil.rmtree(self.root_dir) @@ -38,7 +38,9 @@ def test(self): surf=poscar.split('/')[-3] st1=Structure.from_file(surf+'.POSCAR') st2=Structure.from_file(poscar) - self.assertEqual(st1,st2) + vacuum_size=float(Element(self.jdata['elements'][0]).atomic_radius*2) + self.assertTrue(st1.lattice.c+vacuum_size-st2.lattice.c<0.01) + for surf in self.surfs: elongs=glob.glob("POSCAR.01x01x01/01.scale_pert/"+surf+"/sys-*/scale-1.000/el*") diff --git a/tests/dispatcher/shell/test_shell_local.py b/tests/dispatcher/shell/test_shell_local.py index b4d02ba67..4c47136c1 100644 --- a/tests/dispatcher/shell/test_shell_local.py +++ b/tests/dispatcher/shell/test_shell_local.py @@ -74,7 +74,7 @@ def test_sub_success(self) : self.assertTrue(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task0/tag_1_finished'))) self.assertTrue(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task1/tag_0_finished'))) self.assertTrue(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task1/tag_1_finished'))) - self.assertTrue(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'tag_finished'))) + self.assertTrue(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, '%s_tag_finished' % self.shell.context.job_uuid))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task0/test1'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task1/test1'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task0/test2'))) @@ -101,7 +101,7 @@ def test_sub_scancel(self) : self.assertFalse(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task0/tag_1_finished'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task1/tag_0_finished'))) self.assertFalse(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task1/tag_1_finished'))) - self.assertFalse(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'tag_finished'))) + self.assertFalse(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, '%s_tag_finished' % self.shell.context.job_uuid))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task0/test1'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task1/test1'))) self.assertFalse(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task0/test2'))) @@ -117,7 +117,7 @@ def test_sub_scancel(self) : self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task0/tag_1_finished'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task1/tag_0_finished'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task1/tag_1_finished'))) - self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'tag_finished'))) + self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, '%s_tag_finished' % self.shell.context.job_uuid))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task0/test1'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task1/test1'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task0/test2'))) diff --git a/tests/dispatcher/shell/test_shell_ssh.py b/tests/dispatcher/shell/test_shell_ssh.py index 10da439a4..d80bb5688 100644 --- a/tests/dispatcher/shell/test_shell_ssh.py +++ b/tests/dispatcher/shell/test_shell_ssh.py @@ -59,7 +59,7 @@ def test_sub_success(self) : self.assertTrue(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task0/tag_1_finished'))) self.assertTrue(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task1/tag_0_finished'))) self.assertTrue(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task1/tag_1_finished'))) - self.assertTrue(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'tag_finished'))) + self.assertTrue(os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, '%s_tag_finished' % self.shell.context.job_uuid))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task0/test1'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task1/test1'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.shell.context.remote_root, 'task0/test2'))) diff --git a/tests/dispatcher/slurm/test_dispatcher_lazy_local.py b/tests/dispatcher/slurm/test_dispatcher_lazy_local.py index 7d21a2fe7..89fd9b9a4 100644 --- a/tests/dispatcher/slurm/test_dispatcher_lazy_local.py +++ b/tests/dispatcher/slurm/test_dispatcher_lazy_local.py @@ -22,15 +22,14 @@ def setUp(self) : for ii in ['loc/task0', 'loc/task1', 'loc/task2']: with open(os.path.join(ii, 'test0'),'w') as fp: fp.write('this is test0 from ' + ii + '\n') - work_profile = None + work_profile = {} self.disp = Dispatcher(work_profile, 'lazy-local', 'slurm') def tearDown(self): - # shutil.rmtree('loc') - # shutil.rmtree('rmt') - # if os.path.exists('dpgen.log'): - # os.remove('dpgen.log') - pass + shutil.rmtree('loc') + shutil.rmtree('rmt') + if os.path.exists('dpgen.log'): + os.remove('dpgen.log') def test_sub_success(self): tasks = ['task0', 'task1', 'task2'] diff --git a/tests/dispatcher/slurm/test_slurm_local.py b/tests/dispatcher/slurm/test_slurm_local.py index bca72a54a..0aeca1f75 100644 --- a/tests/dispatcher/slurm/test_slurm_local.py +++ b/tests/dispatcher/slurm/test_slurm_local.py @@ -50,7 +50,7 @@ def test_sub_success(self) : self.assertTrue(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/tag_1_finished'))) self.assertTrue(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/tag_0_finished'))) self.assertTrue(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/tag_1_finished'))) - self.assertTrue(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'tag_finished'))) + self.assertTrue(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, '%s_tag_finished' % self.slurm.context.job_uuid))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/test1'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/test1'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/test2'))) @@ -76,7 +76,7 @@ def test_sub_scancel(self) : self.assertFalse(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/tag_1_finished'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/tag_0_finished'))) self.assertFalse(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/tag_1_finished'))) - self.assertFalse(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'tag_finished'))) + self.assertFalse(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, '%s_tag_finished' % self.slurm.context.job_uuid))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/test1'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/test1'))) self.assertFalse(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/test2'))) @@ -92,7 +92,7 @@ def test_sub_scancel(self) : self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/tag_1_finished'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/tag_0_finished'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/tag_1_finished'))) - self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'tag_finished'))) + self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, '%s_tag_finished' % self.slurm.context.job_uuid))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/test1'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/test1'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/test2'))) diff --git a/tests/dispatcher/slurm/test_slurm_ssh.py b/tests/dispatcher/slurm/test_slurm_ssh.py index 204e2191f..774650110 100644 --- a/tests/dispatcher/slurm/test_slurm_ssh.py +++ b/tests/dispatcher/slurm/test_slurm_ssh.py @@ -37,7 +37,7 @@ def test_gen_sub_script(self): self.slurm.context.write_file('run.sub', ret) with open('run.sub', 'w') as fp: fp.write(ret) - ret1 = self.slurm.sub_script(job_dirs, ['touch', 'touch'], [['test1 ', 'test2 '], ['test1 ', 'test2 ']]) + ret1 = self.slurm.sub_script(job_dirs, ['touch', 'touch'], [['test1 ', 'test1 '], ['test2 ', 'test2 ']]) with open('run.sub.1', 'w') as fp: fp.write(ret1) my_file_cmp(self, 'run.sub.1', 'run.sub') @@ -55,7 +55,7 @@ def test_sub_success(self) : self.assertTrue(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/tag_1_finished'))) self.assertTrue(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/tag_0_finished'))) self.assertTrue(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/tag_1_finished'))) - self.assertTrue(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'tag_finished'))) + self.assertTrue(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, '%s_tag_finished' % self.slurm.context.job_uuid))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/test1'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/test1'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/test2'))) @@ -81,7 +81,7 @@ def test_sub_scancel(self) : self.assertFalse(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/tag_1_finished'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/tag_0_finished'))) self.assertFalse(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/tag_1_finished'))) - self.assertFalse(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'tag_finished'))) + self.assertFalse(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, '%s_tag_finished' % self.slurm.context.job_uuid))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/test1'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/test1'))) self.assertFalse(os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/test2'))) @@ -97,7 +97,7 @@ def test_sub_scancel(self) : self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/tag_1_finished'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/tag_0_finished'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/tag_1_finished'))) - self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'tag_finished'))) + self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, '%s_tag_finished' % self.slurm.context.job_uuid))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/test1'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task1/test1'))) self.assertTrue (os.path.isfile(os.path.join('rmt', self.slurm.context.remote_root, 'task0/test2'))) diff --git a/tests/dispatcher/test_lazy_local_context.py b/tests/dispatcher/test_lazy_local_context.py index 6f34f2f14..8de1875d8 100644 --- a/tests/dispatcher/test_lazy_local_context.py +++ b/tests/dispatcher/test_lazy_local_context.py @@ -39,6 +39,83 @@ def test_download(self): self.job.upload(tasks, ['test0', 'dir0']) self.job.download(tasks, ['test0', 'dir0']) + def test_download_check_mark(self): + # upload files + self.job = LazyLocalContext('loc', None) + tasks = ['task0', 'task1'] + self.job.upload(tasks, ['test0', 'dir0']) + record_uuid = [] + # generate extra donwload files + for ii in tasks : + for jj in ['test6', 'test7'] : + if (ii == 'task1' and jj == 'test7') or \ + (ii == 'task0' and jj == 'test6'): + continue + with open(os.path.join('loc',ii,jj), 'w') as fp: + tmp = str(uuid.uuid4()) + fp.write(tmp) + record_uuid.append(tmp) + self.job.download(tasks, ['test6', 'test7', 'dir1'], check_exists = True, mark_failure = True) + # check dlded + cc = 0 + for ii in tasks : + for jj in ['test6', 'test7'] : + if (ii == 'task1' and jj == 'test7') or \ + (ii == 'task0' and jj == 'test6') : + self.assertFalse(os.path.exists(os.path.join('loc', ii, jj)), + msg = 'found ' + os.path.join('loc', ii, jj)) + self.assertTrue(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj)), + msg = 'failed to find ' + os.path.join('loc', ii, 'tag_failure_download_%s' % jj)) + continue + with open(os.path.join('loc',ii,jj), 'r') as fp: + tmp = fp.read() + self.assertEqual(tmp, record_uuid[cc]) + cc += 1 + for ii in tasks : + for jj in ['dir1'] : + self.assertFalse(os.path.exists(os.path.join('loc', ii, jj))) + self.assertTrue(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj))) + + + def test_download_check_nomark(self): + # upload files + self.job = LazyLocalContext('loc', None) + tasks = ['task0', 'task1'] + self.job.upload(tasks, ['test0', 'dir0']) + record_uuid = [] + # generate extra donwload files + for ii in tasks : + for jj in ['test6', 'test7'] : + if (ii == 'task1' and jj == 'test7') or \ + (ii == 'task0' and jj == 'test6'): + continue + with open(os.path.join('loc',ii,jj), 'w') as fp: + tmp = str(uuid.uuid4()) + fp.write(tmp) + record_uuid.append(tmp) + self.job.download(tasks, ['test6', 'test7', 'dir1'], check_exists = True, mark_failure = False) + # check dlded + cc = 0 + for ii in tasks : + for jj in ['test6', 'test7'] : + if (ii == 'task1' and jj == 'test7') or \ + (ii == 'task0' and jj == 'test6') : + self.assertFalse(os.path.exists(os.path.join('loc', ii, jj)), + msg = 'found ' + os.path.join('loc', ii, jj)) + self.assertFalse(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj)), + msg = 'found ' + os.path.join('loc', ii, 'tag_failure_download_%s' % jj)) + continue + with open(os.path.join('loc',ii,jj), 'r') as fp: + tmp = fp.read() + self.assertEqual(tmp, record_uuid[cc]) + cc += 1 + for ii in tasks : + for jj in ['dir1'] : + self.assertFalse(os.path.exists(os.path.join('loc', ii, jj))) + self.assertFalse(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj))) + + + def test_block_call(self) : self.job = LazyLocalContext('loc', None) tasks = ['task0', 'task1'] diff --git a/tests/dispatcher/test_local_context.py b/tests/dispatcher/test_local_context.py index 49d4e2efc..694d48fa2 100644 --- a/tests/dispatcher/test_local_context.py +++ b/tests/dispatcher/test_local_context.py @@ -217,6 +217,88 @@ def test_download(self): rmtf = os.path.join('rmt', self.job.job_uuid, ii, jj, kk) self.assertEqual(os.path.realpath(locf), os.path.realpath(rmtf)) + + def test_download_check_mark(self): + # upload files + work_profile = LocalSession({'work_path':'rmt'}) + self.job = LocalContext('loc', work_profile) + tasks = ['task0', 'task1'] + self.job.upload(tasks, ['test0', 'dir0']) + # generate extra donwload files + record_uuid = [] + for ii in tasks : + for jj in ['test7', 'test8'] : + if (ii == 'task1' and jj == 'test7') or \ + (ii == 'task0' and jj == 'test6') : + continue + with open(os.path.join('rmt',self.job.job_uuid,ii,jj), 'w') as fp: + tmp = str(uuid.uuid4()) + fp.write(tmp) + record_uuid.append(tmp) + # donwload + files = ['test7', 'test8', 'dir1'] + self.job.download(tasks, files, check_exists = True, mark_failure = True) + # check dlded + cc = 0 + for ii in tasks : + for jj in ['test7', 'test8'] : + if (ii == 'task1' and jj == 'test7') or \ + (ii == 'task0' and jj == 'test6') : + self.assertFalse(os.path.exists(os.path.join('loc', ii, jj)), + msg = 'found ' + os.path.join('loc', ii, jj)) + self.assertTrue(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj)), + msg = 'failed to find ' + os.path.join('loc', ii, 'tag_failure_download_%s' % jj)) + continue + with open(os.path.join('loc',ii,jj), 'r') as fp: + tmp = fp.read() + self.assertEqual(tmp, record_uuid[cc]) + cc += 1 + for ii in tasks : + for jj in ['dir1'] : + self.assertFalse(os.path.exists(os.path.join('loc', ii, jj))) + self.assertTrue(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj))) + + + def test_download_check_nomark(self): + # upload files + work_profile = LocalSession({'work_path':'rmt'}) + self.job = LocalContext('loc', work_profile) + tasks = ['task0', 'task1'] + self.job.upload(tasks, ['test0', 'dir0']) + # generate extra donwload files + record_uuid = [] + for ii in tasks : + for jj in ['test7', 'test8'] : + if (ii == 'task1' and jj == 'test7') or \ + (ii == 'task0' and jj == 'test6') : + continue + with open(os.path.join('rmt',self.job.job_uuid,ii,jj), 'w') as fp: + tmp = str(uuid.uuid4()) + fp.write(tmp) + record_uuid.append(tmp) + # donwload + files = ['test7', 'test8', 'dir1'] + self.job.download(tasks, files, check_exists = True, mark_failure = False) + # check dlded + cc = 0 + for ii in tasks : + for jj in ['test7', 'test8'] : + if (ii == 'task1' and jj == 'test7') or \ + (ii == 'task0' and jj == 'test6') : + self.assertFalse(os.path.exists(os.path.join('loc', ii, jj)), + msg = 'found ' + os.path.join('loc', ii, jj)) + self.assertFalse(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj)), + msg = 'found ' + os.path.join('loc', ii, 'tag_failure_download_%s' % jj)) + continue + with open(os.path.join('loc',ii,jj), 'r') as fp: + tmp = fp.read() + self.assertEqual(tmp, record_uuid[cc]) + cc += 1 + for ii in tasks : + for jj in ['dir1'] : + self.assertFalse(os.path.exists(os.path.join('loc', ii, jj))) + self.assertFalse(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj))) + def test_block_call(self) : work_profile = LocalSession({'work_path':'rmt'}) diff --git a/tests/dispatcher/test_ssh_context.py b/tests/dispatcher/test_ssh_context.py index 909923d2d..550fcaa12 100644 --- a/tests/dispatcher/test_ssh_context.py +++ b/tests/dispatcher/test_ssh_context.py @@ -110,6 +110,87 @@ def test_donwload(self): cc += 1 + def test_donwload_check_mark(self): + tasks = ['task0', 'task1'] + self.job.upload(tasks, ['test0', 'dir0']) + # generate extra donwload files + record_uuid = [] + for ii in tasks : + for jj in ['test6', 'test7'] : + if (ii == 'task1' and jj == 'test7') or \ + (ii == 'task0' and jj == 'test6'): + continue + with open(os.path.join('rmt',self.job.job_uuid,ii,jj), 'w') as fp: + tmp = str(uuid.uuid4()) + fp.write(tmp) + record_uuid.append(tmp) + # donwload + files = ['test6', 'test7', 'dir1'] + self.job.download(tasks, files, check_exists = True, mark_failure = True) + # check dlded + cc = 0 + for ii in tasks : + for jj in ['test6', 'test7'] : + if (ii == 'task1' and jj == 'test7') or \ + (ii == 'task0' and jj == 'test6') : + self.assertFalse(os.path.exists(os.path.join('loc', ii, jj)), + msg = 'found ' + os.path.join('loc', ii, jj)) + self.assertTrue(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj)), + msg = 'failed to find ' + os.path.join('loc', ii, 'tag_failure_download_%s' % jj)) + continue + with open(os.path.join('loc',ii,jj), 'r') as fp: + tmp = fp.read() + self.assertEqual(tmp, record_uuid[cc]) + cc += 1 + for ii in tasks : + for jj in ['dir1'] : + self.assertFalse(os.path.exists(os.path.join('loc', ii, jj))) + self.assertTrue(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj))) + + + def test_donwload_check_nomark(self): + tasks = ['task0', 'task1'] + self.job.upload(tasks, ['test0', 'dir0']) + # generate extra donwload files + record_uuid = [] + for ii in tasks : + for jj in ['test6', 'test7'] : + if ii == 'task1' and jj == 'test7' : + continue + if ii == 'task0' and jj == 'test6' : + continue + with open(os.path.join('rmt',self.job.job_uuid,ii,jj), 'w') as fp: + tmp = str(uuid.uuid4()) + fp.write(tmp) + record_uuid.append(tmp) + # donwload + files = ['test6', 'test7', 'dir1'] + self.job.download(tasks, files, check_exists = True, mark_failure = False) + # check dlded + cc = 0 + for ii in tasks : + for jj in ['test6', 'test7'] : + if ii == 'task1' and jj == 'test7' : + self.assertFalse(os.path.exists(os.path.join('loc', ii, jj)), + msg = 'found ' + os.path.join('loc', ii, jj)) + self.assertFalse(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj)), + msg = 'found ' + os.path.join('loc', ii, 'tag_failure_download_%s' % jj)) + continue + if ii == 'task0' and jj == 'test6' : + self.assertFalse(os.path.exists(os.path.join('loc', ii, jj)), + msg = 'found ' + os.path.join('loc', ii, jj)) + self.assertFalse(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj)), + msg = 'found ' + os.path.join('loc', ii, 'tag_failure_download_%s' % jj)) + continue + with open(os.path.join('loc',ii,jj), 'r') as fp: + tmp = fp.read() + self.assertEqual(tmp, record_uuid[cc]) + cc += 1 + for ii in tasks : + for jj in ['dir1'] : + self.assertFalse(os.path.exists(os.path.join('loc', ii, jj))) + self.assertFalse(os.path.exists(os.path.join('loc', ii, 'tag_failure_download_%s' % jj))) + def test_block_call(self) : tasks = ['task0', 'task1'] files = ['test0', 'test1'] @@ -147,3 +228,4 @@ def test_file(self) : tmp1 = self.job.read_file('aaa') self.assertEqual(tmp, tmp1) + diff --git a/tests/generator/C.SG15.PBE.UPF b/tests/generator/C.SG15.PBE.UPF new file mode 100644 index 000000000..e69de29bb diff --git a/tests/generator/H.SG15.PBE.UPF b/tests/generator/H.SG15.PBE.UPF new file mode 100644 index 000000000..e69de29bb diff --git a/tests/generator/N.SG15.PBE.UPF b/tests/generator/N.SG15.PBE.UPF new file mode 100644 index 000000000..e69de29bb diff --git a/tests/generator/check_bad_box/bad.height.POSCAR b/tests/generator/check_bad_box/bad.height.POSCAR new file mode 100644 index 000000000..7bffef9c3 --- /dev/null +++ b/tests/generator/check_bad_box/bad.height.POSCAR @@ -0,0 +1,16 @@ +Si8 +1.0 +1.5044712174663648e+01 0.0000000000000000e+00 0.0000000000000000e+00 +2.1550292128393154e+00 4.6052924123850527e+00 0.0000000000000000e+00 +-4.8723010805707583e+00 -2.0033682930713113e+00 1.5924563614826326e+00 +Si +8 +Cartesian + 4.9743339600 0.3464058271 1.0711743625 + -0.3457810400 2.5396598271 1.1364543625 + 8.0859689600 0.8673568271 0.7907843625 + 8.6812089600 -1.3051601729 1.3749843625 + 3.4236889600 2.1462798271 0.9487443625 + 10.9770889600 0.2924648271 0.3900343625 + 2.4104489600 3.8329998271 -0.0757256375 + 5.9863139600 2.6933598271 -0.0617356375 diff --git a/tests/generator/check_bad_box/bad.length.lammpstrj b/tests/generator/check_bad_box/bad.length.lammpstrj new file mode 100644 index 000000000..72219f300 --- /dev/null +++ b/tests/generator/check_bad_box/bad.length.lammpstrj @@ -0,0 +1,17 @@ +ITEM: TIMESTEP +5280 +ITEM: NUMBER OF ATOMS +8 +ITEM: BOX BOUNDS xy xz yz pp pp pp +-9.8598738203143235e+00 2.5613607315592056e+01 2.2564569351176282e+00 +2.0941875002685144e+00 4.1475715876280130e+00 8.0019818936636522e+00 +1.1929631605542741e+00 4.3055923370457059e+00 7.7239367239652201e-01 +ITEM: ATOMS id type x y z +1 1 -4.99391 3.35438 2.10847 +2 1 -0.969424 2.43249 2.0988 +3 1 7.28407 2.8735 2.62347 +4 1 12.7535 2.79479 2.52587 +5 1 11.2843 3.48997 4.38948 +6 1 18.5323 3.79371 3.76776 +7 1 22.4848 3.28742 3.92357 +8 1 4.68871 3.71 3.677 diff --git a/tests/generator/check_bad_box/good.lammpstrj b/tests/generator/check_bad_box/good.lammpstrj new file mode 100644 index 000000000..de83eb30d --- /dev/null +++ b/tests/generator/check_bad_box/good.lammpstrj @@ -0,0 +1,17 @@ +ITEM: TIMESTEP +1000 +ITEM: NUMBER OF ATOMS +8 +ITEM: BOX BOUNDS xy xz yz pp pp pp +1.8139178622650964e-01 5.3315679568790921e+00 6.0140898224392482e-02 +1.5585645904442166e-01 5.2801927596078837e+00 -4.2475821618813588e-02 +1.7257600456519429e-01 5.3259794930348008e+00 -3.3316196847694178e-02 +ITEM: ATOMS id type x y z +1 1 1.56205 1.38625 1.45027 +2 1 2.81674 2.78587 0.383858 +3 1 1.42276 3.82071 4.00808 +4 1 2.70217 5.14268 2.79767 +5 1 3.8694 1.49549 4.01017 +6 1 5.26061 2.60465 2.6905 +7 1 4.17094 4.22066 1.50687 +8 1 0.254383 0.28016 5.15933 diff --git a/tests/generator/context.py b/tests/generator/context.py index 0668e7fe2..d79bbfeb4 100644 --- a/tests/generator/context.py +++ b/tests/generator/context.py @@ -18,6 +18,7 @@ machine_file = 'machine-local.json' machine_file_v1 = 'machine-local-v1.json' param_diy_file = 'param-mg-vasp-diy.json' +param_pwmat_file = 'param-pyridine-pwmat.json' def my_file_cmp(test, f0, f1): with open(f0) as fp0 : diff --git a/tests/generator/data/mg.fcc.02x02x02 b/tests/generator/data/mg.fcc.02x02x02 new file mode 120000 index 000000000..32b1ad2c2 --- /dev/null +++ b/tests/generator/data/mg.fcc.02x02x02 @@ -0,0 +1 @@ +al.fcc.02x02x02 \ No newline at end of file diff --git a/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000000/OUT.MLMD b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000000/OUT.MLMD new file mode 100644 index 000000000..7477360a4 --- /dev/null +++ b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000000/OUT.MLMD @@ -0,0 +1,20 @@ + 6 atoms,Iteration (fs) = -0.1000000000E+01, Etot,Ep,Ek (eV) = -0.8463607047E+03 -0.8463607047E+03 0.0000000000E+00, SCF = 36 + Lattice vector (Angstrom) + 0.1080440044E+02 0.0000000000E+00 0.0000000000E+00 + 0.0000000000E+00 0.1049750233E+02 0.0000000000E+00 + 0.0000000000E+00 0.0000000000E+00 0.1095603371E+02 + Position (normalized), move_x, move_y, move_z + 6 0.095086720000000 0.162577930000000 0.146264240000000 1 1 1 + 6 0.366858130000000 0.378822980000000 0.293448980000000 1 1 1 + 1 0.404086650000000 0.560402750000000 0.498746690000000 1 1 1 + 1 0.141461450000000 0.215485070000000 0.346088290000000 1 1 1 + 7 0.626253130000000 0.697621110000000 0.674972590000000 1 1 1 + 7 0.745915950000000 0.874152660000000 0.525583090000000 1 1 1 + Force (eV/Angstrom) + 6 0.000000000000000 0.000000000000000 0.000000000000000 + 6 0.000000000000000 0.000000000000000 0.000000000000000 + 1 0.000000000000000 0.000000000000000 0.000000000000000 + 1 0.000000000000000 0.000000000000000 0.000000000000000 + 7 0.000000000000000 0.000000000000000 0.000000000000000 + 7 0.000000000000000 0.000000000000000 0.000000000000000 + ----------------------------------------------END diff --git a/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000000/etot.input b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000000/etot.input new file mode 100644 index 000000000..0e919853e --- /dev/null +++ b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000000/etot.input @@ -0,0 +1,17 @@ +4 1 +JOB = SCF +IN.ATOM = atom.config +IN.PSP1 = C.SG15.PBE.UPF +IN.PSP2 = H.SG15.PBE.UPF +IN.PSP3 = N.SG15.PBE.UPF +ECUT = 50 +E_ERROR = 1.0E-4 +RHO_ERROR = 1.0E-4 +SCF_ITER0_1 = 6 4 3 0.0000 0.025 2 +SCF_ITER0_2 = 94 4 3 1.0000 0.025 2 +XCFUNCTIONAL = PBE +CONVERGENCE = EASY +MP_N123 = 2 2 2 0 0 0 2 +OUT.RHO = F +OUT.WG = F +OUT.MLMD = T diff --git a/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000000/info b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000000/info new file mode 100644 index 000000000..fe7207fc2 --- /dev/null +++ b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000000/info @@ -0,0 +1 @@ +/home/linfengz/TIG/deepgen/pyridine/00/iter.000022/02.fp/task.006.000076/input diff --git a/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000000/poscar2config.x b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000000/poscar2config.x new file mode 100755 index 000000000..868a02263 Binary files /dev/null and b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000000/poscar2config.x differ diff --git a/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000001/OUT.MLMD b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000001/OUT.MLMD new file mode 100644 index 000000000..7477360a4 --- /dev/null +++ b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000001/OUT.MLMD @@ -0,0 +1,20 @@ + 6 atoms,Iteration (fs) = -0.1000000000E+01, Etot,Ep,Ek (eV) = -0.8463607047E+03 -0.8463607047E+03 0.0000000000E+00, SCF = 36 + Lattice vector (Angstrom) + 0.1080440044E+02 0.0000000000E+00 0.0000000000E+00 + 0.0000000000E+00 0.1049750233E+02 0.0000000000E+00 + 0.0000000000E+00 0.0000000000E+00 0.1095603371E+02 + Position (normalized), move_x, move_y, move_z + 6 0.095086720000000 0.162577930000000 0.146264240000000 1 1 1 + 6 0.366858130000000 0.378822980000000 0.293448980000000 1 1 1 + 1 0.404086650000000 0.560402750000000 0.498746690000000 1 1 1 + 1 0.141461450000000 0.215485070000000 0.346088290000000 1 1 1 + 7 0.626253130000000 0.697621110000000 0.674972590000000 1 1 1 + 7 0.745915950000000 0.874152660000000 0.525583090000000 1 1 1 + Force (eV/Angstrom) + 6 0.000000000000000 0.000000000000000 0.000000000000000 + 6 0.000000000000000 0.000000000000000 0.000000000000000 + 1 0.000000000000000 0.000000000000000 0.000000000000000 + 1 0.000000000000000 0.000000000000000 0.000000000000000 + 7 0.000000000000000 0.000000000000000 0.000000000000000 + 7 0.000000000000000 0.000000000000000 0.000000000000000 + ----------------------------------------------END diff --git a/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000001/etot.input b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000001/etot.input new file mode 100644 index 000000000..0e919853e --- /dev/null +++ b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000001/etot.input @@ -0,0 +1,17 @@ +4 1 +JOB = SCF +IN.ATOM = atom.config +IN.PSP1 = C.SG15.PBE.UPF +IN.PSP2 = H.SG15.PBE.UPF +IN.PSP3 = N.SG15.PBE.UPF +ECUT = 50 +E_ERROR = 1.0E-4 +RHO_ERROR = 1.0E-4 +SCF_ITER0_1 = 6 4 3 0.0000 0.025 2 +SCF_ITER0_2 = 94 4 3 1.0000 0.025 2 +XCFUNCTIONAL = PBE +CONVERGENCE = EASY +MP_N123 = 2 2 2 0 0 0 2 +OUT.RHO = F +OUT.WG = F +OUT.MLMD = T diff --git a/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000001/info b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000001/info new file mode 100644 index 000000000..02dfaee17 --- /dev/null +++ b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000001/info @@ -0,0 +1 @@ +/home/linfengz/TIG/deepgen/pyridine/00/iter.000022/02.fp/task.006.000070/input diff --git a/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000001/poscar2config.x b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000001/poscar2config.x new file mode 100755 index 000000000..868a02263 Binary files /dev/null and b/tests/generator/out_data_post_fp_pwmat/02.fp/task.000.000001/poscar2config.x differ diff --git a/tests/generator/out_data_post_fp_pwmat/orig/box.raw b/tests/generator/out_data_post_fp_pwmat/orig/box.raw new file mode 100644 index 000000000..f6099445f --- /dev/null +++ b/tests/generator/out_data_post_fp_pwmat/orig/box.raw @@ -0,0 +1,2 @@ +1.080440044000000022e+01 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.049750232999999966e+01 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.095603370999999981e+01 +1.080440044000000022e+01 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.049750232999999966e+01 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.095603370999999981e+01 diff --git a/tests/generator/out_data_post_fp_pwmat/orig/coord.raw b/tests/generator/out_data_post_fp_pwmat/orig/coord.raw new file mode 100644 index 000000000..9bb2eff6c --- /dev/null +++ b/tests/generator/out_data_post_fp_pwmat/orig/coord.raw @@ -0,0 +1,2 @@ +1.528406152623037917e+00 2.262055024405213111e+00 3.791754971876255986e+00 4.365913979058126060e+00 5.882829173863406425e+00 5.464285548390919622e+00 1.027354999406156821e+00 1.706662198981576894e+00 1.602475944007530240e+00 3.963682141189577290e+00 3.976695115207542930e+00 3.215036917045115850e+00 6.766289593323377538e+00 7.323279227682186310e+00 7.395022449366008388e+00 8.059174618383018185e+00 9.176419585125698219e+00 5.758306051445964080e+00 +1.528406152623037917e+00 2.262055024405213111e+00 3.791754971876255986e+00 4.365913979058126060e+00 5.882829173863406425e+00 5.464285548390919622e+00 1.027354999406156821e+00 1.706662198981576894e+00 1.602475944007530240e+00 3.963682141189577290e+00 3.976695115207542930e+00 3.215036917045115850e+00 6.766289593323377538e+00 7.323279227682186310e+00 7.395022449366008388e+00 8.059174618383018185e+00 9.176419585125698219e+00 5.758306051445964080e+00 diff --git a/tests/generator/out_data_post_fp_pwmat/orig/energy.raw b/tests/generator/out_data_post_fp_pwmat/orig/energy.raw new file mode 100644 index 000000000..b43284887 --- /dev/null +++ b/tests/generator/out_data_post_fp_pwmat/orig/energy.raw @@ -0,0 +1,2 @@ +-8.463607047000000421e+02 +-8.463607047000000421e+02 diff --git a/tests/generator/out_data_post_fp_pwmat/orig/force.raw b/tests/generator/out_data_post_fp_pwmat/orig/force.raw new file mode 100644 index 000000000..7e93007b7 --- /dev/null +++ b/tests/generator/out_data_post_fp_pwmat/orig/force.raw @@ -0,0 +1,2 @@ +0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 diff --git a/tests/generator/out_data_post_fp_pwmat/orig/type.raw b/tests/generator/out_data_post_fp_pwmat/orig/type.raw new file mode 100644 index 000000000..0970af7bc --- /dev/null +++ b/tests/generator/out_data_post_fp_pwmat/orig/type.raw @@ -0,0 +1,6 @@ +0 +0 +1 +1 +2 +2 diff --git a/tests/generator/out_data_post_fp_pwmat/orig/type_map.raw b/tests/generator/out_data_post_fp_pwmat/orig/type_map.raw new file mode 100644 index 000000000..57b04d593 --- /dev/null +++ b/tests/generator/out_data_post_fp_pwmat/orig/type_map.raw @@ -0,0 +1,3 @@ +H +C +N diff --git a/tests/generator/param-mg-vasp.json b/tests/generator/param-mg-vasp.json index 908be9afb..f2ecdcf8f 100644 --- a/tests/generator/param-mg-vasp.json +++ b/tests/generator/param-mg-vasp.json @@ -7,8 +7,8 @@ ], "init_batch_size": [16], "sys_configs": [ - ["data/mg.fcc.02x02x02/01.scale_pert/sys-0032/scale*/000010/POSCAR"], - ["data/mg.fcc.02x02x02/01.scale_pert/sys-0032/scale*/00000[8-9]/POSCAR"] + ["data/mg.fcc.02x02x02/01.scale_pert/sys-0032/scale*/000000/POSCAR"], + ["data/mg.fcc.02x02x02/01.scale_pert/sys-0032/scale*/000001/POSCAR"] ], "_comment": "0 1 2 3", "_comment": "4 5 6 7", diff --git a/tests/generator/param-pyridine-pwmat.json b/tests/generator/param-pyridine-pwmat.json new file mode 100644 index 000000000..cc2e59b39 --- /dev/null +++ b/tests/generator/param-pyridine-pwmat.json @@ -0,0 +1,123 @@ +{ + "type_map": ["H", "C", "N"], + "mass_map": [2, 16, 14], + + "init_data_prefix": "/home/linfengz/SCR/wanghan/deepgen.pyridine/init", + "init_data_sys": ["Pyridine-I", + "Pyridine-II" + ], + "init_batch_size": [1, 1], + "sys_configs": [ + ["/home/linfengz/SCR/wanghan/data/pyridine/pyI.POSCAR.01x01x01/01.scale_pert/sys-0080-0080-0016/scale-1.000/00009?/POSCAR"], + ["/home/linfengz/SCR/wanghan/data/pyridine/pyI.POSCAR.01x01x01/01.scale_pert/sys-0080-0080-0016/scale-1.000/0000[7-8]?/POSCAR"], + ["/home/linfengz/SCR/wanghan/data/pyridine/pyI.POSCAR.01x01x01/01.scale_pert/sys-0080-0080-0016/scale-1.000/0000[5-6]?/POSCAR"], + ["/home/linfengz/SCR/wanghan/data/pyridine/pyI.POSCAR.01x01x01/01.scale_pert/sys-0080-0080-0016/scale-1.000/0000[0-4]?/POSCAR"], + ["/home/linfengz/SCR/wanghan/data/pyridine/pyII.POSCAR.01x01x01/01.scale_pert/sys-0080-0080-0016/scale-1.000/00009?/POSCAR"], + ["/home/linfengz/SCR/wanghan/data/pyridine/pyII.POSCAR.01x01x01/01.scale_pert/sys-0080-0080-0016/scale-1.000/0000[7-8]?/POSCAR"], + ["/home/linfengz/SCR/wanghan/data/pyridine/pyII.POSCAR.01x01x01/01.scale_pert/sys-0080-0080-0016/scale-1.000/0000[5-6]?/POSCAR"], + ["/home/linfengz/SCR/wanghan/data/pyridine/pyII.POSCAR.01x01x01/01.scale_pert/sys-0080-0080-0016/scale-1.000/0000[0-4]?/POSCAR"] + ], + "_comment": "0 1 2 3", + "_comment": "4 5 6 7", + "sys_batch_size": [1, 1, 1, 1, + 1, 1, 1, 1 + ], + + "_comment": " 00.train ", + "numb_models": 4, + "train_param": "input.json", + "default_training_param" : { + "_comment": " model parameters", + "use_smooth": true, + "sel_a": [81, 81, 20], + "rcut_smth": 0.50, + "rcut": 6.50, + "filter_neuron": [25, 50, 100], + "filter_resnet_dt": false, + "n_axis_neuron": 12, + "n_neuron": [240, 240, 240], + "resnet_dt": true, + "coord_norm": true, + "type_fitting_net": false, + + "_comment": " traing controls", + "systems": [], + "set_prefix": "set", + "stop_batch": 400000, + "batch_size": 1, + "start_lr": 0.002, + "decay_steps": 2000, + "decay_rate": 0.95, + "seed": 0, + + "start_pref_e": 0.02, + "limit_pref_e": 2, + "start_pref_f": 1000, + "limit_pref_f": 1, + "start_pref_v": 0.0, + "limit_pref_v": 0.0, + + "_comment": " display and restart", + "_comment": " frequencies counted in batch", + "disp_file": "lcurve.out", + "disp_freq": 2000, + "numb_test": 10, + "save_freq": 20000, + "save_ckpt": "model.ckpt", + "load_ckpt": "model.ckpt", + "disp_training": true, + "time_training": true, + "profiling": false, + "profiling_file": "timeline.json", + + "_comment": "that's all" + }, + + "_comment": " 01.model_devi ", + "_comment": "model_devi_skip: the first x of the recorded frames", + "model_devi_dt": 0.001, + "model_devi_skip": 0, + "model_devi_f_trust_lo": 0.050, + "model_devi_f_trust_hi": 0.150, + "model_devi_e_trust_lo": 1e10, + "model_devi_e_trust_hi": 1e10, + "model_devi_clean_traj": false, + "model_devi_jobs": [ + {"sys_idx": [0,4], "temps": [ 50], "press": [1e0,1e1,1e2,1e3,1e4,2e4,4e4], "trj_freq": 10, "nsteps": 1000, "ensemble": "npt", "_idx": "00"}, + {"sys_idx": [1,5], "temps": [ 50], "press": [1e0,1e1,1e2,1e3,1e4,2e4,4e4], "trj_freq": 10, "nsteps": 1000, "ensemble": "npt", "_idx": "01"}, + {"sys_idx": [0,4], "temps": [ 50], "press": [1e0,1e1,1e2,1e3,1e4,2e4,4e4], "trj_freq": 10, "nsteps": 1000, "ensemble": "npt", "_idx": "02"}, + {"sys_idx": [1,5], "temps": [ 50], "press": [1e0,1e1,1e2,1e3,1e4,2e4,4e4], "trj_freq": 10, "nsteps": 1000, "ensemble": "npt", "_idx": "03"}, + {"sys_idx": [0,4], "temps": [ 100], "press": [1e0,1e1,1e2,1e3,1e4,2e4,4e4], "trj_freq": 10, "nsteps": 1000, "ensemble": "npt", "_idx": "04"}, + {"sys_idx": [1,5], "temps": [ 100], "press": [1e0,1e1,1e2,1e3,1e4,2e4,4e4], "trj_freq": 10, "nsteps": 1000, "ensemble": "npt", "_idx": "05"}, + {"sys_idx": [0,4], "temps": [ 100], "press": [1e0,1e1,1e2,1e3,1e4,2e4,4e4], "trj_freq": 10, "nsteps": 1000, "ensemble": "npt", "_idx": "06"}, + {"sys_idx": [1,5], "temps": [ 100], "press": [1e0,1e1,1e2,1e3,1e4,2e4,4e4], "trj_freq": 10, "nsteps": 1000, "ensemble": "npt", "_idx": "07"} + ], + + "_comment": " 02.fp ", + "fp_style": "pwmat", + "shuffle_poscar": false, + "fp_task_max": 100, + "fp_task_min": 10, + "fp_pp_path": ".", + "fp_pp_files": ["C.SG15.PBE.UPF", "H.SG15.PBE.UPF", "N.SG15.PBE.UPF"], + "user_fp_params": { + "node1": 4, + "node2": 1, + "job": "SCF", + "in.atom": "atom.config", + "in.psp1": "C.SG15.PBE.UPF", + "in.psp2": "H.SG15.PBE.UPF", + "in.psp3": "N.SG15.PBE.UPF", + "ecut": 50, + "flag_symm": 2, + "e_error": 1.0E-4, + "rho_error": 1.0E-4, + "scf_iter0_1": "6 4 3 0.0000 0.025 2", + "scf_iter0_2": "94 4 3 1.0000 0.025 2", + "xcfunctional": "PBE", + "convergence": "EASY", + "kspacing": 0.1 + + }, + "_comment": " that's all " +} diff --git a/tests/generator/test_check_bad_box.py b/tests/generator/test_check_bad_box.py new file mode 100644 index 000000000..ba210128e --- /dev/null +++ b/tests/generator/test_check_bad_box.py @@ -0,0 +1,22 @@ +import os,sys,json,glob,shutil +import dpdata +import numpy as np +import unittest + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +__package__ = 'generator' +from .context import check_bad_box + +class TestCheckBadBox(unittest.TestCase): + def test_length_ratio(self): + dirname = os.path.dirname(__file__) + conf_bad = os.path.join(dirname, 'check_bad_box', 'bad.length.lammpstrj') + conf_good = os.path.join(dirname, 'check_bad_box', 'good.lammpstrj') + self.assertTrue(check_bad_box(conf_bad, 'length_ratio:5')) + self.assertFalse(check_bad_box(conf_good, 'length_ratio:5')) + + def test_height_ratio(self): + dirname = os.path.dirname(__file__) + conf_bad = os.path.join(dirname, 'check_bad_box', 'bad.height.POSCAR') + self.assertTrue(check_bad_box(conf_bad, 'height_ratio:5', fmt = 'vasp/POSCAR')) + self.assertFalse(check_bad_box(conf_bad, 'length_ratio:5', fmt = 'vasp/POSCAR')) diff --git a/tests/generator/test_make_fp.py b/tests/generator/test_make_fp.py index e77e94d36..8e679f1e9 100644 --- a/tests/generator/test_make_fp.py +++ b/tests/generator/test_make_fp.py @@ -2,6 +2,7 @@ import dpdata import numpy as np import unittest +import importlib sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) __package__ = 'generator' @@ -26,6 +27,7 @@ from .comp_sys import test_coord from .comp_sys import test_cell from pymatgen.io.vasp import Kpoints,Incar +from .context import param_pwmat_file import scipy.constants as pc vasp_incar_ref = "PREC=A\n\ @@ -185,7 +187,23 @@ &FORCES ON\n\ &END FORCES\n\ &END PRINT\n\ -&END FORCE_EVAL\n" +&END FORCE_EVAL\n"; + +pwmat_input_ref = "4 1\n\ +in.atom=atom.config\n\ +ecut=50\n\ +e_error=0.0001\n\ +rho_error=0.0001\n\ +scf_iter0_1=6 4 3 0.0000 0.025 2\n\ +scf_iter0_2=94 4 3 1.0000 0.025 2\n\ +mp_n123=147 57 39 0 0 0 2\n\ +out.wg=F\n\ +out.rho=F\n\ +out.mlmd=T\n\ +job=scf\n\ +IN.PSP1 = C.SG15.PBE.UPF\n\ +IN.PSP2 = H.SG15.PBE.UPF\n\ +IN.PSP3 = N.SG15.PBE.UPF\n"; @@ -469,6 +487,18 @@ def _check_cp2k_input_head(testCase, idx) : +def _check_pwmat_input(testCase, idx): + fp_path = os.path.join('iter.%06d' % idx, '02.fp') + tasks = glob.glob(os.path.join(fp_path, 'task.*')) + cwd = os.getcwd() + for ii in tasks : + os.chdir(ii) + os.system("sed -i '8c mp_n123=147 57 39 0 0 0 2' etot.input") + with open('etot.input') as fp: + lines = fp.read() + testCase.assertEqual(lines.strip(), pwmat_input_ref.strip()) + os.chdir(cwd) + class TestMakeFPPwscf(unittest.TestCase): def test_make_fp_pwscf(self): if os.path.isdir('iter.000000') : @@ -710,11 +740,12 @@ def test_make_fp_vasp_ele_temp(self): class TestMakeFPGaussian(unittest.TestCase): - def test_make_fp_gaussian(self): + def make_fp_gaussian(self, multiplicity="auto"): if os.path.isdir('iter.000000') : shutil.rmtree('iter.000000') with open (param_gaussian_file, 'r') as fp : jdata = json.load (fp) + jdata['user_fp_params']['multiplicity'] = multiplicity with open (machine_file, 'r') as fp: mdata = json.load (fp) md_descript = [] @@ -736,6 +767,13 @@ def test_make_fp_gaussian(self): _check_potcar(self, 0, jdata['fp_pp_path'], jdata['fp_pp_files']) shutil.rmtree('iter.000000') + @unittest.skipIf(importlib.util.find_spec("openbabel") is None, "requires openbabel") + def test_make_fp_gaussian(self): + self.make_fp_gaussian() + + def test_make_fp_gaussian_multiplicity_one(self): + self.make_fp_gaussian(multiplicity=1) + def test_detect_multiplicity(self): # oxygen O2 3 self._check_multiplicity(['O', 'O'], 3) @@ -779,6 +817,33 @@ def test_make_fp_cp2k(self): shutil.rmtree('iter.000000') +class TestMakeFPPWmat(unittest.TestCase): + def test_make_fp_pwmat(self): + if os.path.isdir('iter.000000') : + shutil.rmtree('iter.000000') + with open (param_pwmat_file, 'r') as fp : + jdata = json.load (fp) + with open (machine_file, 'r') as fp: + mdata = json.load (fp) + md_descript = [] + nsys = 2 + nmd = 3 + n_frame = 10 + for ii in range(nsys) : + tmp = [] + for jj in range(nmd) : + tmp.append(np.arange(0, 0.29, 0.29/10)) + md_descript.append(tmp) + atom_types = [0, 1, 2, 2, 0, 1] + type_map = jdata['type_map'] + _make_fake_md(0, md_descript, atom_types, type_map) + make_fp(0, jdata, {}) + _check_sel(self, 0, jdata['fp_task_max'], jdata['model_devi_f_trust_lo'], jdata['model_devi_f_trust_hi']) + _check_poscars(self, 0, jdata['fp_task_max'], jdata['type_map']) + _check_pwmat_input(self, 0) + _check_potcar(self, 0, jdata['fp_pp_path'], jdata['fp_pp_files']) + shutil.rmtree('iter.000000') + if __name__ == '__main__': unittest.main() diff --git a/tests/generator/test_make_md.py b/tests/generator/test_make_md.py index 3ee930344..4f67b1fbe 100644 --- a/tests/generator/test_make_md.py +++ b/tests/generator/test_make_md.py @@ -146,8 +146,38 @@ def test_make_model_devi (self) : _check_confs(self, 0, jdata) _check_traj_dir(self, 0) _check_pt(self, 0, jdata) - shutil.rmtree('iter.000000') - + #shutil.rmtree('iter.000000') + + def test_make_model_devi_nopbc_npt (self) : + if os.path.isdir('iter.000000') : + shutil.rmtree('iter.000000') + with open (param_file, 'r') as fp : + jdata = json.load (fp) + jdata['model_devi_nopbc'] = True + with open (machine_file, 'r') as fp: + mdata = json.load (fp) + _make_fake_models(0, jdata['numb_models']) + cwd = os.getcwd() + with self.assertRaises(RuntimeError) : + make_model_devi(0, jdata, mdata) + os.chdir(cwd) + + def test_make_model_devi_nopbc_nvt (self) : + if os.path.isdir('iter.000000') : + shutil.rmtree('iter.000000') + with open (param_file, 'r') as fp : + jdata = json.load (fp) + jdata['model_devi_nopbc'] = True + jdata['model_devi_jobs'][0]['ensemble'] = 'nvt' + with open (machine_file, 'r') as fp: + mdata = json.load (fp) + _make_fake_models(0, jdata['numb_models']) + make_model_devi(0, jdata, mdata) + _check_pb(self, 0) + # _check_confs(self, 0, jdata) + _check_traj_dir(self, 0) + _check_pt(self, 0, jdata) + # shutil.rmtree('iter.000000') class TestMakeModelDeviRevMat(unittest.TestCase): diff --git a/tests/generator/test_make_train.py b/tests/generator/test_make_train.py index c64487f55..3fa820d82 100644 --- a/tests/generator/test_make_train.py +++ b/tests/generator/test_make_train.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import os,sys,json,glob,shutil +import os,sys,json,glob,shutil,dpdata import numpy as np import unittest @@ -101,7 +101,7 @@ def _check_model_input_dict(testCase, input_dict, init_data_sys, init_batch_size testCase.assertEqual(input_dict[ii], default_training_param[ii]) -def _check_model_inputs_v1(testCase, iter_idx, jdata) : +def _check_model_inputs_v1(testCase, iter_idx, jdata, reuse = False) : train_param = jdata.get('train_param', 'input.json') numb_models = jdata['numb_models'] use_ele_temp = jdata.get('use_ele_temp', 0) @@ -133,6 +133,18 @@ def _check_model_inputs_v1(testCase, iter_idx, jdata) : _check_model_input_dict(testCase, jdata0['loss'], init_data_sys, init_batch_size, default_training_param['loss']) _check_model_input_dict(testCase, jdata0['learning_rate'], init_data_sys, init_batch_size, default_training_param['learning_rate']) _check_model_input_dict(testCase, jdata0['training'], init_data_sys, init_batch_size, default_training_param['training']) + if reuse: + testCase.assertEqual(jdata['training_reuse_stop_batch'], + jdata0['training']['stop_batch']) + testCase.assertEqual(jdata['training_reuse_start_lr'], + jdata0['learning_rate']['start_lr']) + testCase.assertEqual(jdata['training_reuse_start_pref_e'], + jdata0['loss']['start_pref_e']) + testCase.assertEqual(jdata['training_reuse_start_pref_f'], + jdata0['loss']['start_pref_f']) + old_ratio = jdata['training_reuse_old_ratio'] + testCase.assertEqual(jdata0['training']['auto_prob_style'], + "prob_sys_size; 0:1:%f; 1:2:%f" % (old_ratio, 1-old_ratio)) def _make_fake_fp(iter_idx, sys_idx, nframes): @@ -143,13 +155,14 @@ def _make_fake_fp(iter_idx, sys_idx, nframes): os.makedirs(dirname, exist_ok = True) dirname = os.path.join('iter.%06d' % iter_idx, '02.fp', - 'data.%03d' % sys_idx) + 'data.%03d' % sys_idx) os.makedirs(dirname, exist_ok = True) - box_str = ['0' for ii in range(9)] - box_str = ' '.join(box_str) - with open(os.path.join(dirname, 'box.raw'), 'w') as fp : - for ii in range(nframes) : - fp.write(box_str + '\n') + tmp_sys = dpdata.LabeledSystem('out_data_post_fp_vasp/02.fp/task.000.000000/OUTCAR') + tmp_sys1 = tmp_sys.sub_system([0]) + tmp_sys2 = tmp_sys1 + for ii in range(1, nframes): + tmp_sys2.append(tmp_sys1) + tmp_sys2.to('deepmd/npy', dirname) def _check_pb_link(testCase, iter_idx, numb_models) : @@ -235,6 +248,38 @@ def test_1_data_v1(self) : # remove testing dirs shutil.rmtree('iter.000001') shutil.rmtree('iter.000000') + + + def test_1_data_reuse_v1(self) : + with open (param_file_v1, 'r') as fp : + jdata = json.load (fp) + jdata.pop('use_ele_temp', None) + jdata['training_reuse_iter'] = 1 + jdata['training_reuse_old_ratio'] = 0.8 + jdata['training_reuse_stop_batch'] = 400000 + jdata['training_reuse_start_lr'] = 1e-4 + jdata['training_reuse_start_pref_e'] = 0.1 + jdata['training_reuse_start_pref_f'] = 100 + with open (machine_file_v1, 'r') as fp: + mdata = json.load (fp) + make_train(0, jdata, mdata) + # make fake fp results #data == fp_task_min + _make_fake_fp(0, 0, jdata['fp_task_min']) + # make iter1 train + make_train(1, jdata, mdata) + # check data is linked + self.assertTrue(os.path.isdir(os.path.join('iter.000001', '00.train', 'data.iters', 'iter.000000', '02.fp'))) + # check old models are linked + self.assertTrue(os.path.isdir(os.path.join('iter.000001', '00.train', '000', 'old'))) + self.assertTrue(os.path.isdir(os.path.join('iter.000001', '00.train', '001', 'old'))) + self.assertTrue(os.path.isdir(os.path.join('iter.000001', '00.train', '002', 'old'))) + self.assertTrue(os.path.isdir(os.path.join('iter.000001', '00.train', '003', 'old'))) + # check models inputs + _check_model_inputs_v1(self, 1, jdata, reuse = True) + # remove testing dirs + shutil.rmtree('iter.000001') + shutil.rmtree('iter.000000') + def test_1_data_v1_eletron_temp(self) : with open (param_file_v1_et, 'r') as fp : diff --git a/tests/generator/test_post_fp.py b/tests/generator/test_post_fp.py index 652f627ab..629f416b8 100644 --- a/tests/generator/test_post_fp.py +++ b/tests/generator/test_post_fp.py @@ -25,6 +25,7 @@ from .comp_sys import test_coord from .comp_sys import test_cell from .comp_sys import CompLabeledSys +from .context import param_pwmat_file class TestPostFPVasp(unittest.TestCase): @@ -225,6 +226,22 @@ def setUp(self): self.system_2 = dpdata.LabeledSystem('iter.000000/02.fp/data.000', fmt = 'deepmd/raw') +class TestPostFPPWmat(unittest.TestCase, CompLabeledSys): + def setUp(self): + self.places = 5 + self.e_places = 5 + self.f_places = 5 + self.v_places = 2 + assert os.path.isdir('out_data_post_fp_pwmat'), 'out data for post fp pwmat should exist' + if os.path.isdir('iter.000000') : + shutil.rmtree('iter.000000') + shutil.copytree('out_data_post_fp_pwmat', 'iter.000000') + with open (param_pwmat_file, 'r') as fp : + jdata = json.load (fp) + post_fp(0, jdata) + self.system_1 = dpdata.LabeledSystem('iter.000000/orig', fmt = 'deepmd/raw') + self.system_2 = dpdata.LabeledSystem('iter.000000/02.fp/data.000', fmt = 'deepmd/raw') + if __name__ == '__main__': unittest.main()