From 6bfb5f1120a7943e972facf2fe130ad5bf3e8061 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Sun, 24 Dec 2017 20:25:51 -0500 Subject: [PATCH 01/51] Jenkins test --- examples/jenkins.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/examples/jenkins.sh b/examples/jenkins.sh index 9d9087e6..ac060b88 100644 --- a/examples/jenkins.sh +++ b/examples/jenkins.sh @@ -1,15 +1,21 @@ #!/bin/bash rm /tigress/alexeys/model_checkpoints/* +rm -rf /tigress/alexeys/processed_shots +rm -rf /tigress/alexeys/processed_shotlists +rm -rf /tigress/alexeys/normalization + ls ${PWD} -module load anaconda +echo "Jenkins test Python3.6" +export PYTHONHASHSEED=0 +module load anaconda3 +source activate PPPL_dev3 module load cudatoolkit/8.0 -module load openmpi/intel-17.0/2.1.0/64 intel/17.0/64/17.0.4.196 intel-mkl/2017.3/4/64 module load cudnn/cuda-8.0/6.0 -source activate PPPL - +module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 +module load intel/17.0/64/17.0.4.196 export OMPI_MCA_btl="tcp,self,sm" echo $SLURM_NODELIST From 2249774974eceeb38f89cbd7f0dc334bc30985ca Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Sun, 24 Dec 2017 20:27:18 -0500 Subject: [PATCH 02/51] Tab-space insonsistency --- plasma/primitives/data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plasma/primitives/data.py b/plasma/primitives/data.py index d6139f66..d537038b 100644 --- a/plasma/primitives/data.py +++ b/plasma/primitives/data.py @@ -121,8 +121,8 @@ def load_data(self,prepath,shot,dtype='float32'): def fetch_data_basic(self,machine,shot_num,c,path=None): - if path is None: - path = self.get_path(machine) + if path is None: + path = self.get_path(machine) success = False mapping = None try: @@ -283,7 +283,7 @@ def get_channel_num(self,machine): def fetch_data(self,machine,shot_num,c): time,data,mapping,success = self.fetch_data_basic(machine,shot_num,c) - mapping = None #we are not interested in the whole profile + mapping = None #we are not interested in the whole profile channel_num = self.get_channel_num(machine) if channel_num is not None and success: if np.ndim(data) != 2: From b31baec830e08719003aedbf8938cfaf81005c1b Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Mon, 25 Dec 2017 12:55:43 -0500 Subject: [PATCH 03/51] Notebooks belong in examples/notebooks --- .../notebooks/Analyze Hyperparameter Tuning.ipynb | 0 .../notebooks/Signal Influence.ipynb | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename Analyze Hyperparameter Tuning.ipynb => examples/notebooks/Analyze Hyperparameter Tuning.ipynb (100%) rename Signal Influence.ipynb => examples/notebooks/Signal Influence.ipynb (100%) diff --git a/Analyze Hyperparameter Tuning.ipynb b/examples/notebooks/Analyze Hyperparameter Tuning.ipynb similarity index 100% rename from Analyze Hyperparameter Tuning.ipynb rename to examples/notebooks/Analyze Hyperparameter Tuning.ipynb diff --git a/Signal Influence.ipynb b/examples/notebooks/Signal Influence.ipynb similarity index 100% rename from Signal Influence.ipynb rename to examples/notebooks/Signal Influence.ipynb From 14bb70f2144414aa8d2efe8a518e77169843b1bc Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Mon, 25 Dec 2017 12:56:59 -0500 Subject: [PATCH 04/51] Fix IndexError in Python3 -- floating point slice index --- plasma/primitives/data.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plasma/primitives/data.py b/plasma/primitives/data.py index d537038b..9085f8ee 100644 --- a/plasma/primitives/data.py +++ b/plasma/primitives/data.py @@ -1,3 +1,4 @@ +from __future__ import division import numpy as np import time import sys,os @@ -205,7 +206,7 @@ def load_data(self,prepath,shot,dtype='float32'): if np.ndim(data) == 1: data = np.expand_dims(data,axis=0) #_ = data[0,0] - T = data.shape[0]/2 #time is stored twice, once for mapping and once for signal + T = data.shape[0]//2 #time is stored twice, once for mapping and once for signal mapping = data[:T,1:] remapping = np.linspace(self.mapping_range[0],self.mapping_range[1],self.num_channels) t = data[:T,0] From 7ff580639af20ddb7f0f764ab9b711494ae9ecc1 Mon Sep 17 00:00:00 2001 From: Alexey Svyatkovskiy Date: Mon, 25 Dec 2017 13:47:03 -0500 Subject: [PATCH 05/51] Trigger Jenkins build --- examples/jenkins.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/jenkins.sh b/examples/jenkins.sh index ac060b88..f442388e 100644 --- a/examples/jenkins.sh +++ b/examples/jenkins.sh @@ -5,7 +5,6 @@ rm -rf /tigress/alexeys/processed_shots rm -rf /tigress/alexeys/processed_shotlists rm -rf /tigress/alexeys/normalization - ls ${PWD} echo "Jenkins test Python3.6" From 1e25e7c4130665e51151df04b9679b717f35f7e6 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Mon, 25 Dec 2017 14:59:57 -0500 Subject: [PATCH 06/51] Fix TypeError in layer definitions in Python3 --- plasma/models/builder.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/plasma/models/builder.py b/plasma/models/builder.py index 2e0fc0c0..29d47fb0 100644 --- a/plasma/models/builder.py +++ b/plasma/models/builder.py @@ -1,3 +1,4 @@ +from __future__ import division import keras from keras.models import Sequential, Model from keras.layers import Input @@ -139,20 +140,20 @@ def slicer_output_shape(input_shape,indices): for i in range(model_conf['num_conv_layers']): div_fac = 2**i - pre_rnn_1D = Convolution1D(num_conv_filters/div_fac,size_conv_filters,padding='valid',activation='relu') (pre_rnn_1D) - pre_rnn_1D = Convolution1D(num_conv_filters/div_fac,1,padding='valid',activation='relu') (pre_rnn_1D) + pre_rnn_1D = Convolution1D(num_conv_filters//div_fac,size_conv_filters,padding='valid',activation='relu') (pre_rnn_1D) + pre_rnn_1D = Convolution1D(num_conv_filters//div_fac,1,padding='valid',activation='relu') (pre_rnn_1D) pre_rnn_1D = MaxPooling1D(pool_size) (pre_rnn_1D) pre_rnn_1D = Flatten() (pre_rnn_1D) pre_rnn_1D = Dense(dense_size,activation='relu',kernel_regularizer=l2(dense_regularization),bias_regularizer=l2(dense_regularization),activity_regularizer=l2(dense_regularization)) (pre_rnn_1D) - pre_rnn_1D = Dense(dense_size/4,activation='relu',kernel_regularizer=l2(dense_regularization),bias_regularizer=l2(dense_regularization),activity_regularizer=l2(dense_regularization)) (pre_rnn_1D) + pre_rnn_1D = Dense(dense_size//4,activation='relu',kernel_regularizer=l2(dense_regularization),bias_regularizer=l2(dense_regularization),activity_regularizer=l2(dense_regularization)) (pre_rnn_1D) pre_rnn = Concatenate() ([pre_rnn_0D,pre_rnn_1D]) else: pre_rnn = pre_rnn_input if model_conf['rnn_layers'] == 0 or model_conf['extra_dense_input']: pre_rnn = Dense(dense_size,activation='relu',kernel_regularizer=l2(dense_regularization),bias_regularizer=l2(dense_regularization),activity_regularizer=l2(dense_regularization)) (pre_rnn) - pre_rnn = Dense(dense_size/2,activation='relu',kernel_regularizer=l2(dense_regularization),bias_regularizer=l2(dense_regularization),activity_regularizer=l2(dense_regularization)) (pre_rnn) - pre_rnn = Dense(dense_size/4,activation='relu',kernel_regularizer=l2(dense_regularization),bias_regularizer=l2(dense_regularization),activity_regularizer=l2(dense_regularization)) (pre_rnn) + pre_rnn = Dense(dense_size//2,activation='relu',kernel_regularizer=l2(dense_regularization),bias_regularizer=l2(dense_regularization),activity_regularizer=l2(dense_regularization)) (pre_rnn) + pre_rnn = Dense(dense_size//4,activation='relu',kernel_regularizer=l2(dense_regularization),bias_regularizer=l2(dense_regularization),activity_regularizer=l2(dense_regularization)) (pre_rnn) pre_rnn_model = Model(inputs = pre_rnn_input,outputs=pre_rnn) x_input = Input(batch_shape = batch_input_shape) From 45a797d1a1ef323df0e6703e3c48df5999e29396 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Mon, 25 Dec 2017 18:37:12 -0500 Subject: [PATCH 07/51] Restrict Jenkins build to 2 epochs --- examples/jenkins.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/jenkins.sh b/examples/jenkins.sh index f442388e..c3c33a25 100644 --- a/examples/jenkins.sh +++ b/examples/jenkins.sh @@ -17,5 +17,7 @@ module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 module load intel/17.0/64/17.0.4.196 export OMPI_MCA_btl="tcp,self,sm" +sed -i -e 's/num_epochs: 1000/num_epochs: 2/g' conf.yaml + echo $SLURM_NODELIST srun python mpi_learn.py From 4b88bf0f01093938e1d841742c1eda06fe50bbe1 Mon Sep 17 00:00:00 2001 From: Alexey Svyatkovskiy Date: Mon, 25 Dec 2017 22:51:58 -0500 Subject: [PATCH 08/51] Trigger Jenkins build From ee385337702b7eac6bd3a69808e173d032dadcd0 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Tue, 26 Dec 2017 10:28:27 -0500 Subject: [PATCH 09/51] No longer in use --- examples/compare_batch_iterators.py | 75 ----------------------------- 1 file changed, 75 deletions(-) delete mode 100644 examples/compare_batch_iterators.py diff --git a/examples/compare_batch_iterators.py b/examples/compare_batch_iterators.py deleted file mode 100644 index ce57250f..00000000 --- a/examples/compare_batch_iterators.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python -from functools import partial -from itertools import tee - -class Loader(): - def example_batch_generator(self,n): - for batch in range(n): - yield batch - -class MPIModel(): - def __init__(self,batch_generator): - self.batch_iterator = batch_generator - - def train_epochs(self,M): - num_total = 8 - for epoch in range(M): - num_so_far = 0 - print ("Batch iter. summary: {}{}".format(self,self.batch_iterator)) - for batch in self.batch_iterator(): - num_so_far += 1 - - whatever=batch - print ("Next batch id: {}".format(batch)) - if num_so_far > num_total: break - print "+++++++" - - -class MPIModel_default(): - def __init__(self,batch_generator): - self.batch_iterator = batch_generator - - def train_epochs(self,M): - num_total = 8 #number of samples per epoch - batch_generator_func = self.batch_iterator() - - for iepoch in range(M): - #print ("Batch iter. summary: {}{} epoch: {}".format(self,self.batch_iterator,iepoch)) - num_so_far = 0 - - while num_so_far < num_total: - num_so_far += 1 - - try: - batch = batch_generator_func.next() - except StopIteration: - batch_generator_func = self.batch_iterator() - batch = batch_generator_func.next() - print ("Next batch id: {}".format(batch)) - - print "+++++++" - - - -def main(): - num_batches = 10 - epochs = 3 - - loader = Loader() - batch_generator = partial(loader.example_batch_generator,n=num_batches) - my_example_class = MPIModel(batch_generator) - my_example_class.train_epochs(epochs) - -def main_default(): - num_batches = 10 - epochs = 3 - - loader = Loader() - batch_generator = partial(loader.example_batch_generator,n=num_batches) - my_example_class = MPIModel_default(batch_generator) - my_example_class.train_epochs(epochs) - -if __name__=='__main__': - import timeit - #print min(timeit.Timer(setup=main).repeat(7, 1000)) - print min(timeit.Timer(setup=main_default).repeat(7, 1000)) From 81838b6350cd06d9635885eb7604c8b8090583c7 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Tue, 26 Dec 2017 12:00:33 -0500 Subject: [PATCH 10/51] PYTHONHASHSEED is not needed with hashlb md5 --- examples/guarantee_preprocessed.py | 2 -- examples/mpi_learn.py | 2 -- examples/signal_influence.py | 2 -- examples/simple_augmentation.py | 2 -- plasma/utils/batch_jobs.py | 4 ++-- 5 files changed, 2 insertions(+), 10 deletions(-) diff --git a/examples/guarantee_preprocessed.py b/examples/guarantee_preprocessed.py index 3f005f0e..67826ad0 100644 --- a/examples/guarantee_preprocessed.py +++ b/examples/guarantee_preprocessed.py @@ -11,8 +11,6 @@ pprint(conf) from plasma.preprocessor.preprocess import guarantee_preprocessed -os.environ["PYTHONHASHSEED"] = "0" - ##################################################### ####################PREPROCESSING#################### ##################################################### diff --git a/examples/mpi_learn.py b/examples/mpi_learn.py index e6fd6503..a965a86b 100644 --- a/examples/mpi_learn.py +++ b/examples/mpi_learn.py @@ -24,8 +24,6 @@ import random import numpy as np -os.environ["PYTHONHASHSEED"] = "0" - import matplotlib matplotlib.use('Agg') diff --git a/examples/signal_influence.py b/examples/signal_influence.py index 509af551..fc987177 100644 --- a/examples/signal_influence.py +++ b/examples/signal_influence.py @@ -26,8 +26,6 @@ import copy from functools import partial -os.environ["PYTHONHASHSEED"] = "0" - import matplotlib matplotlib.use('Agg') diff --git a/examples/simple_augmentation.py b/examples/simple_augmentation.py index e6c05a78..3afd19c9 100644 --- a/examples/simple_augmentation.py +++ b/examples/simple_augmentation.py @@ -26,8 +26,6 @@ import copy from functools import partial -os.environ["PYTHONHASHSEED"] = "0" - import matplotlib matplotlib.use('Agg') diff --git a/plasma/utils/batch_jobs.py b/plasma/utils/batch_jobs.py index f0bcea7f..a2c50a31 100644 --- a/plasma/utils/batch_jobs.py +++ b/plasma/utils/batch_jobs.py @@ -53,7 +53,7 @@ def create_slurm_script(subdir,num_nodes,idx,executable_name,use_mpi,env_name="f # f.write('rm -f /tigress/{}/model_checkpoints/*.h5\n'.format(user)) f.write('cd {}\n'.format(subdir)) f.write('export OMPI_MCA_btl=\"tcp,self,sm\"\n') - f.write('srun env PYTHONHASHSEED=0 python {}\n'.format(executable_name)) + f.write('srun python {}\n'.format(executable_name)) f.write('echo "done."') return filepath @@ -72,7 +72,7 @@ def create_pbs_script(subdir,num_nodes,idx,executable_name,use_mpi,env_name="frn f.write('module load tensorflow\n') # f.write('rm $HOME/tigress/alexeys/model_checkpoints/*\n') f.write('cd {}\n'.format(subdir)) - f.write('aprun -n {} -N1 env PYTHONHASHSEED=0 env KERAS_HOME={} singularity exec $TENSORFLOW_CONTAINER python3 {}\n'.format(str(num_nodes),subdir,executable_name)) + f.write('aprun -n {} -N1 env KERAS_HOME={} singularity exec $TENSORFLOW_CONTAINER python3 {}\n'.format(str(num_nodes),subdir,executable_name)) f.write('echo "done."') return filepath From db55cbceeb3d73d8fade33b16982833c795e331c Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Tue, 26 Dec 2017 12:01:12 -0500 Subject: [PATCH 11/51] Switch to hashlib.md5 hashing --- plasma/conf_parser.py | 3 ++- plasma/models/builder.py | 3 ++- plasma/models/shallow_runner.py | 6 ++++-- plasma/primitives/data.py | 3 ++- plasma/primitives/shots.py | 4 ++-- 5 files changed, 12 insertions(+), 7 deletions(-) diff --git a/plasma/conf_parser.py b/plasma/conf_parser.py index 1ded3a1a..42982ecd 100644 --- a/plasma/conf_parser.py +++ b/plasma/conf_parser.py @@ -6,6 +6,7 @@ import uuid import yaml +import hashlib def parameters(input_file): """Parse yaml file of configuration parameters.""" @@ -167,7 +168,7 @@ def parameters(input_file): return params def get_unique_signal_hash(signals): - return hash(tuple(sorted(signals))) + return int(hashlib.md5(''.join(tuple(map(lambda x: x.description, sorted(signals)))).encode('utf-8')).hexdigest(),16) #make sure 1D signals come last! This is necessary for model builder. def sort_by_channels(list_of_signals): diff --git a/plasma/models/builder.py b/plasma/models/builder.py index 29d47fb0..0b223356 100644 --- a/plasma/models/builder.py +++ b/plasma/models/builder.py @@ -22,6 +22,7 @@ from copy import deepcopy from plasma.utils.downloading import makedirs_process_safe +import hashlib class LossHistory(Callback): def on_train_begin(self, logs=None): @@ -40,7 +41,7 @@ def get_unique_id(self): this_conf = deepcopy(self.conf) #don't make hash dependent on number of epochs. this_conf['training']['num_epochs'] = 0 - unique_id = hash(dill.dumps(this_conf)) + unique_id = int(hashlib.md5((dill.dumps(this_conf).decode('unicode_escape')).encode('utf-8')).hexdigest(),16) return unique_id def get_0D_1D_indices(self): diff --git a/plasma/models/shallow_runner.py b/plasma/models/shallow_runner.py index 140454cc..b9ccf883 100644 --- a/plasma/models/shallow_runner.py +++ b/plasma/models/shallow_runner.py @@ -29,6 +29,8 @@ from keras.utils.generic_utils import Progbar +import hashlib + debug_use_shots = 100000 model_filename = "saved_model.pkl" dataset_path = "dataset.npz" @@ -85,8 +87,8 @@ def load_shots(self,shot_list,is_inference=False,as_list=False,num_samples=np.In def get_save_prepath(self): prepath = self.loader.conf['paths']['processed_prepath'] use_signals = self.loader.conf['paths']['use_signals'] - identifying_tuple = tuple(sorted(use_signals)) - save_prepath = prepath + "shallow/use_signals_{}/".format(hash(identifying_tuple)) + identifying_tuple = ''.join(tuple(map(lambda x: x.description, sorted(use_signals)))).encode('utf-8') + save_prepath = prepath + "shallow/use_signals_{}/".format(int(hashlib.md5(identifying_tuple).hexdigest(),16)) return save_prepath def process(self,shot): diff --git a/plasma/primitives/data.py b/plasma/primitives/data.py index 9085f8ee..c18dd1f4 100644 --- a/plasma/primitives/data.py +++ b/plasma/primitives/data.py @@ -184,7 +184,8 @@ def __lt__(self,other): return self.description.__lt__(other.description) def __hash__(self): - return self.description.__hash__() + import hashlib + return int(hashlib.md5(self.description.encode('utf-8')).hexdigest(),16) def __str__(self): return self.description diff --git a/plasma/primitives/shots.py b/plasma/primitives/shots.py index 383ed7e6..09685093 100644 --- a/plasma/primitives/shots.py +++ b/plasma/primitives/shots.py @@ -323,8 +323,8 @@ def __eq__(self,other): return self.get_id_str().__eq__(other.get_id_str()) def __hash__(self): - return self.get_id_str().__hash__() - + import hashlib + return int(hashlib.md5(self.get_id_str().encode('utf-8')).hexdigest(),16) def __str__(self): string = 'number: {}\n'.format(self.number) From c15f05ebe6c61966f23e6023da2f9e86534ccecd Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Tue, 26 Dec 2017 12:02:01 -0500 Subject: [PATCH 12/51] Full build integration chain for Python 2 and Python 3 --- examples/jenkins.sh | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/examples/jenkins.sh b/examples/jenkins.sh index c3c33a25..88b65a09 100644 --- a/examples/jenkins.sh +++ b/examples/jenkins.sh @@ -1,23 +1,38 @@ #!/bin/bash -rm /tigress/alexeys/model_checkpoints/* -rm -rf /tigress/alexeys/processed_shots -rm -rf /tigress/alexeys/processed_shotlists -rm -rf /tigress/alexeys/normalization +export OMPI_MCA_btl="tcp,self,sm" ls ${PWD} echo "Jenkins test Python3.6" -export PYTHONHASHSEED=0 +rm /tigress/alexeys/model_checkpoints/* +rm -rf /tigress/alexeys/processed_shots +rm -rf /tigress/alexeys/processed_shotlists +rm -rf /tigress/alexeys/normalization module load anaconda3 source activate PPPL_dev3 module load cudatoolkit/8.0 module load cudnn/cuda-8.0/6.0 module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 module load intel/17.0/64/17.0.4.196 -export OMPI_MCA_btl="tcp,self,sm" sed -i -e 's/num_epochs: 1000/num_epochs: 2/g' conf.yaml echo $SLURM_NODELIST srun python mpi_learn.py + +echo "Jenkins test Python2.7" +rm /tigress/alexeys/model_checkpoints/* +rm -rf /tigress/alexeys/normalization + +source deactivate +module purge +module load anaconda +source activate PPPL +module load cudatoolkit/8.0 +module load cudnn/cuda-8.0/6.0 +module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 +module load intel/17.0/64/17.0.4.196 + +echo $SLURM_NODELIST +srun python mpi_learn.py From 8bfac27b09a052b2860d8e4bd15997428ac3e656 Mon Sep 17 00:00:00 2001 From: Alexey Svyatkovskiy Date: Tue, 26 Dec 2017 12:39:56 -0500 Subject: [PATCH 13/51] Trigger jenkins test From 47025539b3904642102d0c282b948c813296cc3a Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Tue, 26 Dec 2017 13:58:54 -0500 Subject: [PATCH 14/51] Refactor --- examples/jenkins.sh | 38 -------------------------------------- 1 file changed, 38 deletions(-) delete mode 100644 examples/jenkins.sh diff --git a/examples/jenkins.sh b/examples/jenkins.sh deleted file mode 100644 index 88b65a09..00000000 --- a/examples/jenkins.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -export OMPI_MCA_btl="tcp,self,sm" - -ls ${PWD} - -echo "Jenkins test Python3.6" -rm /tigress/alexeys/model_checkpoints/* -rm -rf /tigress/alexeys/processed_shots -rm -rf /tigress/alexeys/processed_shotlists -rm -rf /tigress/alexeys/normalization -module load anaconda3 -source activate PPPL_dev3 -module load cudatoolkit/8.0 -module load cudnn/cuda-8.0/6.0 -module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 -module load intel/17.0/64/17.0.4.196 - -sed -i -e 's/num_epochs: 1000/num_epochs: 2/g' conf.yaml - -echo $SLURM_NODELIST -srun python mpi_learn.py - -echo "Jenkins test Python2.7" -rm /tigress/alexeys/model_checkpoints/* -rm -rf /tigress/alexeys/normalization - -source deactivate -module purge -module load anaconda -source activate PPPL -module load cudatoolkit/8.0 -module load cudnn/cuda-8.0/6.0 -module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 -module load intel/17.0/64/17.0.4.196 - -echo $SLURM_NODELIST -srun python mpi_learn.py From ffc35915b695a032a999a0f72f870fc0dc5787ff Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Tue, 26 Dec 2017 13:59:05 -0500 Subject: [PATCH 15/51] Refactor --- jenkins-ci/jenkins.sh | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 jenkins-ci/jenkins.sh diff --git a/jenkins-ci/jenkins.sh b/jenkins-ci/jenkins.sh new file mode 100644 index 00000000..88b65a09 --- /dev/null +++ b/jenkins-ci/jenkins.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +export OMPI_MCA_btl="tcp,self,sm" + +ls ${PWD} + +echo "Jenkins test Python3.6" +rm /tigress/alexeys/model_checkpoints/* +rm -rf /tigress/alexeys/processed_shots +rm -rf /tigress/alexeys/processed_shotlists +rm -rf /tigress/alexeys/normalization +module load anaconda3 +source activate PPPL_dev3 +module load cudatoolkit/8.0 +module load cudnn/cuda-8.0/6.0 +module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 +module load intel/17.0/64/17.0.4.196 + +sed -i -e 's/num_epochs: 1000/num_epochs: 2/g' conf.yaml + +echo $SLURM_NODELIST +srun python mpi_learn.py + +echo "Jenkins test Python2.7" +rm /tigress/alexeys/model_checkpoints/* +rm -rf /tigress/alexeys/normalization + +source deactivate +module purge +module load anaconda +source activate PPPL +module load cudatoolkit/8.0 +module load cudnn/cuda-8.0/6.0 +module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 +module load intel/17.0/64/17.0.4.196 + +echo $SLURM_NODELIST +srun python mpi_learn.py From 3da647581a3c81707ec17cd6194b3199e5b2dbaa Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Tue, 26 Dec 2017 13:59:43 -0500 Subject: [PATCH 16/51] Refactor --- jenkins-ci/{jenkins.sh => jenkins3.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename jenkins-ci/{jenkins.sh => jenkins3.sh} (100%) diff --git a/jenkins-ci/jenkins.sh b/jenkins-ci/jenkins3.sh similarity index 100% rename from jenkins-ci/jenkins.sh rename to jenkins-ci/jenkins3.sh From 02b5921425dee5c7204689a283acdcdf38aff862 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Tue, 26 Dec 2017 14:00:12 -0500 Subject: [PATCH 17/51] Python3 only --- jenkins-ci/jenkins3.sh | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/jenkins-ci/jenkins3.sh b/jenkins-ci/jenkins3.sh index 88b65a09..699e78bc 100644 --- a/jenkins-ci/jenkins3.sh +++ b/jenkins-ci/jenkins3.sh @@ -20,19 +20,3 @@ sed -i -e 's/num_epochs: 1000/num_epochs: 2/g' conf.yaml echo $SLURM_NODELIST srun python mpi_learn.py - -echo "Jenkins test Python2.7" -rm /tigress/alexeys/model_checkpoints/* -rm -rf /tigress/alexeys/normalization - -source deactivate -module purge -module load anaconda -source activate PPPL -module load cudatoolkit/8.0 -module load cudnn/cuda-8.0/6.0 -module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 -module load intel/17.0/64/17.0.4.196 - -echo $SLURM_NODELIST -srun python mpi_learn.py From 0f63dd240f0763ffa3ace4069f1780c19f423cf5 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Tue, 26 Dec 2017 14:01:19 -0500 Subject: [PATCH 18/51] Python2 only --- jenkins-ci/jenkins2.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 jenkins-ci/jenkins2.sh diff --git a/jenkins-ci/jenkins2.sh b/jenkins-ci/jenkins2.sh new file mode 100644 index 00000000..87e3e282 --- /dev/null +++ b/jenkins-ci/jenkins2.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +ls ${PWD} + +echo "Jenkins test Python2.7" +rm /tigress/alexeys/model_checkpoints/* +rm -rf /tigress/alexeys/normalization + +source deactivate +module purge +module load anaconda +source activate PPPL +module load cudatoolkit/8.0 +module load cudnn/cuda-8.0/6.0 +module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 +module load intel/17.0/64/17.0.4.196 + +echo $SLURM_NODELIST +srun python mpi_learn.py From d9d30b7683c98dca5fa404ed9fd4c21d54c062e0 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Tue, 26 Dec 2017 14:02:13 -0500 Subject: [PATCH 19/51] Adjust folder structure --- jenkins-ci/jenkins2.sh | 1 + jenkins-ci/jenkins3.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/jenkins-ci/jenkins2.sh b/jenkins-ci/jenkins2.sh index 87e3e282..201ee7f8 100644 --- a/jenkins-ci/jenkins2.sh +++ b/jenkins-ci/jenkins2.sh @@ -16,4 +16,5 @@ module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 module load intel/17.0/64/17.0.4.196 echo $SLURM_NODELIST +cd examples srun python mpi_learn.py diff --git a/jenkins-ci/jenkins3.sh b/jenkins-ci/jenkins3.sh index 699e78bc..068e20ec 100644 --- a/jenkins-ci/jenkins3.sh +++ b/jenkins-ci/jenkins3.sh @@ -19,4 +19,5 @@ module load intel/17.0/64/17.0.4.196 sed -i -e 's/num_epochs: 1000/num_epochs: 2/g' conf.yaml echo $SLURM_NODELIST +cd examples srun python mpi_learn.py From 9b973c94da4e3ab00b79efe3dc7bea195ba749c4 Mon Sep 17 00:00:00 2001 From: Alexey Svyatkovskiy Date: Tue, 26 Dec 2017 14:28:42 -0500 Subject: [PATCH 20/51] Trigger jenkins test From a79e511fda749d8f34869bac0db07a51e4b3b515 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Tue, 26 Dec 2017 17:40:03 -0500 Subject: [PATCH 21/51] Roll back to the single script option --- jenkins-ci/{jenkins3.sh => jenkins.sh} | 17 +++++++++++++++++ jenkins-ci/jenkins2.sh | 20 -------------------- 2 files changed, 17 insertions(+), 20 deletions(-) rename jenkins-ci/{jenkins3.sh => jenkins.sh} (59%) delete mode 100644 jenkins-ci/jenkins2.sh diff --git a/jenkins-ci/jenkins3.sh b/jenkins-ci/jenkins.sh similarity index 59% rename from jenkins-ci/jenkins3.sh rename to jenkins-ci/jenkins.sh index 068e20ec..b9a6d457 100644 --- a/jenkins-ci/jenkins3.sh +++ b/jenkins-ci/jenkins.sh @@ -21,3 +21,20 @@ sed -i -e 's/num_epochs: 1000/num_epochs: 2/g' conf.yaml echo $SLURM_NODELIST cd examples srun python mpi_learn.py + +echo "Jenkins test Python2.7" +rm /tigress/alexeys/model_checkpoints/* +rm -rf /tigress/alexeys/normalization + +source deactivate +module purge +module load anaconda +source activate PPPL +module load cudatoolkit/8.0 +module load cudnn/cuda-8.0/6.0 +module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 +module load intel/17.0/64/17.0.4.196 + +echo $SLURM_NODELIST +cd examples +srun python mpi_learn.py diff --git a/jenkins-ci/jenkins2.sh b/jenkins-ci/jenkins2.sh deleted file mode 100644 index 201ee7f8..00000000 --- a/jenkins-ci/jenkins2.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -ls ${PWD} - -echo "Jenkins test Python2.7" -rm /tigress/alexeys/model_checkpoints/* -rm -rf /tigress/alexeys/normalization - -source deactivate -module purge -module load anaconda -source activate PPPL -module load cudatoolkit/8.0 -module load cudnn/cuda-8.0/6.0 -module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 -module load intel/17.0/64/17.0.4.196 - -echo $SLURM_NODELIST -cd examples -srun python mpi_learn.py From 09d2bda1a3dda7afe07e39eb24c57a200eb8d02c Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Tue, 26 Dec 2017 17:41:54 -0500 Subject: [PATCH 22/51] Reinstall for each Python --- jenkins-ci/jenkins.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/jenkins-ci/jenkins.sh b/jenkins-ci/jenkins.sh index b9a6d457..7941bc94 100644 --- a/jenkins-ci/jenkins.sh +++ b/jenkins-ci/jenkins.sh @@ -16,6 +16,8 @@ module load cudnn/cuda-8.0/6.0 module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 module load intel/17.0/64/17.0.4.196 +python setup.py install + sed -i -e 's/num_epochs: 1000/num_epochs: 2/g' conf.yaml echo $SLURM_NODELIST @@ -35,6 +37,9 @@ module load cudnn/cuda-8.0/6.0 module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 module load intel/17.0/64/17.0.4.196 +cd .. +python setup.py install + echo $SLURM_NODELIST cd examples srun python mpi_learn.py From 65cb115c5da1a76754ced7fb1664e3063eee2a89 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Tue, 26 Dec 2017 21:16:11 -0500 Subject: [PATCH 23/51] Add Jenkins Slurm methods --- plasma/utils/batch_jobs.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/plasma/utils/batch_jobs.py b/plasma/utils/batch_jobs.py index a2c50a31..5ea2e03a 100644 --- a/plasma/utils/batch_jobs.py +++ b/plasma/utils/batch_jobs.py @@ -11,8 +11,6 @@ def generate_working_dirname(run_directory): s += "_{}".format(uuid.uuid4()) return run_directory + s - - def get_executable_name(conf): shallow = conf['model']['shallow'] if shallow: @@ -31,6 +29,36 @@ def start_slurm_job(subdir,num_nodes,i,conf,shallow,env_name="frnn",env_type="an sp.Popen("sbatch "+script,shell=True) +def create_jenkins_script(subdir,num_nodes,executable_name,test_configuration,env_name="frnn",env_type="anaconda"): + filename = "jenkins_{}_{}.cmd".format(test_configuration[0],test_configuration[1]) #version of Python and the dataset + filepath = os.path.join(subdir,filename) + user = getpass.getuser() + with open(filepath,"w") as f: + f.write('#!/usr/bin/bash\n') + f.write('export OMPI_MCA_btl=\"tcp,self,sm\"\n') + f.write('echo \"Jenkins test {}\"\n'.format(test_configuration[1])) + f.write('rm /tigress/alexeys/model_checkpoints/*\n') + f.write('rm -rf /tigress/alexeys/processed_shots\n') + f.write('rm -rf /tigress/alexeys/processed_shotlists\n') + f.write('rm -rf /tigress/alexeys/normalization\n') + f.write('module load {}\n'.format(env_type)) + f.write('source activate {}\n'.format(env_name)) + f.write('module load cudatoolkit/8.0\n') + f.write('module load cudnn/cuda-8.0/6.0\n') + f.write('module load openmpi/cuda-8.0/intel-17.0/2.1.0/64\n') + f.write('module load intel/17.0/64/17.0.4.196\n') + f.write('cd /home/alexeys/jenkins/workspace/FRNM/PPPL\n') + f.write('python setup.py install\n') + f.write('cd {}\n'.format(subdir)) + f.write('srun -N 2 -n 8 python {}\n'.format(executable_name)) + + return filepath + +def start_jenkins_job(subdir,num_nodes,executable_name,test_configuration,env_name,env_type): + os.system(" ".join(["cp -p",executable_name,subdir])) + script = create_jenkins_script(subdir,num_nodes,executable_name,test_configuration,env_name,env_type) + sp.Popen("sh "+script,shell=True) + def start_pbs_job(subdir,num_nodes,i,conf,shallow,env_name="frnn",env_type="anaconda"): executable_name,use_mpi = get_executable_name(conf) os.system(" ".join(["cp -p",executable_name,subdir])) From ced42ed77577e588af5e9edb7caa99241b4aea52 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Tue, 26 Dec 2017 21:16:30 -0500 Subject: [PATCH 24/51] Jenkins driver script --- jenkins-ci/run_jenkins.py | 56 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 jenkins-ci/run_jenkins.py diff --git a/jenkins-ci/run_jenkins.py b/jenkins-ci/run_jenkins.py new file mode 100644 index 00000000..1cc1d36d --- /dev/null +++ b/jenkins-ci/run_jenkins.py @@ -0,0 +1,56 @@ +from plasma.utils.batch_jobs import generate_working_dirname,copy_files_to_environment,start_jenkins_job +import yaml +import sys,os,getpass +import plasma.conf + +num_nodes = 3 +test_matrix = [("Python3","jet_data"),("Python2","jet_data")] + +run_directory = "{}/{}/jenkins/".format(plasma.conf.conf['fs_path'],getpass.getuser()) +template_path = os.environ['PWD'] +conf_name = "conf.yaml" +executable_name = "mpi_learn.py" + +def generate_conf_file(test_configuration,template_path = "../",save_path = "./",conf_name="conf.yaml"): + assert(template_path != save_path) + with open(os.path.join(template_path,conf_name), 'r') as yaml_file: + conf = yaml.load(yaml_file) + conf['training']['num_epochs'] = 2 + conf['paths']['data'] = test_configuration[1] + if test_configuration[1] == "Python3": + conf['env']['name'] = "PPPL_dev3" + conf['env']['type'] = "anaconda3" + else: + conf['env']['name'] = "PPPL" + conf['env']['type'] = "anaconda" + + with open(os.path.join(save_path,conf_name), 'w') as outfile: + yaml.dump(conf, outfile, default_flow_style=False) + return conf + +working_directory = generate_working_dirname(run_directory) +os.makedirs(working_directory) + +os.system(" ".join(["cp -p",os.path.join(template_path,conf_name),working_directory])) +os.system(" ".join(["cp -p",os.path.join(template_path,executable_name),working_directory])) + +#os.chdir(working_directory) +#print("Going into {}".format(working_directory)) + +for ci in test_matrix: + subdir = working_directory + "/{}/".format(ci[0]) + os.makedirs(subdir) + copy_files_to_environment(subdir) + print("Making modified conf") + conf = generate_conf_file(ci,working_directory,subdir,conf_name) + print("Starting job") + if ci[1] == "Python3": + env_name = "PPPL_dev3" + env_type = "anaconda3" + else: + env_name = "PPPL" + env_type = "anaconda" + start_jenkins_job(subdir,num_nodes,executable_name,ci,env_name,env_type) + + +print("submitted jobs.") From 61fa87927ad20cb889197f4e9935f4989d99a62c Mon Sep 17 00:00:00 2001 From: Alexey Svyatkovskiy Date: Tue, 26 Dec 2017 21:19:27 -0500 Subject: [PATCH 25/51] Trigger jenkins test From 638539313e829e1442717e167eeebe36f5c26a53 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Tue, 26 Dec 2017 21:22:41 -0500 Subject: [PATCH 26/51] Assigned based on half of num_nodes --- plasma/utils/batch_jobs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plasma/utils/batch_jobs.py b/plasma/utils/batch_jobs.py index 5ea2e03a..56203b97 100644 --- a/plasma/utils/batch_jobs.py +++ b/plasma/utils/batch_jobs.py @@ -1,3 +1,4 @@ +from __future__ import division from pprint import pprint import yaml import datetime @@ -50,7 +51,7 @@ def create_jenkins_script(subdir,num_nodes,executable_name,test_configuration,en f.write('cd /home/alexeys/jenkins/workspace/FRNM/PPPL\n') f.write('python setup.py install\n') f.write('cd {}\n'.format(subdir)) - f.write('srun -N 2 -n 8 python {}\n'.format(executable_name)) + f.write('srun -N {} -n {} python {}\n'.format(num_nodes//2, num_nodes//2*4, executable_name)) return filepath From acc988758e01da84989e37aafa4234e1bbb80dd7 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Tue, 26 Dec 2017 21:23:17 -0500 Subject: [PATCH 27/51] Set num_nodes as in Jenkins project area --- jenkins-ci/run_jenkins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-ci/run_jenkins.py b/jenkins-ci/run_jenkins.py index 1cc1d36d..57be2675 100644 --- a/jenkins-ci/run_jenkins.py +++ b/jenkins-ci/run_jenkins.py @@ -3,7 +3,7 @@ import sys,os,getpass import plasma.conf -num_nodes = 3 +num_nodes = 4 #Set in the Jenkins project area!! test_matrix = [("Python3","jet_data"),("Python2","jet_data")] run_directory = "{}/{}/jenkins/".format(plasma.conf.conf['fs_path'],getpass.getuser()) From 76caa749794185e3e9f99698fcefc5614adec960 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Tue, 26 Dec 2017 21:37:03 -0500 Subject: [PATCH 28/51] Easier parsing with sed --- examples/conf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/conf.yaml b/examples/conf.yaml index 1b308cd8..a71947fb 100644 --- a/examples/conf.yaml +++ b/examples/conf.yaml @@ -10,7 +10,7 @@ paths: signal_prepath: '/signal_data/' #/signal_data/jet/ shot_list_dir: '/shot_lists/' tensorboard_save_path: '/Graph/' - data: 'jet_data' #'d3d_to_jet_data' #'d3d_to_jet_data' # 'jet_to_d3d_data' #jet_data + data: jet_data #'d3d_to_jet_data' #'d3d_to_jet_data' # 'jet_to_d3d_data' #jet_data specific_signals: [] #['q95','li','ip','betan','energy','lm','pradcore','pradedge','pradtot','pin','torquein','tmamp1','tmamp2','tmfreq1','tmfreq2','pechin','energydt','ipdirect','etemp_profile','edens_profile'] #if left empty will use all valid signals defined on a machine. Only use if need a custom set executable: "mpi_learn.py" shallow_executable: "learn.py" From 7d09aad76530898dac4aac192899bb6db645fb94 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Tue, 26 Dec 2017 21:39:42 -0500 Subject: [PATCH 29/51] Python based driver scripot for Jenkins --- jenkins-ci/jenkins.py | 48 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 jenkins-ci/jenkins.py diff --git a/jenkins-ci/jenkins.py b/jenkins-ci/jenkins.py new file mode 100644 index 00000000..f7d4f252 --- /dev/null +++ b/jenkins-ci/jenkins.py @@ -0,0 +1,48 @@ +#!/bin/bash +from subprocess import Popen + +Popen("export OMPI_MCA_btl=\"tcp,self,sm\"",shell=True) + +Popen("echo \"Jenkins test Python3.6\"",shell=True) +Popen("rm /tigress/alexeys/model_checkpoints/*",shell=True) +Popen("rm -rf /tigress/alexeys/processed_shots",shell=True) +Popen("rm -rf /tigress/alexeys/processed_shotlists",shell=True) +Popen("rm -rf /tigress/alexeys/normalization",shell=True) +Popen("module load anaconda3",shell=True) +Popen("source activate PPPL_dev3",shell=True) +Popen("module load cudatoolkit/8.0",shell=True) +Popen("module load cudnn/cuda-8.0/6.0",shell=True) +Popen("module load openmpi/cuda-8.0/intel-17.0/2.1.0/64",shell=True) +Popen("module load intel/17.0/64/17.0.4.196",shell=True) + +Popen("python setup.py install",shell=True) + +Popen("sed -i -e 's/num_epochs: 1000/num_epochs: 2/g' conf.yaml",shell=True) + +Popen("echo $SLURM_NODELIST",shell=True) +Popen("cd examples",shell=True) +Popen("srun python mpi_learn.py",shell=True).wait() + +Popen("echo \"Jenkins test Python2.7\"",shell=True) +Popen("rm /tigress/alexeys/model_checkpoints/*",shell=True) +Popen("rm -rf /tigress/alexeys/processed_shots",shell=True) +Popen("rm -rf /tigress/alexeys/processed_shotlists",shell=True) +Popen("rm -rf /tigress/alexeys/normalization",shell=True) + +Popen("source deactivate",shell=True) +Popen("module purge",shell=True) +Popen("module load anaconda",shell=True) +Popen("source activate PPPL",shell=True) +Popen("module load cudatoolkit/8.0",shell=True) +Popen("module load cudnn/cuda-8.0/6.0",shell=True) +Popen("module load openmpi/cuda-8.0/intel-17.0/2.1.0/64",shell=True) +Popen("module load intel/17.0/64/17.0.4.196",shell=True) + +Popen("cd ..",shell=True) +Popen("python setup.py install",shell=True) + +Popen("sed -i -e 's/data: jet_data/data: d3d_data/g' conf.yaml",shell=True) + +Popen("echo $SLURM_NODELIST",shell=True) +Popen("cd examples",shell=True) +Popen("srun python mpi_learn.py",shell=True).wait() From eff3c9be66668e0ddeae2b055702065faffdbeec Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Wed, 27 Dec 2017 11:54:42 -0500 Subject: [PATCH 30/51] Fix error in UnivariateSpline affecting edens_profile --- plasma/primitives/data.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/plasma/primitives/data.py b/plasma/primitives/data.py index c18dd1f4..3e8136e8 100644 --- a/plasma/primitives/data.py +++ b/plasma/primitives/data.py @@ -226,8 +226,11 @@ def load_data(self,prepath,shot,dtype='float32'): sig_interp = np.zeros((timesteps,self.num_channels)) for i in range(timesteps): _,order = np.unique(mapping[i,:],return_index=True) #make sure the mapping is ordered and unique - f = UnivariateSpline(mapping[i,order],sig[i,order],s=0,k=1,ext=0) - sig_interp[i,:] = f(remapping) + if sig[i,order].shape[0] > 2: + f = UnivariateSpline(mapping[i,order],sig[i,order],s=0,k=1,ext=0) + sig_interp[i,:] = f(remapping) + else: + print('Signal {}, shot {} has not enough points for linear interpolation. dfitpack.error: (m>k) failed for hidden m: fpcurf0:m=1'.format(self.description,shot.number)) return t,sig_interp,True From 2aa248623fc1b66363705ac86e6f36f14b425df6 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Wed, 27 Dec 2017 11:55:38 -0500 Subject: [PATCH 31/51] Version 1.0 --- plasma/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plasma/version.py b/plasma/version.py index 78235354..758ad1a2 100644 --- a/plasma/version.py +++ b/plasma/version.py @@ -1,6 +1,6 @@ import re -__version__ = "0.3.0" +__version__ = "1.0.0" version = __version__ From d13ee33f4301474eae56c77f26e12fc240703476 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Wed, 27 Dec 2017 11:59:51 -0500 Subject: [PATCH 32/51] return None,None,False in case of problem --- plasma/primitives/data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/plasma/primitives/data.py b/plasma/primitives/data.py index 3e8136e8..1354f506 100644 --- a/plasma/primitives/data.py +++ b/plasma/primitives/data.py @@ -231,6 +231,7 @@ def load_data(self,prepath,shot,dtype='float32'): sig_interp[i,:] = f(remapping) else: print('Signal {}, shot {} has not enough points for linear interpolation. dfitpack.error: (m>k) failed for hidden m: fpcurf0:m=1'.format(self.description,shot.number)) + return None,None,False return t,sig_interp,True From 6027021e93a407316e917331f9fea717dac34aea Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Thu, 28 Dec 2017 11:59:26 -0500 Subject: [PATCH 33/51] Add dedicated jenkins shotlists --- jenkins-ci/jenkins.py | 3 ++- jenkins-ci/jenkins.sh | 4 +++- plasma/conf_parser.py | 15 +++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/jenkins-ci/jenkins.py b/jenkins-ci/jenkins.py index f7d4f252..f42c3a78 100644 --- a/jenkins-ci/jenkins.py +++ b/jenkins-ci/jenkins.py @@ -18,6 +18,7 @@ Popen("python setup.py install",shell=True) Popen("sed -i -e 's/num_epochs: 1000/num_epochs: 2/g' conf.yaml",shell=True) +Popen("sed -i -e 's/data: jet_data/data: jenkins_jet/g' conf.yaml",shell=True) Popen("echo $SLURM_NODELIST",shell=True) Popen("cd examples",shell=True) @@ -41,7 +42,7 @@ Popen("cd ..",shell=True) Popen("python setup.py install",shell=True) -Popen("sed -i -e 's/data: jet_data/data: d3d_data/g' conf.yaml",shell=True) +Popen("sed -i -e 's/data: jenkins_jet/data: jenkins_d3d/g' conf.yaml",shell=True) Popen("echo $SLURM_NODELIST",shell=True) Popen("cd examples",shell=True) diff --git a/jenkins-ci/jenkins.sh b/jenkins-ci/jenkins.sh index 7941bc94..218b251c 100644 --- a/jenkins-ci/jenkins.sh +++ b/jenkins-ci/jenkins.sh @@ -19,6 +19,7 @@ module load intel/17.0/64/17.0.4.196 python setup.py install sed -i -e 's/num_epochs: 1000/num_epochs: 2/g' conf.yaml +sed -i -e 's/data: jet_data/data: jenkins_jet/g' conf.yaml echo $SLURM_NODELIST cd examples @@ -26,7 +27,6 @@ srun python mpi_learn.py echo "Jenkins test Python2.7" rm /tigress/alexeys/model_checkpoints/* -rm -rf /tigress/alexeys/normalization source deactivate module purge @@ -40,6 +40,8 @@ module load intel/17.0/64/17.0.4.196 cd .. python setup.py install +sed -i -e 's/data: jenkins_jet/data: jenkins_d3d/g' conf.yaml + echo $SLURM_NODELIST cd examples srun python mpi_learn.py diff --git a/plasma/conf_parser.py b/plasma/conf_parser.py index 42982ecd..15bf4dbe 100644 --- a/plasma/conf_parser.py +++ b/plasma/conf_parser.py @@ -70,12 +70,17 @@ def parameters(input_file): #shot lists jet_carbon_wall = ShotListFiles(jet,params['paths']['shot_list_dir'],['CWall_clear.txt','CFC_unint.txt'],'jet carbon wall data') jet_iterlike_wall = ShotListFiles(jet,params['paths']['shot_list_dir'],['ILW_unint.txt','BeWall_clear.txt'],'jet iter like wall data') + + jenkins_jet_carbon_wall = ShotListFiles(jet,params['paths']['shot_list_dir'],['jenkins_CWall_clear.txt','jenkins_CFC_unint.txt'],'Subset of jet carbon wall data for Jenkins tests') + jenkins_jet_iterlike_wall = ShotListFiles(jet,params['paths']['shot_list_dir'],['jenkins_ILW_unint.txt','jenkins_BeWall_clear.txt'],'Subset of jet iter like wall data for Jenkins tests') + jet_full = ShotListFiles(jet,params['paths']['shot_list_dir'],['ILW_unint.txt','BeWall_clear.txt','CWall_clear.txt','CFC_unint.txt'],'jet full data') d3d_10000 = ShotListFiles(d3d,params['paths']['shot_list_dir'],['d3d_clear_10000.txt','d3d_disrupt_10000.txt'],'d3d data 10000 ND and D shots') d3d_1000 = ShotListFiles(d3d,params['paths']['shot_list_dir'],['d3d_clear_1000.txt','d3d_disrupt_1000.txt'],'d3d data 1000 ND and D shots') d3d_100 = ShotListFiles(d3d,params['paths']['shot_list_dir'],['d3d_clear_100.txt','d3d_disrupt_100.txt'],'d3d data 100 ND and D shots') d3d_full = ShotListFiles(d3d,params['paths']['shot_list_dir'],['d3d_clear_data_avail.txt','d3d_disrupt_data_avail.txt'],'d3d data since shot 125500') + d3d_jenkins = ShotListFiles(d3d,params['paths']['shot_list_dir'],['jenkins_d3d_clear.txt','jenkins_d3d_disrupt.txt'],'Subset of d3d data for Jenkins test') d3d_jb_full = ShotListFiles(d3d,params['paths']['shot_list_dir'],['shotlist_JaysonBarr_clear.txt','shotlist_JaysonBarr_disrupt.txt'],'d3d shots since 160000-170000') nstx_full = ShotListFiles(nstx,params['paths']['shot_list_dir'],['disrupt_nstx.txt'],'nstx shots (all are disruptive') @@ -96,6 +101,10 @@ def parameters(input_file): params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = jet_signals + elif params['paths']['data'] == 'jenkins_jet': + params['paths']['shot_files'] = [jenkins_jet_carbon_wall] + params['paths']['shot_files_test'] = [jenkins_jet_iterlike_wall] + params['paths']['use_signals_dict'] = jet_signals elif params['paths']['data'] == 'd3d_data': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] @@ -117,6 +126,12 @@ def parameters(input_file): params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = d3d_signals + elif params['paths']['data'] == 'jenkins_d3d': + params['paths']['shot_files'] = [d3d_jenkins] + params['paths']['shot_files_test'] = [] + params['paths']['use_signals_dict'] = {'q95':q95,'li':li,'ip':ip,'lm':lm,'betan':betan,'energy':energy,'dens':dens,'pradcore':pradcore,'pradedge':pradedge,'pin':pin,'torquein':torquein,'ipdirect':ipdirect,'iptarget':iptarget,'iperr':iperr, +'etemp_profile':etemp_profile ,'edens_profile':edens_profile} + #cross-machine elif params['paths']['data'] == 'jet_to_d3d_data': From d194f50ddd267e6908953344a033ffe25dd7639e Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Thu, 28 Dec 2017 12:15:10 -0500 Subject: [PATCH 34/51] Proper folder structure --- jenkins-ci/jenkins.py | 10 ++++++---- jenkins-ci/jenkins.sh | 8 ++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/jenkins-ci/jenkins.py b/jenkins-ci/jenkins.py index f42c3a78..13685a03 100644 --- a/jenkins-ci/jenkins.py +++ b/jenkins-ci/jenkins.py @@ -17,11 +17,12 @@ Popen("python setup.py install",shell=True) +Popen("echo $SLURM_NODELIST",shell=True) +Popen("cd examples",shell=True) + Popen("sed -i -e 's/num_epochs: 1000/num_epochs: 2/g' conf.yaml",shell=True) Popen("sed -i -e 's/data: jet_data/data: jenkins_jet/g' conf.yaml",shell=True) -Popen("echo $SLURM_NODELIST",shell=True) -Popen("cd examples",shell=True) Popen("srun python mpi_learn.py",shell=True).wait() Popen("echo \"Jenkins test Python2.7\"",shell=True) @@ -42,8 +43,9 @@ Popen("cd ..",shell=True) Popen("python setup.py install",shell=True) -Popen("sed -i -e 's/data: jenkins_jet/data: jenkins_d3d/g' conf.yaml",shell=True) - Popen("echo $SLURM_NODELIST",shell=True) Popen("cd examples",shell=True) + +Popen("sed -i -e 's/data: jenkins_jet/data: jenkins_d3d/g' conf.yaml",shell=True) + Popen("srun python mpi_learn.py",shell=True).wait() diff --git a/jenkins-ci/jenkins.sh b/jenkins-ci/jenkins.sh index 218b251c..1d6f5dbb 100644 --- a/jenkins-ci/jenkins.sh +++ b/jenkins-ci/jenkins.sh @@ -18,11 +18,11 @@ module load intel/17.0/64/17.0.4.196 python setup.py install +echo $SLURM_NODELIST +cd examples sed -i -e 's/num_epochs: 1000/num_epochs: 2/g' conf.yaml sed -i -e 's/data: jet_data/data: jenkins_jet/g' conf.yaml -echo $SLURM_NODELIST -cd examples srun python mpi_learn.py echo "Jenkins test Python2.7" @@ -40,8 +40,8 @@ module load intel/17.0/64/17.0.4.196 cd .. python setup.py install -sed -i -e 's/data: jenkins_jet/data: jenkins_d3d/g' conf.yaml - echo $SLURM_NODELIST cd examples +sed -i -e 's/data: jenkins_jet/data: jenkins_d3d/g' conf.yaml + srun python mpi_learn.py From 574a77c327eff6a7b561b82689a39f65bd55cde4 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Thu, 28 Dec 2017 12:32:08 -0500 Subject: [PATCH 35/51] Proper folder structure --- jenkins-ci/jenkins.py | 1 + jenkins-ci/jenkins.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/jenkins-ci/jenkins.py b/jenkins-ci/jenkins.py index 13685a03..bb5f07a0 100644 --- a/jenkins-ci/jenkins.py +++ b/jenkins-ci/jenkins.py @@ -15,6 +15,7 @@ Popen("module load openmpi/cuda-8.0/intel-17.0/2.1.0/64",shell=True) Popen("module load intel/17.0/64/17.0.4.196",shell=True) +Popen("cd ..",shell=True) Popen("python setup.py install",shell=True) Popen("echo $SLURM_NODELIST",shell=True) diff --git a/jenkins-ci/jenkins.sh b/jenkins-ci/jenkins.sh index 1d6f5dbb..8e1000fa 100644 --- a/jenkins-ci/jenkins.sh +++ b/jenkins-ci/jenkins.sh @@ -16,6 +16,7 @@ module load cudnn/cuda-8.0/6.0 module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 module load intel/17.0/64/17.0.4.196 +cd .. python setup.py install echo $SLURM_NODELIST From 53626d31b4520ed0ce6aae4c8947dfd128bb6eb2 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Thu, 28 Dec 2017 12:46:32 -0500 Subject: [PATCH 36/51] Proper folder structure --- jenkins-ci/jenkins.py | 2 +- jenkins-ci/jenkins.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-ci/jenkins.py b/jenkins-ci/jenkins.py index bb5f07a0..38d6a4fe 100644 --- a/jenkins-ci/jenkins.py +++ b/jenkins-ci/jenkins.py @@ -15,7 +15,7 @@ Popen("module load openmpi/cuda-8.0/intel-17.0/2.1.0/64",shell=True) Popen("module load intel/17.0/64/17.0.4.196",shell=True) -Popen("cd ..",shell=True) +Popen("cd /home/alexeys/jenkins/workspace/FRNM/PPPL",shell=True) Popen("python setup.py install",shell=True) Popen("echo $SLURM_NODELIST",shell=True) diff --git a/jenkins-ci/jenkins.sh b/jenkins-ci/jenkins.sh index 8e1000fa..a84a7845 100644 --- a/jenkins-ci/jenkins.sh +++ b/jenkins-ci/jenkins.sh @@ -16,7 +16,7 @@ module load cudnn/cuda-8.0/6.0 module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 module load intel/17.0/64/17.0.4.196 -cd .. +cd /home/alexeys/jenkins/workspace/FRNM/PPPL python setup.py install echo $SLURM_NODELIST From 4527475e8eba394446fd35f5776f0e3066d87b48 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Thu, 28 Dec 2017 12:51:02 -0500 Subject: [PATCH 37/51] Debug --- jenkins-ci/jenkins.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/jenkins-ci/jenkins.sh b/jenkins-ci/jenkins.sh index a84a7845..ce02f8e9 100644 --- a/jenkins-ci/jenkins.sh +++ b/jenkins-ci/jenkins.sh @@ -2,7 +2,7 @@ export OMPI_MCA_btl="tcp,self,sm" -ls ${PWD} +echo ${PWD} echo "Jenkins test Python3.6" rm /tigress/alexeys/model_checkpoints/* @@ -16,11 +16,16 @@ module load cudnn/cuda-8.0/6.0 module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 module load intel/17.0/64/17.0.4.196 +echo ${PWD} cd /home/alexeys/jenkins/workspace/FRNM/PPPL +echo ${PWD} python setup.py install +echo ${PWD} echo $SLURM_NODELIST cd examples +echo ${PWD} +ls ${PWD} sed -i -e 's/num_epochs: 1000/num_epochs: 2/g' conf.yaml sed -i -e 's/data: jet_data/data: jenkins_jet/g' conf.yaml From 2b85a83c886923a5f38402822d82bf2a4a8645a2 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Thu, 28 Dec 2017 12:52:37 -0500 Subject: [PATCH 38/51] Debug --- jenkins-ci/jenkins.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/jenkins-ci/jenkins.sh b/jenkins-ci/jenkins.sh index ce02f8e9..234759f2 100644 --- a/jenkins-ci/jenkins.sh +++ b/jenkins-ci/jenkins.sh @@ -49,5 +49,4 @@ python setup.py install echo $SLURM_NODELIST cd examples sed -i -e 's/data: jenkins_jet/data: jenkins_d3d/g' conf.yaml - srun python mpi_learn.py From 5fe9636742c813cef6daed42de2d26f9c13da9e5 Mon Sep 17 00:00:00 2001 From: Alexey Svyatkovskiy Date: Thu, 28 Dec 2017 21:19:04 -0500 Subject: [PATCH 39/51] Don't trus latest anaconda for now, roll back to 4.4.0 --- jenkins-ci/jenkins.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-ci/jenkins.sh b/jenkins-ci/jenkins.sh index 234759f2..584d2a54 100644 --- a/jenkins-ci/jenkins.sh +++ b/jenkins-ci/jenkins.sh @@ -9,7 +9,7 @@ rm /tigress/alexeys/model_checkpoints/* rm -rf /tigress/alexeys/processed_shots rm -rf /tigress/alexeys/processed_shotlists rm -rf /tigress/alexeys/normalization -module load anaconda3 +module load anaconda3/4.4.0 source activate PPPL_dev3 module load cudatoolkit/8.0 module load cudnn/cuda-8.0/6.0 @@ -36,7 +36,7 @@ rm /tigress/alexeys/model_checkpoints/* source deactivate module purge -module load anaconda +module load anaconda/4.4.0 source activate PPPL module load cudatoolkit/8.0 module load cudnn/cuda-8.0/6.0 From 1f92ec0bf8b0743cc77ad7d698214fa0f3a2f202 Mon Sep 17 00:00:00 2001 From: Alexey Svyatkovskiy Date: Thu, 28 Dec 2017 21:43:17 -0500 Subject: [PATCH 40/51] Trigger jenkins build --- jenkins-ci/jenkins.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/jenkins-ci/jenkins.sh b/jenkins-ci/jenkins.sh index 584d2a54..9312a06e 100644 --- a/jenkins-ci/jenkins.sh +++ b/jenkins-ci/jenkins.sh @@ -21,7 +21,6 @@ cd /home/alexeys/jenkins/workspace/FRNM/PPPL echo ${PWD} python setup.py install -echo ${PWD} echo $SLURM_NODELIST cd examples echo ${PWD} From d3beb0956eb95401e796dd401e804414c12c0d6a Mon Sep 17 00:00:00 2001 From: Alexey Svyatkovskiy Date: Fri, 29 Dec 2017 09:41:23 -0500 Subject: [PATCH 41/51] Trigger jenkins test --- jenkins-ci/jenkins.sh | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/jenkins-ci/jenkins.sh b/jenkins-ci/jenkins.sh index 9312a06e..f0f4a8f3 100644 --- a/jenkins-ci/jenkins.sh +++ b/jenkins-ci/jenkins.sh @@ -31,21 +31,21 @@ sed -i -e 's/data: jet_data/data: jenkins_jet/g' conf.yaml srun python mpi_learn.py echo "Jenkins test Python2.7" -rm /tigress/alexeys/model_checkpoints/* - -source deactivate -module purge -module load anaconda/4.4.0 -source activate PPPL -module load cudatoolkit/8.0 -module load cudnn/cuda-8.0/6.0 -module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 -module load intel/17.0/64/17.0.4.196 - -cd .. -python setup.py install - -echo $SLURM_NODELIST -cd examples -sed -i -e 's/data: jenkins_jet/data: jenkins_d3d/g' conf.yaml -srun python mpi_learn.py +#rm /tigress/alexeys/model_checkpoints/* + +#source deactivate +#module purge +#module load anaconda/4.4.0 +#source activate PPPL +#module load cudatoolkit/8.0 +#module load cudnn/cuda-8.0/6.0 +#module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 +#module load intel/17.0/64/17.0.4.196 + +#cd .. +#python setup.py install + +#echo $SLURM_NODELIST +#cd examples +#sed -i -e 's/data: jenkins_jet/data: jenkins_d3d/g' conf.yaml +#srun python mpi_learn.py From 90ba457bca1a26541684b6bbc88537bfdb033535 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Fri, 29 Dec 2017 09:59:36 -0500 Subject: [PATCH 42/51] Obsolete --- jenkins-ci/jenkins.py | 52 ------------------------------------------- 1 file changed, 52 deletions(-) delete mode 100644 jenkins-ci/jenkins.py diff --git a/jenkins-ci/jenkins.py b/jenkins-ci/jenkins.py deleted file mode 100644 index 38d6a4fe..00000000 --- a/jenkins-ci/jenkins.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -from subprocess import Popen - -Popen("export OMPI_MCA_btl=\"tcp,self,sm\"",shell=True) - -Popen("echo \"Jenkins test Python3.6\"",shell=True) -Popen("rm /tigress/alexeys/model_checkpoints/*",shell=True) -Popen("rm -rf /tigress/alexeys/processed_shots",shell=True) -Popen("rm -rf /tigress/alexeys/processed_shotlists",shell=True) -Popen("rm -rf /tigress/alexeys/normalization",shell=True) -Popen("module load anaconda3",shell=True) -Popen("source activate PPPL_dev3",shell=True) -Popen("module load cudatoolkit/8.0",shell=True) -Popen("module load cudnn/cuda-8.0/6.0",shell=True) -Popen("module load openmpi/cuda-8.0/intel-17.0/2.1.0/64",shell=True) -Popen("module load intel/17.0/64/17.0.4.196",shell=True) - -Popen("cd /home/alexeys/jenkins/workspace/FRNM/PPPL",shell=True) -Popen("python setup.py install",shell=True) - -Popen("echo $SLURM_NODELIST",shell=True) -Popen("cd examples",shell=True) - -Popen("sed -i -e 's/num_epochs: 1000/num_epochs: 2/g' conf.yaml",shell=True) -Popen("sed -i -e 's/data: jet_data/data: jenkins_jet/g' conf.yaml",shell=True) - -Popen("srun python mpi_learn.py",shell=True).wait() - -Popen("echo \"Jenkins test Python2.7\"",shell=True) -Popen("rm /tigress/alexeys/model_checkpoints/*",shell=True) -Popen("rm -rf /tigress/alexeys/processed_shots",shell=True) -Popen("rm -rf /tigress/alexeys/processed_shotlists",shell=True) -Popen("rm -rf /tigress/alexeys/normalization",shell=True) - -Popen("source deactivate",shell=True) -Popen("module purge",shell=True) -Popen("module load anaconda",shell=True) -Popen("source activate PPPL",shell=True) -Popen("module load cudatoolkit/8.0",shell=True) -Popen("module load cudnn/cuda-8.0/6.0",shell=True) -Popen("module load openmpi/cuda-8.0/intel-17.0/2.1.0/64",shell=True) -Popen("module load intel/17.0/64/17.0.4.196",shell=True) - -Popen("cd ..",shell=True) -Popen("python setup.py install",shell=True) - -Popen("echo $SLURM_NODELIST",shell=True) -Popen("cd examples",shell=True) - -Popen("sed -i -e 's/data: jenkins_jet/data: jenkins_d3d/g' conf.yaml",shell=True) - -Popen("srun python mpi_learn.py",shell=True).wait() From ea0b6b295353d005e04e67b3834434fb047c7927 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Fri, 29 Dec 2017 10:05:58 -0500 Subject: [PATCH 43/51] Validate jenkins --- jenkins-ci/validate_jenkins.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 jenkins-ci/validate_jenkins.py diff --git a/jenkins-ci/validate_jenkins.py b/jenkins-ci/validate_jenkins.py new file mode 100644 index 00000000..6014fb2b --- /dev/null +++ b/jenkins-ci/validate_jenkins.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python + +import mpi4py as mmm +print(mmm.__version__) + +import keras as kk +print(kk.__version__) + +import tensorflow as tf +print(tf.__version__) + +from mpi4py import MPI +import sys + +size = MPI.COMM_WORLD.Get_size() +rank = MPI.COMM_WORLD.Get_rank() +name = MPI.Get_processor_name() + +sys.stdout.write( + "Hello, World! I am process %d of %d on %s.\n" + % (rank, size, name)) From a2cd354fd7fbace3b7eb9fa57466ad5c2bf0c40e Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Fri, 29 Dec 2017 10:06:18 -0500 Subject: [PATCH 44/51] Validate jenkins --- jenkins-ci/validate_jenkins.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 jenkins-ci/validate_jenkins.sh diff --git a/jenkins-ci/validate_jenkins.sh b/jenkins-ci/validate_jenkins.sh new file mode 100644 index 00000000..1d925ad2 --- /dev/null +++ b/jenkins-ci/validate_jenkins.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +export OMPI_MCA_btl="tcp,self,sm" + +module load anaconda3 +source activate PPPL_dev3 +module load cudatoolkit/8.0 +module load cudnn/cuda-8.0/6.0 +module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 +module load intel/17.0/64/17.0.4.196 + +cd /home/alexeys/jenkins/workspace/FRNM/PPPL +python setup.py install + +echo `which python` +echo `which mpicc` + +echo ${PWD} +echo $SLURM_NODELIST + +cd examples +echo ${PWD} +ls ${PWD} + +srun python validate_jenkins.py From 47df24a26e528fff2ec0640577b0b5531d53673a Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Fri, 29 Dec 2017 10:14:56 -0500 Subject: [PATCH 45/51] Create new anaconda env every test --- jenkins-ci/validate_jenkins.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jenkins-ci/validate_jenkins.sh b/jenkins-ci/validate_jenkins.sh index 1d925ad2..37950a3f 100644 --- a/jenkins-ci/validate_jenkins.sh +++ b/jenkins-ci/validate_jenkins.sh @@ -2,8 +2,9 @@ export OMPI_MCA_btl="tcp,self,sm" -module load anaconda3 -source activate PPPL_dev3 +module load anaconda3/4.4.0 +conda create --yes --name jenkins3 --prefix /tigress/alexeys/jenkins --file requirements-travis.txt +source activate jenkins3 module load cudatoolkit/8.0 module load cudnn/cuda-8.0/6.0 module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 From 7768b6861bc9c2b5e1dc433f2285a29e7b3a68a0 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Fri, 29 Dec 2017 10:23:46 -0500 Subject: [PATCH 46/51] Can't create named conda envs with a specific desitnation --- jenkins-ci/validate_jenkins.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-ci/validate_jenkins.sh b/jenkins-ci/validate_jenkins.sh index 37950a3f..5d4ad87a 100644 --- a/jenkins-ci/validate_jenkins.sh +++ b/jenkins-ci/validate_jenkins.sh @@ -3,7 +3,7 @@ export OMPI_MCA_btl="tcp,self,sm" module load anaconda3/4.4.0 -conda create --yes --name jenkins3 --prefix /tigress/alexeys/jenkins --file requirements-travis.txt +conda create --yes --prefix /tigress/alexeys/jenkins/.conda/envs/jenkins3 --file requirements-travis.txt source activate jenkins3 module load cudatoolkit/8.0 module load cudnn/cuda-8.0/6.0 From 0edcd7240cf2d62769b012d83e9377044c28b76f Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Fri, 29 Dec 2017 10:33:19 -0500 Subject: [PATCH 47/51] Install path is the conda env name --- jenkins-ci/validate_jenkins.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jenkins-ci/validate_jenkins.sh b/jenkins-ci/validate_jenkins.sh index 5d4ad87a..4f68e122 100644 --- a/jenkins-ci/validate_jenkins.sh +++ b/jenkins-ci/validate_jenkins.sh @@ -3,8 +3,7 @@ export OMPI_MCA_btl="tcp,self,sm" module load anaconda3/4.4.0 -conda create --yes --prefix /tigress/alexeys/jenkins/.conda/envs/jenkins3 --file requirements-travis.txt -source activate jenkins3 +source activate /tigress/alexeys/jenkins/.conda/envs/jenkins3 module load cudatoolkit/8.0 module load cudnn/cuda-8.0/6.0 module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 From 309e62cf132ce34436953f758702dcd7d2ab882c Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Fri, 29 Dec 2017 10:41:29 -0500 Subject: [PATCH 48/51] Correct path --- jenkins-ci/validate_jenkins.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-ci/validate_jenkins.sh b/jenkins-ci/validate_jenkins.sh index 4f68e122..f7d6b0d5 100644 --- a/jenkins-ci/validate_jenkins.sh +++ b/jenkins-ci/validate_jenkins.sh @@ -18,7 +18,7 @@ echo `which mpicc` echo ${PWD} echo $SLURM_NODELIST -cd examples +cd jenkins-ci echo ${PWD} ls ${PWD} From ea39f24677e7022ee3ab7b0e63befdce42e2aac0 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Fri, 29 Dec 2017 10:52:33 -0500 Subject: [PATCH 49/51] Adjust anaconda env --- jenkins-ci/jenkins.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-ci/jenkins.sh b/jenkins-ci/jenkins.sh index f0f4a8f3..1fa10ffe 100644 --- a/jenkins-ci/jenkins.sh +++ b/jenkins-ci/jenkins.sh @@ -10,7 +10,7 @@ rm -rf /tigress/alexeys/processed_shots rm -rf /tigress/alexeys/processed_shotlists rm -rf /tigress/alexeys/normalization module load anaconda3/4.4.0 -source activate PPPL_dev3 +source activate /tigress/alexeys/jenkins/.conda/envs/jenkins3 module load cudatoolkit/8.0 module load cudnn/cuda-8.0/6.0 module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 @@ -36,7 +36,7 @@ echo "Jenkins test Python2.7" #source deactivate #module purge #module load anaconda/4.4.0 -#source activate PPPL +#source activate /tigress/alexeys/jenkins/.conda/envs/jenkins2 #module load cudatoolkit/8.0 #module load cudnn/cuda-8.0/6.0 #module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 From 45014294abf3acbe846262c8932e83b6c4324b11 Mon Sep 17 00:00:00 2001 From: Alexey Svyatkovskiy Date: Fri, 29 Dec 2017 10:58:35 -0500 Subject: [PATCH 50/51] Trigger jenkins test From 9e36e137c6269e3b25d722905fe92c7c2feb42dd Mon Sep 17 00:00:00 2001 From: Alexey Svyatkovskiy Date: Fri, 29 Dec 2017 12:17:41 -0500 Subject: [PATCH 51/51] Exclude bad nodes, trigger jenkins build