Merge pull request #26 from PPPLDeepLearning/jenkins_test

Jenkins test + Python3 and 1D Profile Fixes

examples/conf.yaml

@@ -10,7 +10,7 @@ paths:
     signal_prepath: '/signal_data/' #/signal_data/jet/
     shot_list_dir: '/shot_lists/'
     tensorboard_save_path: '/Graph/'
-    data: 'jet_data' #'d3d_to_jet_data' #'d3d_to_jet_data' # 'jet_to_d3d_data' #jet_data
+    data: jet_data #'d3d_to_jet_data' #'d3d_to_jet_data' # 'jet_to_d3d_data' #jet_data
     specific_signals: [] #['q95','li','ip','betan','energy','lm','pradcore','pradedge','pradtot','pin','torquein','tmamp1','tmamp2','tmfreq1','tmfreq2','pechin','energydt','ipdirect','etemp_profile','edens_profile'] #if left empty will use all valid signals defined on a machine. Only use if need a custom set
     executable: "mpi_learn.py"
     shallow_executable: "learn.py"

examples/guarantee_preprocessed.py

@@ -11,8 +11,6 @@
 pprint(conf)
 from plasma.preprocessor.preprocess import guarantee_preprocessed
 
-os.environ["PYTHONHASHSEED"] = "0"
-
 #####################################################
 ####################PREPROCESSING####################
 #####################################################

examples/mpi_learn.py

@@ -24,8 +24,6 @@
 import random
 import numpy as np
 
-os.environ["PYTHONHASHSEED"] = "0"
-
 import matplotlib
 matplotlib.use('Agg')
 

examples/signal_influence.py

@@ -26,8 +26,6 @@
 import copy
 from functools import partial
 
-os.environ["PYTHONHASHSEED"] = "0"
-
 import matplotlib
 matplotlib.use('Agg')
 

examples/simple_augmentation.py

@@ -26,8 +26,6 @@
 import copy
 from functools import partial
 
-os.environ["PYTHONHASHSEED"] = "0"
-
 import matplotlib
 matplotlib.use('Agg')
 

jenkins-ci/jenkins.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+export OMPI_MCA_btl="tcp,self,sm"
+
+echo ${PWD}
+
+echo "Jenkins test Python3.6"
+rm /tigress/alexeys/model_checkpoints/*
+rm -rf /tigress/alexeys/processed_shots
+rm -rf /tigress/alexeys/processed_shotlists
+rm -rf /tigress/alexeys/normalization
+module load anaconda3/4.4.0
+source activate /tigress/alexeys/jenkins/.conda/envs/jenkins3
+module load cudatoolkit/8.0
+module load cudnn/cuda-8.0/6.0
+module load openmpi/cuda-8.0/intel-17.0/2.1.0/64
+module load intel/17.0/64/17.0.4.196
+
+echo ${PWD}
+cd /home/alexeys/jenkins/workspace/FRNM/PPPL
+echo ${PWD}
+python setup.py install
+
+echo $SLURM_NODELIST
+cd examples
+echo ${PWD}
+ls ${PWD}
+sed -i -e 's/num_epochs: 1000/num_epochs: 2/g' conf.yaml
+sed -i -e 's/data: jet_data/data: jenkins_jet/g' conf.yaml
+
+srun python mpi_learn.py
+
+echo "Jenkins test Python2.7"
+#rm /tigress/alexeys/model_checkpoints/*
+
+#source deactivate
+#module purge
+#module load anaconda/4.4.0
+#source activate /tigress/alexeys/jenkins/.conda/envs/jenkins2
+#module load cudatoolkit/8.0
+#module load cudnn/cuda-8.0/6.0
+#module load openmpi/cuda-8.0/intel-17.0/2.1.0/64
+#module load intel/17.0/64/17.0.4.196
+
+#cd ..
+#python setup.py install
+
+#echo $SLURM_NODELIST
+#cd examples
+#sed -i -e 's/data: jenkins_jet/data: jenkins_d3d/g' conf.yaml
+#srun python mpi_learn.py

jenkins-ci/run_jenkins.py

@@ -0,0 +1,56 @@
+from plasma.utils.batch_jobs import generate_working_dirname,copy_files_to_environment,start_jenkins_job
+import yaml
+import sys,os,getpass
+import plasma.conf
+
+num_nodes = 4 #Set in the Jenkins project area!!
+test_matrix = [("Python3","jet_data"),("Python2","jet_data")]
+
+run_directory = "{}/{}/jenkins/".format(plasma.conf.conf['fs_path'],getpass.getuser())
+template_path = os.environ['PWD']
+conf_name = "conf.yaml"
+executable_name = "mpi_learn.py"
+
+def generate_conf_file(test_configuration,template_path = "../",save_path = "./",conf_name="conf.yaml"):
+    assert(template_path != save_path)
+    with open(os.path.join(template_path,conf_name), 'r') as yaml_file:
+        conf = yaml.load(yaml_file)
+    conf['training']['num_epochs'] = 2
+    conf['paths']['data'] = test_configuration[1]
+    if test_configuration[1] == "Python3":
+        conf['env']['name'] = "PPPL_dev3"
+        conf['env']['type'] = "anaconda3"
+    else:
+        conf['env']['name'] = "PPPL"
+        conf['env']['type'] = "anaconda"
+
+    with open(os.path.join(save_path,conf_name), 'w') as outfile:
+        yaml.dump(conf, outfile, default_flow_style=False)
+    return conf
+
+working_directory = generate_working_dirname(run_directory)
+os.makedirs(working_directory)
+
+os.system(" ".join(["cp -p",os.path.join(template_path,conf_name),working_directory]))
+os.system(" ".join(["cp -p",os.path.join(template_path,executable_name),working_directory]))
+
+#os.chdir(working_directory)
+#print("Going into {}".format(working_directory))
+
+for ci in test_matrix:
+    subdir = working_directory + "/{}/".format(ci[0]) 
+    os.makedirs(subdir)
+    copy_files_to_environment(subdir)
+    print("Making modified conf")
+    conf = generate_conf_file(ci,working_directory,subdir,conf_name)
+    print("Starting job")
+    if ci[1] == "Python3":
+        env_name = "PPPL_dev3"
+        env_type = "anaconda3"
+    else:
+        env_name = "PPPL"
+        env_type = "anaconda"
+    start_jenkins_job(subdir,num_nodes,executable_name,ci,env_name,env_type)
+
+
+print("submitted jobs.")

jenkins-ci/validate_jenkins.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+
+import mpi4py as mmm
+print(mmm.__version__)
+
+import keras as kk
+print(kk.__version__)
+
+import tensorflow as tf
+print(tf.__version__)
+
+from mpi4py import MPI
+import sys
+
+size = MPI.COMM_WORLD.Get_size()
+rank = MPI.COMM_WORLD.Get_rank()
+name = MPI.Get_processor_name()
+
+sys.stdout.write(
+    "Hello, World! I am process %d of %d on %s.\n"
+    % (rank, size, name))

jenkins-ci/validate_jenkins.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+export OMPI_MCA_btl="tcp,self,sm"
+
+module load anaconda3/4.4.0
+source activate /tigress/alexeys/jenkins/.conda/envs/jenkins3
+module load cudatoolkit/8.0
+module load cudnn/cuda-8.0/6.0
+module load openmpi/cuda-8.0/intel-17.0/2.1.0/64
+module load intel/17.0/64/17.0.4.196
+
+cd /home/alexeys/jenkins/workspace/FRNM/PPPL
+python setup.py install
+
+echo `which python`
+echo `which mpicc`
+
+echo ${PWD}
+echo $SLURM_NODELIST
+
+cd jenkins-ci
+echo ${PWD}
+ls ${PWD}
+
+srun python validate_jenkins.py

plasma/conf_parser.py

@@ -6,6 +6,7 @@
 import uuid
 import yaml
 
+import hashlib
 
 def parameters(input_file):
     """Parse yaml file of configuration parameters."""
@@ -69,12 +70,17 @@ def parameters(input_file):
         #shot lists
         jet_carbon_wall = ShotListFiles(jet,params['paths']['shot_list_dir'],['CWall_clear.txt','CFC_unint.txt'],'jet carbon wall data')
         jet_iterlike_wall = ShotListFiles(jet,params['paths']['shot_list_dir'],['ILW_unint.txt','BeWall_clear.txt'],'jet iter like wall data')
+
+        jenkins_jet_carbon_wall = ShotListFiles(jet,params['paths']['shot_list_dir'],['jenkins_CWall_clear.txt','jenkins_CFC_unint.txt'],'Subset of jet carbon wall data for Jenkins tests')
+        jenkins_jet_iterlike_wall = ShotListFiles(jet,params['paths']['shot_list_dir'],['jenkins_ILW_unint.txt','jenkins_BeWall_clear.txt'],'Subset of jet iter like wall data for Jenkins tests')
+
         jet_full = ShotListFiles(jet,params['paths']['shot_list_dir'],['ILW_unint.txt','BeWall_clear.txt','CWall_clear.txt','CFC_unint.txt'],'jet full data')
 
         d3d_10000 = ShotListFiles(d3d,params['paths']['shot_list_dir'],['d3d_clear_10000.txt','d3d_disrupt_10000.txt'],'d3d data 10000 ND and D shots')
         d3d_1000 = ShotListFiles(d3d,params['paths']['shot_list_dir'],['d3d_clear_1000.txt','d3d_disrupt_1000.txt'],'d3d data 1000 ND and D shots')
         d3d_100 = ShotListFiles(d3d,params['paths']['shot_list_dir'],['d3d_clear_100.txt','d3d_disrupt_100.txt'],'d3d data 100 ND and D shots')
         d3d_full = ShotListFiles(d3d,params['paths']['shot_list_dir'],['d3d_clear_data_avail.txt','d3d_disrupt_data_avail.txt'],'d3d data since shot 125500')
+        d3d_jenkins = ShotListFiles(d3d,params['paths']['shot_list_dir'],['jenkins_d3d_clear.txt','jenkins_d3d_disrupt.txt'],'Subset of d3d data for Jenkins test')
         d3d_jb_full = ShotListFiles(d3d,params['paths']['shot_list_dir'],['shotlist_JaysonBarr_clear.txt','shotlist_JaysonBarr_disrupt.txt'],'d3d shots since 160000-170000')
 
         nstx_full = ShotListFiles(nstx,params['paths']['shot_list_dir'],['disrupt_nstx.txt'],'nstx shots (all are disruptive')
@@ -95,6 +101,10 @@ def parameters(input_file):
             params['paths']['shot_files'] = [jet_full]
             params['paths']['shot_files_test'] = []
             params['paths']['use_signals_dict'] = jet_signals
+        elif params['paths']['data'] == 'jenkins_jet':
+            params['paths']['shot_files'] = [jenkins_jet_carbon_wall]
+            params['paths']['shot_files_test'] = [jenkins_jet_iterlike_wall]
+            params['paths']['use_signals_dict'] = jet_signals
         elif params['paths']['data'] == 'd3d_data':
             params['paths']['shot_files'] = [d3d_full]
             params['paths']['shot_files_test'] = [] 
@@ -116,6 +126,12 @@ def parameters(input_file):
             params['paths']['shot_files'] = [d3d_full]
             params['paths']['shot_files_test'] = [] 
             params['paths']['use_signals_dict'] = d3d_signals
+        elif params['paths']['data'] == 'jenkins_d3d':
+            params['paths']['shot_files'] = [d3d_jenkins]
+            params['paths']['shot_files_test'] = []
+            params['paths']['use_signals_dict'] = {'q95':q95,'li':li,'ip':ip,'lm':lm,'betan':betan,'energy':energy,'dens':dens,'pradcore':pradcore,'pradedge':pradedge,'pin':pin,'torquein':torquein,'ipdirect':ipdirect,'iptarget':iptarget,'iperr':iperr,
+'etemp_profile':etemp_profile ,'edens_profile':edens_profile}
+
 
         #cross-machine
         elif params['paths']['data'] == 'jet_to_d3d_data':
@@ -167,7 +183,7 @@ def parameters(input_file):
     return params
 
 def get_unique_signal_hash(signals):
-    return hash(tuple(sorted(signals)))
+    return int(hashlib.md5(''.join(tuple(map(lambda x: x.description, sorted(signals)))).encode('utf-8')).hexdigest(),16)
 
 #make sure 1D signals come last! This is necessary for model builder.
 def sort_by_channels(list_of_signals):