Updating to newer python (#1933)

* Trying with tensorflow 2 float.

* list size varies between python versions.

* Testing removing cbcpy.

* Regolding tests/framework/Samplers/AdaptiveBatch/ThickenedBand

* Regolding tensorflow tests.

* Trying specifying tensorflow version.

* Use 2.4 on mac and 2.3 on linux and windows.

* Adding min_python_version check in rook and using for test

* Fix pylint problems.

* Mac needs a different version of tensorflow.

* Switching to different gold file.

* If gold file not found, print out expected filename.

* Revert LOGOS change.

* Updating LOGOS and HERON submodules.

* Forcing version of protobuf because of ray incompatibility.

Co-authored-by: civet runner <[email protected]>

dependencies.xml

@@ -45,7 +45,8 @@ Note all install methods after "main" take
     <matplotlib>3.2</matplotlib>
     <statsmodels>0.12</statsmodels>
     <cloudpickle>1.6</cloudpickle>
-    <tensorflow>2.0</tensorflow>
+    <tensorflow os='windows,linux'>2.3</tensorflow>
+    <tensorflow os='mac'>2.7</tensorflow>
     <python skip_check='True'>3</python>
     <hdf5 skip_check='True'/>
     <swig skip_check='True'/>
@@ -74,6 +75,7 @@ Note all install methods after "main" take
     <ipopt skip_check='True' optional='True'/>
     <cyipopt optional='True'/>
     <pyomo-extensions source="pyomo" skip_check='True' optional='True'/>
+    <protobuf source="pip">3</protobuf> <!-- needed because protobuf 4 incompatible with current ray -->
     <setuptools/>
   </main>
   <alternate name="pip">

rook/OrderedCSVDiffer.py

@@ -137,7 +137,7 @@ def diff(self):
           gold_csv_file = open(gold_filename, 'rb')
       # if file doesn't exist, that's another problem
       except IOError:
-        msg.append('Gold file does not exist!')
+        msg.append('Gold file does not exist:'+gold_filename)
         same = False
       # if either file did not exist, clean up and go to next outfile
       if not same:

rook/Tester.py

@@ -385,6 +385,9 @@ def get_valid_params():
                      'if true, then the test should fails, and if it passes, it fails.')
     params.add_param('run_types', 'normal', 'The run types that this test is')
     params.add_param('output_wait_time', '-1', 'Number of seconds to wait for output')
+    params.add_param('min_python_version', 'none',
+                     'The Minimum python version required for this test.'+
+                     ' Example 3.8 (note, format is major.minor)')
     return params
 
   def __init__(self, _name, params):
@@ -530,6 +533,15 @@ def _run_backend(self, _):
       self.results.group = self.group_skip
       self.results.message = self.specs['skip']
       return self.results
+    if self.specs['min_python_version'].strip().lower() != 'none':
+      major, minor = self.specs['min_python_version'].strip().split(".")
+      #check to see if current version of python too old.
+      if int(major) > sys.version_info.major or \
+         (int(major) == sys.version_info.major and int(minor) > sys.version_info.minor):
+        self.results.group = self.group_skip
+        self.results.message = "skipped because need python version "\
+          +self.specs['min_python_version'].strip()+" but have "+sys.version
+        return self.results
     if not self.__test_run_type.issubset(self.__base_current_run_type):
       self.results.group = self.group_skip
       self.results.message = "SKIPPED ("+str(self.__test_run_type)+\

tests/framework/ROM/tensorflow_keras/gold/data/outCon1DClassifier.csv

@@ -23,14 +23,14 @@ time,x,y
 0.5625,8.0,11.0
 0.675,9.0,11.0
 0.7875,10.0,11.0
-0.0,5.0,14.0
-0.1125,6.0,14.0
-0.225,7.0,14.0
-0.3375,8.0,14.0
-0.45,9.0,14.0
-0.5625,10.0,14.0
-0.675,11.0,14.0
-0.7875,12.0,14.0
+0.0,5.0,13.0
+0.1125,6.0,13.0
+0.225,7.0,13.0
+0.3375,8.0,13.0
+0.45,9.0,13.0
+0.5625,10.0,13.0
+0.675,11.0,13.0
+0.7875,12.0,13.0
 0.0,7.0,15.0
 0.1125,8.0,15.0
 0.225,9.0,15.0
@@ -55,22 +55,22 @@ time,x,y
 0.5625,18.0,17.0
 0.675,19.0,17.0
 0.7875,20.0,17.0
-0.0,5.0,14.0
-0.1125,6.0,14.0
-0.225,7.0,14.0
-0.3375,8.0,14.0
-0.45,9.0,14.0
-0.5625,10.0,14.0
-0.675,11.0,14.0
-0.7875,12.0,14.0
-0.0,5.0,14.0
-0.1125,6.0,14.0
-0.225,7.0,14.0
-0.3375,8.0,14.0
-0.45,9.0,14.0
-0.5625,10.0,14.0
-0.675,11.0,14.0
-0.7875,12.0,14.0
+0.0,5.0,13.0
+0.1125,6.0,13.0
+0.225,7.0,13.0
+0.3375,8.0,13.0
+0.45,9.0,13.0
+0.5625,10.0,13.0
+0.675,11.0,13.0
+0.7875,12.0,13.0
+0.0,5.0,13.0
+0.1125,6.0,13.0
+0.225,7.0,13.0
+0.3375,8.0,13.0
+0.45,9.0,13.0
+0.5625,10.0,13.0
+0.675,11.0,13.0
+0.7875,12.0,13.0
 0.0,6.0,14.0
 0.1125,7.0,14.0
 0.225,8.0,14.0

tests/framework/ROM/tensorflow_keras/gold/data/outLSTMClassifier.csv

@@ -11,9 +11,9 @@ time,x,y
 0.0,6.0,3.0
 0.3,7.0,3.0
 0.6,8.0,3.0
-0.0,9.0,3.0
-0.3,10.0,3.0
-0.6,11.0,3.0
+0.0,9.0,12.0
+0.3,10.0,12.0
+0.6,11.0,12.0
 0.0,20.0,24.0
 0.3,21.0,24.0
 0.6,22.0,24.0

tests/framework/ROM/tensorflow_keras/gold/data/rom_samples.csv

@@ -1,31 +1,31 @@
-RAVEN_sample_ID,t,presequence,nextval,PointProbability,ProbabilityWeight,ProbabilityWeight-t,ProbabilityWeight-presequence,prefix
-0,0.0,2.0,6.0,1.0,1.0,1.0,1.0,1
-0,0.1,3.0,6.0,1.0,1.0,1.0,1.0,1
-0,0.2,4.0,6.0,1.0,1.0,1.0,1.0,1
-1,0.0,2.0,6.0,1.0,1.0,1.0,1.0,2
-1,0.1,3.0,6.0,1.0,1.0,1.0,1.0,2
-1,0.2,4.0,6.0,1.0,1.0,1.0,1.0,2
-2,0.0,0.0,6.0,1.0,1.0,1.0,1.0,3
-2,0.1,1.0,6.0,1.0,1.0,1.0,1.0,3
-2,0.2,2.0,6.0,1.0,1.0,1.0,1.0,3
-3,0.0,4.0,6.0,1.0,1.0,1.0,1.0,4
-3,0.1,5.0,6.0,1.0,1.0,1.0,1.0,4
-3,0.2,6.0,6.0,1.0,1.0,1.0,1.0,4
-4,0.0,9.0,10.0,1.0,1.0,1.0,1.0,5
-4,0.1,10.0,10.0,1.0,1.0,1.0,1.0,5
-4,0.2,11.0,10.0,1.0,1.0,1.0,1.0,5
-5,0.0,6.0,9.0,1.0,1.0,1.0,1.0,6
-5,0.1,7.0,9.0,1.0,1.0,1.0,1.0,6
-5,0.2,8.0,9.0,1.0,1.0,1.0,1.0,6
-6,0.0,8.0,10.0,1.0,1.0,1.0,1.0,7
-6,0.1,9.0,10.0,1.0,1.0,1.0,1.0,7
-6,0.2,10.0,10.0,1.0,1.0,1.0,1.0,7
-7,0.0,6.0,9.0,1.0,1.0,1.0,1.0,8
-7,0.1,7.0,9.0,1.0,1.0,1.0,1.0,8
-7,0.2,8.0,9.0,1.0,1.0,1.0,1.0,8
-8,0.0,8.0,10.0,1.0,1.0,1.0,1.0,9
-8,0.1,9.0,10.0,1.0,1.0,1.0,1.0,9
-8,0.2,10.0,10.0,1.0,1.0,1.0,1.0,9
-9,0.0,7.0,10.0,1.0,1.0,1.0,1.0,10
-9,0.1,8.0,10.0,1.0,1.0,1.0,1.0,10
-9,0.2,9.0,10.0,1.0,1.0,1.0,1.0,10
+RAVEN_sample_ID,t,presequence,nextval,ProbabilityWeight,prefix,ProbabilityWeight-t,ProbabilityWeight-presequence,PointProbability
+0,0.0,2.0,6.0,1.0,1,1.0,1.0,1.0
+0,0.1,3.0,6.0,1.0,1,1.0,1.0,1.0
+0,0.2,4.0,6.0,1.0,1,1.0,1.0,1.0
+1,0.0,2.0,6.0,1.0,2,1.0,1.0,1.0
+1,0.1,3.0,6.0,1.0,2,1.0,1.0,1.0
+1,0.2,4.0,6.0,1.0,2,1.0,1.0,1.0
+2,0.0,0.0,6.0,1.0,3,1.0,1.0,1.0
+2,0.1,1.0,6.0,1.0,3,1.0,1.0,1.0
+2,0.2,2.0,6.0,1.0,3,1.0,1.0,1.0
+3,0.0,4.0,6.0,1.0,4,1.0,1.0,1.0
+3,0.1,5.0,6.0,1.0,4,1.0,1.0,1.0
+3,0.2,6.0,6.0,1.0,4,1.0,1.0,1.0
+4,0.0,9.0,12.0,1.0,5,1.0,1.0,1.0
+4,0.1,10.0,12.0,1.0,5,1.0,1.0,1.0
+4,0.2,11.0,12.0,1.0,5,1.0,1.0,1.0
+5,0.0,6.0,9.0,1.0,6,1.0,1.0,1.0
+5,0.1,7.0,9.0,1.0,6,1.0,1.0,1.0
+5,0.2,8.0,9.0,1.0,6,1.0,1.0,1.0
+6,0.0,8.0,10.0,1.0,7,1.0,1.0,1.0
+6,0.1,9.0,10.0,1.0,7,1.0,1.0,1.0
+6,0.2,10.0,10.0,1.0,7,1.0,1.0,1.0
+7,0.0,6.0,9.0,1.0,8,1.0,1.0,1.0
+7,0.1,7.0,9.0,1.0,8,1.0,1.0,1.0
+7,0.2,8.0,9.0,1.0,8,1.0,1.0,1.0
+8,0.0,8.0,10.0,1.0,9,1.0,1.0,1.0
+8,0.1,9.0,10.0,1.0,9,1.0,1.0,1.0
+8,0.2,10.0,10.0,1.0,9,1.0,1.0,1.0
+9,0.0,7.0,10.0,1.0,10,1.0,1.0,1.0
+9,0.1,8.0,10.0,1.0,10,1.0,1.0,1.0
+9,0.2,9.0,10.0,1.0,10,1.0,1.0,1.0

tests/framework/ROM/tensorflow_keras/gold/keras_mlp_regression/outMLPClassifier.csv

@@ -1,41 +1,41 @@
 x1,x2,x3,x4,x5,x6,x7,x8,y
-2.0,175.0,88.0,0.0,0.0,22.9,0.326,22.0,-168.534967145
-2.0,92.0,52.0,0.0,0.0,30.1,0.141,22.0,-136.640403488
-3.0,130.0,78.0,23.0,79.0,28.4,0.323,34.0,624.101374858
-8.0,120.0,86.0,0.0,0.0,28.4,0.259,22.0,-250.096262333
-2.0,174.0,88.0,37.0,120.0,44.5,0.646,24.0,1417.05884179
-2.0,106.0,56.0,27.0,165.0,29.0,0.426,22.0,724.428347807
-2.0,105.0,75.0,0.0,0.0,23.3,0.56,53.0,-296.397438805
-4.0,95.0,60.0,32.0,0.0,35.4,0.284,28.0,1161.74621636
-0.0,126.0,86.0,27.0,120.0,27.4,0.515,21.0,684.765842826
-8.0,65.0,72.0,23.0,0.0,32.0,0.6,42.0,622.514089322
-2.0,99.0,60.0,17.0,160.0,36.6,0.453,21.0,398.730881462
-1.0,102.0,74.0,0.0,0.0,39.5,0.293,42.0,-92.226801636
-11.0,120.0,80.0,37.0,150.0,42.3,0.785,48.0,1217.16961815
-3.0,102.0,44.0,20.0,94.0,30.8,0.4,26.0,605.181799346
-1.0,109.0,58.0,18.0,116.0,28.5,0.219,22.0,422.199516942
-9.0,140.0,94.0,0.0,0.0,32.7,0.734,45.0,-175.692705443
-13.0,153.0,88.0,37.0,140.0,40.6,1.174,39.0,1270.77116024
-12.0,100.0,84.0,33.0,105.0,30.0,0.488,46.0,877.836640891
-1.0,147.0,94.0,41.0,0.0,49.3,0.358,27.0,1686.38979997
-1.0,81.0,74.0,41.0,57.0,46.3,1.096,32.0,1508.80888951
-3.0,187.0,70.0,22.0,200.0,36.4,0.408,36.0,720.232358272
-6.0,162.0,62.0,0.0,0.0,24.3,0.178,50.0,-89.3525953073
-4.0,136.0,70.0,0.0,0.0,31.2,1.182,22.0,-61.2443783666
-1.0,121.0,78.0,39.0,74.0,39.0,0.261,28.0,1378.82910041
-3.0,108.0,62.0,24.0,0.0,26.0,0.223,25.0,752.237118717
-0.0,181.0,88.0,44.0,510.0,43.3,0.222,26.0,1175.11924589
-8.0,154.0,78.0,32.0,0.0,32.4,0.443,45.0,1186.87858501
-1.0,128.0,88.0,39.0,110.0,36.5,1.057,37.0,1291.75851705
-7.0,137.0,90.0,41.0,0.0,32.0,0.391,39.0,1438.53006794
-0.0,123.0,72.0,0.0,0.0,36.3,0.258,52.0,-71.1431638126
-1.0,106.0,76.0,0.0,0.0,37.5,0.197,26.0,-112.775338427
-6.0,190.0,92.0,0.0,0.0,35.5,0.278,66.0,-12.5909536144
-2.0,88.0,58.0,26.0,16.0,28.4,0.766,22.0,825.54933236
-9.0,170.0,74.0,31.0,0.0,44.0,0.403,43.0,1355.87012305
-9.0,89.0,62.0,0.0,0.0,22.5,0.142,33.0,-308.15859374
-10.0,101.0,76.0,48.0,180.0,32.9,0.171,63.0,1435.49974544
-2.0,122.0,70.0,27.0,0.0,36.8,0.34,27.0,1014.91176073
-5.0,121.0,72.0,23.0,112.0,26.2,0.245,30.0,551.266019966
-1.0,126.0,60.0,0.0,0.0,30.1,0.349,47.0,-84.8083653316
-1.0,93.0,70.0,31.0,0.0,30.4,0.315,23.0,1015.51785117
+2.0,175.0,88.0,0.0,0.0,22.9,0.326,22.0,-142.136246037
+2.0,92.0,52.0,0.0,0.0,30.1,0.141,22.0,-154.470590718
+3.0,130.0,78.0,23.0,79.0,28.4,0.323,34.0,690.748683734
+8.0,120.0,86.0,0.0,0.0,28.4,0.259,22.0,-218.026603
+2.0,174.0,88.0,37.0,120.0,44.5,0.646,24.0,1363.19543439
+2.0,106.0,56.0,27.0,165.0,29.0,0.426,22.0,740.986486569
+2.0,105.0,75.0,0.0,0.0,23.3,0.56,53.0,-165.461151746
+4.0,95.0,60.0,32.0,0.0,35.4,0.284,28.0,1189.15115395
+0.0,126.0,86.0,27.0,120.0,27.4,0.515,21.0,703.508056956
+8.0,65.0,72.0,23.0,0.0,32.0,0.6,42.0,734.509038205
+2.0,99.0,60.0,17.0,160.0,36.6,0.453,21.0,353.611860204
+1.0,102.0,74.0,0.0,0.0,39.5,0.293,42.0,-103.870274984
+11.0,120.0,80.0,37.0,150.0,42.3,0.785,48.0,1310.15886769
+3.0,102.0,44.0,20.0,94.0,30.8,0.4,26.0,622.639704257
+1.0,109.0,58.0,18.0,116.0,28.5,0.219,22.0,425.52683197
+9.0,140.0,94.0,0.0,0.0,32.7,0.734,45.0,-86.1749330404
+13.0,153.0,88.0,37.0,140.0,40.6,1.174,39.0,1358.0678548
+12.0,100.0,84.0,33.0,105.0,30.0,0.488,46.0,1043.50311204
+1.0,147.0,94.0,41.0,0.0,49.3,0.358,27.0,1622.39297892
+1.0,81.0,74.0,41.0,57.0,46.3,1.096,32.0,1483.06861461
+3.0,187.0,70.0,22.0,200.0,36.4,0.408,36.0,728.739041223
+6.0,162.0,62.0,0.0,0.0,24.3,0.178,50.0,38.6655169531
+4.0,136.0,70.0,0.0,0.0,31.2,1.182,22.0,-67.9863400929
+1.0,121.0,78.0,39.0,74.0,39.0,0.261,28.0,1372.23270588
+3.0,108.0,62.0,24.0,0.0,26.0,0.223,25.0,809.216915703
+0.0,181.0,88.0,44.0,510.0,43.3,0.222,26.0,1103.6863925
+8.0,154.0,78.0,32.0,0.0,32.4,0.443,45.0,1310.39587373
+1.0,128.0,88.0,39.0,110.0,36.5,1.057,37.0,1333.97291722
+7.0,137.0,90.0,41.0,0.0,32.0,0.391,39.0,1554.99359493
+0.0,123.0,72.0,0.0,0.0,36.3,0.258,52.0,-38.9393941617
+1.0,106.0,76.0,0.0,0.0,37.5,0.197,26.0,-165.099892618
+6.0,190.0,92.0,0.0,0.0,35.5,0.278,66.0,102.821566984
+2.0,88.0,58.0,26.0,16.0,28.4,0.766,22.0,858.648338101
+9.0,170.0,74.0,31.0,0.0,44.0,0.403,43.0,1403.80578523
+9.0,89.0,62.0,0.0,0.0,22.5,0.142,33.0,-200.570817881
+10.0,101.0,76.0,48.0,180.0,32.9,0.171,63.0,1639.34768777
+2.0,122.0,70.0,27.0,0.0,36.8,0.34,27.0,1011.61853532
+5.0,121.0,72.0,23.0,112.0,26.2,0.245,30.0,626.971435837
+1.0,126.0,60.0,0.0,0.0,30.1,0.349,47.0,-27.0939771344
+1.0,93.0,70.0,31.0,0.0,30.4,0.315,23.0,1040.56945965

tests/framework/ROM/tensorflow_keras/gold/mac/data/outCon1DClassifier.csv

@@ -0,0 +1,81 @@
+time,x,y
+0.0,11.0,19.0
+0.1125,12.0,19.0
+0.225,13.0,19.0
+0.3375,14.0,19.0
+0.45,15.0,19.0
+0.5625,16.0,19.0
+0.675,17.0,19.0
+0.7875,18.0,19.0
+0.0,2.0,9.0
+0.1125,3.0,9.0
+0.225,4.0,9.0
+0.3375,5.0,9.0
+0.45,6.0,9.0
+0.5625,7.0,9.0
+0.675,8.0,9.0
+0.7875,9.0,9.0
+0.0,3.0,11.0
+0.1125,4.0,11.0
+0.225,5.0,11.0
+0.3375,6.0,11.0
+0.45,7.0,11.0
+0.5625,8.0,11.0
+0.675,9.0,11.0
+0.7875,10.0,11.0
+0.0,5.0,13.0
+0.1125,6.0,13.0
+0.225,7.0,13.0
+0.3375,8.0,13.0
+0.45,9.0,13.0
+0.5625,10.0,13.0
+0.675,11.0,13.0
+0.7875,12.0,13.0
+0.0,7.0,15.0
+0.1125,8.0,15.0
+0.225,9.0,15.0
+0.3375,10.0,15.0
+0.45,11.0,15.0
+0.5625,12.0,15.0
+0.675,13.0,15.0
+0.7875,14.0,15.0
+0.0,15.0,24.0
+0.1125,16.0,24.0
+0.225,17.0,24.0
+0.3375,18.0,24.0
+0.45,19.0,24.0
+0.5625,20.0,24.0
+0.675,21.0,24.0
+0.7875,22.0,24.0
+0.0,13.0,21.0
+0.1125,14.0,21.0
+0.225,15.0,21.0
+0.3375,16.0,21.0
+0.45,17.0,21.0
+0.5625,18.0,21.0
+0.675,19.0,21.0
+0.7875,20.0,21.0
+0.0,5.0,13.0
+0.1125,6.0,13.0
+0.225,7.0,13.0
+0.3375,8.0,13.0
+0.45,9.0,13.0
+0.5625,10.0,13.0
+0.675,11.0,13.0
+0.7875,12.0,13.0
+0.0,5.0,13.0
+0.1125,6.0,13.0
+0.225,7.0,13.0
+0.3375,8.0,13.0
+0.45,9.0,13.0
+0.5625,10.0,13.0
+0.675,11.0,13.0
+0.7875,12.0,13.0
+0.0,6.0,14.0
+0.1125,7.0,14.0
+0.225,8.0,14.0
+0.3375,9.0,14.0
+0.45,10.0,14.0
+0.5625,11.0,14.0
+0.675,12.0,14.0
+0.7875,13.0,14.0

tests/framework/ROM/tensorflow_keras/tests

@@ -2,9 +2,14 @@
   [./tf_cnn1d]
     type  = 'RavenFramework'
     input = 'cnn1d.xml'
-    csv = 'data/outCon1DClassifier.csv'
     required_libraries = 'tensorflow'
     max_time = 450
+    [./data]
+        type = OrderedCSV
+        output = 'data/outCon1DClassifier.csv'
+        #note, mac uses tensorflow 2.7 but others use 2.3
+        mac_gold = 'mac/data/outCon1DClassifier.csv'
+    [../]
   [../]
   [./tf_mlpc]
     type  = 'RavenFramework'