Re #705 Merge branch 'rel_3_5_3' into rel_3_5

# Conflicts:
#	horace_core/GUI/horace.m
#	horace_core/sqw/@sqw/private/binary_op_manager_single.m

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.7)
+cmake_minimum_required(VERSION 3.7.2)
 
 set(Horace_ROOT ${CMAKE_CURRENT_LIST_DIR})
 file(READ "${Horace_ROOT}/VERSION" _version)
@@ -28,24 +28,28 @@ set(Horace_DLL_DIRECTORY "${Horace_ROOT}/horace_core/DLL")
 # This is the directory that contains horace_init.m (used in admin/CMakeLists.txt)
 set(Horace_CORE "${CMAKE_CURRENT_LIST_DIR}/horace_core")
 
-# Set our options
-option(BUILD_TESTS "Build the C++ tests" ON)
-
 # Look for packages early so we can exit if they're not found
 find_package(Herbert REQUIRED)
 find_package(OpenMP)
 include(PACE_AddMex)
 include(PACE_FindMatlab)
 include(horace_FindHDF5)
 include(PACE_Version)
-if(${BUILD_TESTS})
-    include(CTest)
+
+# Make analysis ignore Herbert in case of nesting
+# Needs to have found herbert for Herbert_ROOT
+set(PACE_MLINT_IGNORE ${PACE_MLINT_IGNORE} "${Herbert_ROOT}/**/*.m")
+
+include(PACE_CodeAnalysis)
+include(CTest)
+if(BUILD_TESTING)
     include(PACE_FindGTest)
     enable_testing()
 endif()
+include(PACE_Docs)
 
 add_subdirectory("_LowLevelCode")
-if(${BUILD_TESTS})
+if(BUILD_TESTING)
     add_subdirectory("_test")
 endif()
 add_subdirectory("admin")
@@ -54,7 +58,6 @@ add_subdirectory("admin")
 # Install commands
 # =============================================================================
 include(PACE_CPackConfig)
-
 if(WIN32)
     # Don't package external Linux libraries on Windows
     set(EXTERNAL_IGNORE_PATTERN "*/external/glnxa64")
@@ -72,7 +75,7 @@ install(
     DESTINATION "."
 )
 install(
-    DIRECTORY "${Herbert_ROOT}/"
+    DIRECTORY "${Herbert_CORE}/"
     DESTINATION "Herbert"
     USE_SOURCE_PERMISSIONS
     PATTERN "*.m~" EXCLUDE

_LowLevelCode/cpp/CMakeLists.txt

@@ -17,6 +17,6 @@ foreach(_mex_func ${MEX_FUNCTIONS})
     add_subdirectory("${_mex_func}")
 endforeach(_mex_func)
 
-if(${BUILD_TESTS})
+if(BUILD_TESTING)
     add_subdirectory(test)
 endif()

_LowLevelCode/cpp/sort_pixels_by_bins/CMakeLists.txt

@@ -14,6 +14,7 @@ pace_add_mex(
     NAME "${MEX_NAME}"
     SRC "${SRC_FILES}" "${HDR_FILES}"
 )
+# openmp is currently disabled here but it may be used in a future
 #if(${OPENMP_FOUND})
 #    target_compile_options("${MEX_NAME}" PRIVATE ${OpenMP_CXX_FLAGS})
 #    target_link_options("${MEX_NAME}" PRIVATE ${OpenMP_EXE_LINKER_FLAGS})

_test/performance_test/.gitignore

@@ -1,4 +1,5 @@
 *.nxspe
 *.csv
+*.tmp
 
 

_test/performance_test/profile_machine.m

@@ -10,7 +10,11 @@
     force_perf = false;
 end
 
-hor_tes = test_SQW_GENCUT_perf();
+%hor_tes = test_SQW_GENCUT_perf(); % build new performance results per
+%                                  session
+hor_tes = test_SQW_GENCUT_perf(... % Load previous performance result, do not recalculate
+    fullfile(fileparts(mfilename('fullpath')),'test_SQW_GENCUT_perf_PerfRez.xml'));
+%
 hpcc = hpc_config;
 conf_2store = hpcc.get_data_to_store;
 clob = onCleanup(@()set(hpcc,conf_2store));
@@ -22,20 +26,20 @@
 hrc.delete_tmp = false;
 clob1 = onCleanup(@()set(hrc,'delete_tmp',true));
 
-% get the method used to combine partial sqw files together. Used in
-% calculating test performance name
-comb_method = hor_tes.combine_method();
-
-hor_tes.n_files_to_use=50;
 
-n_workers = [0,1,2,4,6,8,10,12,14,16];
-perf_graph = zeros(numel(n_workers),2);
+hor_tes.n_files_to_use=250;
+%n_workers = [0,1,2,4,6,8,10,12,14,16,20,32];
+n_workers = [0,1,2,4,8,12,14,16]; % local machine
+perf_graph = zeros(numel(n_workers),3);
 
 for i=1:numel(n_workers)
     nwk = num2str(n_workers(i));
-    test_name = sprintf('gen_sqw_nwk%s_comb_%s',nwk,comb_method);
-    per = hor_tes.knownPerformance(test_name);
-    if isempty(per) || force_perf
+    hor_tes.build_default_test_names(nwk);
+    test_names_map = hor_tes.default_test_names;
+    tn = test_names_map('gen_sqw');
+    per1 = hor_tes.known_performance(tn{1});
+    per2 = hor_tes.known_performance(tn{2});
+    if isempty(per1) || isempty(per2) || force_perf
         try
             perf_rez = hor_tes.test_gensqw_performance(n_workers(i),'gen_sqw');
         catch ME
@@ -44,15 +48,40 @@
             getReport(ME)
             rethrow(ME);
         end
-        per = perf_rez.(test_name);
+        per1 = perf_rez.(tn{1});
+        per2 = perf_rez.(tn{2});
     end
 
     perf_graph(i,1) = n_workers(i);
-    perf_graph(i,2) = per.time_sec;
+    perf_graph(i,2) = per1.time_sec/hor_tes.data_size;
+    perf_graph(i,3) = per2.time_sec/hor_tes.data_size;
 
 end
+% Process some averages to display
+min_gen_time = min(perf_graph(:,2));
+max_gen_time = max(perf_graph(:,2));
+min_comb_time = min(perf_graph(:,3));
+max_comb_time = max(perf_graph(:,3));
+
+min_prod_time = round((min_gen_time+min_comb_time)*hor_tes.data_size/60,1); % in minutes
+max_prod_time = round((max_gen_time+max_comb_time)*hor_tes.data_size/60,1); % in minutes
+tc1 = strrep(tn{1},'_','\_');
+tc2 = strrep(tn{2},'_','\_');
+title_string = sprintf(['Dataset silze~ %dGb, %d input files;\n',...
+    ' Final DB test codes:\n %s; %s\n',...
+    'Production time: min=%.1f(min); max=%.1f(min)'],...
+    round(hor_tes.data_size),hor_tes.n_files_to_use,...
+    tc1,tc2,min_prod_time,max_prod_time );
+
+%plot results
 figure;
 plot(perf_graph(:,1),perf_graph(:,2),'o-');
+ylabel('Processing Time (sec/Gb)')
+xlabel('n-workers');
+title(title_string)
+hold on
+plot(perf_graph(:,1),perf_graph(:,3),'*-');
+legend('gen\_tmp perf','combine perf')
 
 buf_val = [-1,0,1024,2048,4*1024,8*1024,16*1024,32*1024,64*1024];
 comb_perf = zeros(numel(buf_val),2);
@@ -95,7 +124,7 @@
     end
     comb_perf(i,1) = buf;
     comb_perf(i,2) = per.time_sec;
-        
+
 end
 figure
 plot(comb_perf(:,1),comb_perf(:,2),'o-');