Generate wiews from dd files in cmake (#80)

Add generates_gui module in CMake. Its purpose is to replace the files genereall.cmd

This cmake function generates_gui is redrirected to a portable python script. This script calls the genere
binary who produces *View.h/cpp starting from .dd files.

The .dd files may be used in multiple projects. The genere_all.cmd were usually called in the .dd directory.
generate_gui is called where the view .cpp/h files are used. In this way it exposes the dependency between
the target library and the dd file.

While the dd.log file can be considered as source file (the cmake projects depend on it
for the genere_views command), we prefer not to track them on git. Pro is
obviously that we don't clone/pull log files, cons is the sources and dd.log files may be
not synchronized and may cause some issue int the generate_views process.

.gitignore

@@ -5,8 +5,8 @@
 .vs/
 .vscode/
 
-# Log file using genere tool
-genere.log
+# Log file using genere tool and the cmake module genere-views
+*.dd.log
 
 # CMake User Preset
 CMakeUserPresets.json

CMakeLists.txt

@@ -43,6 +43,7 @@ option(MPI "Use MPI libraries (ON,OFF)" ON)
 option(TESTING "Build unit tests (ON,OFF)" OFF)
 option(BUILD_LEX_YACC "Re-generate parsing files with lex & yacc" OFF)
 option(BUILD_JARS "Re-generate norm.jar and khiops.jar" OFF)
+option(GENERATE_VIEWS "Generate views sources from dd files" OFF)
 option(C11 "Build whith C++11 libraries (ON,OFF)" ON)
 
 # Print the options
@@ -53,6 +54,7 @@ message("  TESTING=\"${TESTING}\"")
 message("  C11=\"${C11}\"")
 message("  BUILD_LEX_YACC=\"${BUILD_LEX_YACC}\"")
 message("  BUILD_JARS=\"${BUILD_JARS}\"")
+message("  GENERATE_VIEWS=\"${GENERATE_VIEWS}\"")
 message("\n")
 
 # Check if the os is a linux fedora-like distro (CentOS, Rocky etc...)
@@ -105,7 +107,7 @@ message(STATUS "Executables will be stored in ${CMAKE_BINARY_DIR}/bin/")
 message(STATUS "Libraries will be stored in ${CMAKE_BINARY_DIR}/lib/")
 
 # Set module path for the project
-set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/packaging")
+set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/packaging" ; "${CMAKE_CURRENT_SOURCE_DIR}/scripts")
 
 # Use "-fPIC" / "-fPIE" for all targets by default, including static libs
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
@@ -287,6 +289,9 @@ function(display_variables)
   endforeach()
 endfunction()
 
+# Load module to generate views from .dd files
+include(generate_gui)
+
 # Add targets for Norm Parallel and Learning modules
 add_subdirectory(src/Norm)
 add_subdirectory(src/Parallel)

CMakePresets.json

@@ -16,6 +16,7 @@
         "TESTING": "ON",
         "BUILD_JARS": "ON",
         "BUILD_LEX_YACC": "OFF",
+        "GENERATE_VIEWS": "OFF",
         "C11": "ON"
       },
       "binaryDir": "${sourceDir}/build/${presetName}"

scripts/generate_gui.cmake

@@ -0,0 +1,84 @@
+# Launch genere binary
+# cmake-format: off
+# The funtion api is almost the same as genere:
+#
+# Genere <ClassName> <ClassLabel> <AttributeFileName> <LogFile> [options]
+#   <ClassName>: base name for generated classes
+#   <ClassLabel>: label for generated classes
+#   <AttributeFileName>: name of the file (.dd) containing the attribute specifications
+#   <LogFile>: name of the log file, it must be included in the source files of the binary/library
+#
+# Options:
+#   -nomodel no generation of class <ClassName>
+#   -noarrayview no generation of class <ClassName>ArrayView
+#   -noview no generation of classes <ClassName>View and <ClassName>ArrayView
+#   -nousersection no generation of user sections
+#   -specificmodel <SpecificModelClassName> name of a specific model class, to use instead of ClassName
+#   -super <SuperClassName> name of the parent class
+#
+# Do nothing if GENERATE_VIEWS is set to OFF
+#
+# Example to use genere binary in CMakeLists.txt:
+#
+#   include(genere)
+#    generate_gui_add_view(FOO "Foo Example" Foo.dd Foo.dd.log -noarrayview)
+#   add_executable(basetest Foo.cpp) 
+#   if(GENERATE_VIEWS)
+#       target_sources(basetest PRIVATE Foo.dd.log) # Foo.dd.log is genereted by  generate_gui_add_view
+#   endif()
+#    generate_gui_add_view_add_dependency(basetest)
+#
+# The log file Foo.dd.log is generated by  generate_gui_add_view and must be added as a source of the target. 
+# Whithout it, generate_gui_add_view is never triggered.
+# 
+# Note for developpers: we cannot set the h/cpp view files as output of add_custom_command because they are inputs 
+# and outputs. They would be removed at each clean. So we use the logs as output of add_custom_command and as 
+# source files of the binary/library target. This way we have the desired mechanic: The views are generated 
+# only when GENERATE_VIEWS=ON and a .dd file was modified.
+#
+# cmake-format: on
+
+function(generate_gui_add_view ClassName ClassLabel AttributeFileName LogFile)
+
+  if(GENERATE_VIEWS)
+    find_package(Python REQUIRED)
+    add_custom_command(
+      OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/${LogFile}
+      COMMAND
+        ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/scripts/ generate_gui_add_view.py
+        ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/genere ${ARGN} ${ClassName} ${ClassLabel} ${AttributeFileName} ${LogFile}
+      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+      VERBATIM
+      DEPENDS ${AttributeFileName}
+      MAIN_DEPENDENCY ${AttributeFileName}
+      COMMENT "Generating ${AttributeFileName} views")
+  endif()
+endfunction()
+
+# Create dependency between the target parameter and the genere binary. This function should be used with
+# generate_gui_add_view to ensure that:
+#
+# - genere is built before the target
+# - the target is re-built if genere is modified
+#
+# Do nothing if GENERATE_VIEWS is set to OFF
+function(generate_gui_add_view_add_dependency target)
+  if(GENERATE_VIEWS)
+    add_dependencies(${target} genere)
+  endif()
+endfunction()
+
+# Helper function: it builds the list of the .dd.log files starting from the .dd file in the source directory. The list
+# is empty if GENERATE_VIEWS is set to OFF.
+function(generate_gui_add_view_get_log_list OutVariable)
+  if(GENERATE_VIEWS)
+    file(
+      GLOB ddlogfiles
+      RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+      ${CMAKE_CURRENT_SOURCE_DIR}/*.dd)
+    list(TRANSFORM ddlogfiles APPEND ".log")
+    set(${OutVariable}
+        ${ddlogfiles}
+        PARENT_SCOPE)
+  endif()
+endfunction()

scripts/generate_gui.py

@@ -0,0 +1,31 @@
+import sys
+import os
+from subprocess import Popen, PIPE
+
+
+# script called from  generate_gui_add_view.cmake
+# The argument is the command line to launch genere, for example:
+# /bin//genere -outputdir /khiops/src/Learning/KWLearningProblem KWAnalysisSpec Parameters \
+#  KWAnalysisSpec.dd KWAnalysisSpec.dd.log
+# The command line is executed as it is and the stdout is redirect to the log file (last element of the command line).
+# The content of the log is displayed to stdout
+# If the command fails, the log file is removed and it exits with an error
+def main():
+    args = sys.argv[1:]
+    log_path = args[-1]
+    with open(log_path, "w") as log_file:
+        process = Popen(args[:-1], stdout=log_file)
+        process.wait()
+        log_file.close()
+        with open(log_path, "r") as log_file:
+            for line in log_file:
+                print("genere: " + line.strip())
+        if process.returncode != 0:
+            os.remove(log_path)
+            exit(1)
+        else:
+            exit(0)
+
+
+if __name__ == "__main__":
+    main()

src/Learning/CMakeLists.txt

@@ -1,14 +1,11 @@
 add_subdirectory(DTForest)
 add_subdirectory(genum)
 add_subdirectory(khisto)
-
 add_subdirectory(KDDomainKnowledge)
 add_subdirectory(KhiopsNativeInterface)
 add_subdirectory(KIInterpretation)
-
 add_subdirectory(KMDRRuleLibrary)
 add_subdirectory(KNITransfer)
-
 add_subdirectory(KWData)
 add_subdirectory(KWDataPreparation)
 add_subdirectory(KWDataUtils)
@@ -21,6 +18,7 @@ add_subdirectory(MHHistograms)
 add_subdirectory(MODL)
 add_subdirectory(MODL_Coclustering)
 add_subdirectory(SNBPredictor)
+add_subdirectory(samples/sample3)
 
 # build norm.jar
 if(BUILD_JARS)

src/Learning/KDDomainKnowledge/CMakeLists.txt

@@ -1,7 +1,16 @@
 file(GLOB cppfiles ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp)
 add_library(KDDomainKnowledge STATIC "${cppfiles}")
 
+if(GENERATE_VIEWS)
+  target_sources(KDDomainKnowledge PRIVATE KDTextFeatureSpec.dd.log)
+endif()
+
 set_khiops_options(KDDomainKnowledge)
 
 target_include_directories(KDDomainKnowledge PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 target_link_libraries(KDDomainKnowledge PUBLIC KWDataPreparation)
+
+generate_gui_add_view(KDTextFeatureSpec "Text feature parameters" ../KWUserInterface/KDTextFeatureSpec.dd
+                      KDTextFeatureSpec.dd.log -noview)
+
+generate_gui_add_view_add_dependency(KDDomainKnowledge)

src/Learning/KWLearningProblem/CMakeLists.txt

@@ -4,7 +4,9 @@ if(NOT MSVC)
 endif(NOT MSVC)
 
 file(GLOB cppfiles ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp)
-add_library(KWLearningProblem STATIC "${cppfiles}")
+generate_gui_add_view_get_log_list(ddlogfiles)
+
+add_library(KWLearningProblem STATIC "${cppfiles}" "${ddlogfiles}")
 set_khiops_options(KWLearningProblem)
 
 target_include_directories(KWLearningProblem PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
@@ -18,3 +20,23 @@ target_link_libraries(
          KWUserInterface
          SNBPredictor
          MHHistograms)
+
+# cmake-format: off
+
+# Generate cpp files from dd file
+
+# On ne genere que la partie modele, la partie vue etant uniquement composee de deux sous-fiches
+ generate_gui_add_view(KWModelingSpec "Predictors" KWModelingSpec.dd KWModelingSpec.dd.log  -noview -noarrayview)
+
+# On ne genere la partie vue uniquement pour deux vues distinct sur le modele
+ generate_gui_add_view(KWModelingAdvancedSpec "Advanced predictor parameters" KWModelingAdvancedSpec.dd KWModelingAdvancedSpec.dd.log -nomodel -noarrayview -specificmodel KWModelingSpec)
+ generate_gui_add_view(KWModelingExpertSpec "Expert predictor parameters" KWModelingExpertSpec.dd KWModelingExpertSpec.dd.log -nomodel -noarrayview -specificmodel KWModelingSpec)
+ generate_gui_add_view(KWAnalysisSpec "Parameters" KWAnalysisSpec.dd KWAnalysisSpec.dd.log  -noarrayview)     
+ generate_gui_add_view(KWRecoderSpec "Recoders" KWRecoderSpec.dd KWRecoderSpec.dd.log  -noarrayview)
+
+# remarque: tous les champs de choix de noms de fichiers sont exclus de la vue (Invisible a true dans le fichier .dd)
+ generate_gui_add_view(KWAnalysisResults "Results" KWAnalysisResults.dd KWAnalysisResults.dd.log  -noarrayview)
+
+# Add dependency to ensure that genere will be built before the gui generation
+ generate_gui_add_view_add_dependency(KWLearningProblem)
+# cmake-format: on

src/Learning/KWLearningProblem/KWAnalysisSpec.dd

@@ -1,3 +1,3 @@
 Rank;Name;Type;Style;Label
 1;TargetAttributeName  ;ALString;           ;Target variable
-2;MainTargetModality   ;ALString;           ;Main target value
+2;MainTargetModality   ;ALString;           ;Main target value

src/Learning/KWModeling/CMakeLists.txt

@@ -1,6 +1,28 @@
 file(GLOB cppfiles ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp)
+generate_gui_add_view_get_log_list(ddlogfiles)
+
 add_library(KWModeling STATIC "${cppfiles}")
+
+if(GENERATE_VIEWS)
+  target_sources(KWModeling PRIVATE "${ddlogfiles}" KWRecodingSpec.dd.log)
+endif()
+
 set_khiops_options(KWModeling)
 
 target_include_directories(KWModeling PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 target_link_libraries(KWModeling PUBLIC KDDomainKnowledge KWDataPreparation)
+
+# Generate cpp files from dd file
+# cmake-format: off
+
+# La partie vue est geree dans la librairie KWUserInterface
+ generate_gui_add_view(KWTrainParameters "Train parameters" KWTrainParameters.dd KWTrainParameters.dd.log -noview)
+
+# La partie vue est geree dans la librairie KWUserInterface
+ generate_gui_add_view(KWSelectionParameters "Selection parameters" KWSelectionParameters.dd KWSelectionParameters.dd.log -noview)
+ generate_gui_add_view(KWRecodingSpec "Recoding parameters" ../KWUserInterface/KWRecodingSpec.dd KWRecodingSpec.dd.log -noview)
+
+# cmake-format: on
+
+# Add dependency to ensure that genere will be built before the gui generation
+generate_gui_add_view_add_dependency(KWModeling)

src/Learning/KWUserInterface/CMakeLists.txt

@@ -1,7 +1,50 @@
 file(GLOB cppfiles ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp)
+generate_gui_add_view_get_log_list(ddlogfiles)
+
 add_library(KWUserInterface STATIC "${cppfiles}")
+if(GENERATE_VIEWS)
+  target_sources(KWUserInterface PRIVATE "${ddlogfiles}" KWSelectionParameters.dd.log KWTrainParameters.dd.log)
+endif()
 set_khiops_options(KWUserInterface)
 
 target_include_directories(KWUserInterface PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
-
 target_link_libraries(KWUserInterface PUBLIC KDDomainKnowledge KWDataPreparation KWModeling MHHistograms)
+
+# Generate cpp files from dd file
+
+# Dans la plus cas, on ne genere que les classe de type vue, les classe de type modele etant geree par ailleur
+# cmake-format: off
+ generate_gui_add_view(KWDatabase "Database" KWDatabase.dd KWDatabase.dd.log -nomodel -noarrayview)
+ generate_gui_add_view(KWDatabaseSampling "Sampling" KWDatabaseSampling.dd KWDatabaseSampling.dd.log -nomodel -noarrayview -specificmodel KWDatabase)
+ generate_gui_add_view(KWDatabaseSelection "Selection" KWDatabaseSelection.dd KWDatabaseSelection.dd.log -nomodel -noarrayview -specificmodel KWDatabase)
+ generate_gui_add_view(KWSTDatabaseTextFileData "Data" KWSTDatabaseTextFileData.dd KWSTDatabaseTextFileData.dd.log -nomodel -noarrayview -specificmodel KWSTDatabaseTextFile)
+ generate_gui_add_view(KWMTDatabaseTextFileData "Data" KWMTDatabaseTextFileData.dd KWMTDatabaseTextFileData.dd.log -nomodel -noarrayview -specificmodel KWMTDatabaseTextFile)
+ generate_gui_add_view(KWMTDatabaseMapping "Multi-table mapping" KWMTDatabaseMapping.dd KWMTDatabaseMapping.dd.log -nomodel)
+ generate_gui_add_view(KWClassSpec "Dictionary" KWClassSpec.dd KWClassSpec.dd.log)
+ generate_gui_add_view(KWAttributeSpec "Variable" KWAttributeSpec.dd KWAttributeSpec.dd.log)
+ generate_gui_add_view(KWAttributeName "Variable" KWAttributeName.dd KWAttributeName.dd.log -nomodel)
+ generate_gui_add_view(KWAttributePairName "Variable pair" KWAttributePairName.dd KWAttributePairName.dd.log -nomodel)
+ generate_gui_add_view(KWEvaluatedPredictorSpec "Evaluated predictor" KWEvaluatedPredictorSpec.dd KWEvaluatedPredictorSpec.dd.log)
+ generate_gui_add_view(KWPreprocessingSpec "Preprocessing parameters" KWPreprocessingSpec.dd KWPreprocessingSpec.dd.log -nomodel -noarrayview)
+ generate_gui_add_view(KWPreprocessingAdvancedSpec "Unsupervised parameters" KWPreprocessingAdvancedSpec.dd KWPreprocessingAdvancedSpec.dd.log -nomodel -noarrayview -specificmodel KWPreprocessingSpec)
+ generate_gui_add_view(KWDataGridOptimizerParameters "Data Grid optimization" KWDataGridOptimizerParameters.dd KWDataGridOptimizerParameters.dd.log -nomodel -noarrayview)
+ generate_gui_add_view(KWBenchmarkSpec "Benchmark" KWBenchmarkSpec.dd KWBenchmarkSpec.dd.log -nomodel)
+ generate_gui_add_view(KWLearningBenchmark "Learning benchmark" KWLearningBenchmark.dd KWLearningBenchmark.dd.log -nomodel -noarrayview)
+ generate_gui_add_view(KWBenchmarkClassSpec "Benchmark dictionary" KWBenchmarkClassSpec.dd KWBenchmarkClassSpec.dd.log -nomodel -noarrayview)
+
+# La partie modele est geree dans la librairie KWModeling
+ generate_gui_add_view(KWRecodingSpec "Recoding parameters" KWRecodingSpec.dd KWRecodingSpec.dd.log -nomodel -noarrayview)
+ generate_gui_add_view(KWAttributePairsSpec "Variable pairs parameters" KWAttributePairsSpec.dd KWAttributePairsSpec.dd.log -nomodel -noarrayview)
+ generate_gui_add_view(KWAttributeConstructionSpec "Feature engineering parameters" KWAttributeConstructionSpec.dd KWAttributeConstructionSpec.dd.log -nomodel -noarrayview)
+ generate_gui_add_view(KWSelectionParameters "Selection parameters" ../KWModeling/KWSelectionParameters.dd KWSelectionParameters.dd.log  -nomodel -noarrayview)
+ generate_gui_add_view(KWTrainParameters "Train parameters" ../KWModeling/KWTrainParameters.dd KWTrainParameters.dd.log -nomodel -noarrayview)
+
+# La partie modele est geree dans la librairie KDDomainKnowledge
+ generate_gui_add_view(KDTextFeatureSpec "Text feature parameters" KDTextFeatureSpec.dd KDTextFeatureSpec.dd.log -nomodel -noarrayview)
+ generate_gui_add_view(KDConstructionDomain "Variable construction parameters" KDConstructionDomain.dd KDConstructionDomain.dd.log -nomodel -noarrayview)
+ generate_gui_add_view(KDConstructionRule "Construction rule" KDConstructionRule.dd KDConstructionRule.dd.log -nomodel)
+
+# cmake-format: on
+
+# Add dependency to ensure that genere will be built before the gui generation
+generate_gui_add_view_add_dependency(KWUserInterface)