merge

.github/workflows/helpers/test_libs.sh → .github/workflows/helpers/test_target.sh

@@ -7,7 +7,7 @@ DIR="$(realpath -- "$(dirname "${BASH_SOURCE[0]}")")"
 REPO="$(realpath -- "$DIR/../../../")"
 
 TEST_LIBS=("${@/%/-tests}")
-REGEX="^$(IFS='|'; echo "${TEST_LIBS[*]}")\$"
+REGEX="^($(IFS='|'; echo "${TEST_LIBS[*]}"))\$"
 
 cd "$REPO/build-ci"
 make -j $(( $(nproc) < 2 ? 1 : $(nproc)-1 )) "${TEST_LIBS[@]}"

.github/workflows/per-lib-check.yml

@@ -23,6 +23,9 @@ jobs:
       - name: Add helpers directory to path
         run: echo "${PWD}/.github/workflows/helpers" >> $GITHUB_PATH
 
+      - name: Free additional space on runner
+        run: free_space_on_runner.sh
+
       - name: Install nix
         uses: cachix/install-nix-action@v25
         with:
@@ -62,59 +65,79 @@ jobs:
 
       - name: Build utils
         run: |
-          build_libs.sh utils
+          build_target.sh utils
 
       - name: Build op-attrs
         run: |
-          build_libs.sh op-attrs
+          build_target.sh op-attrs
 
       - name: Build pcg
         run: |
-          build_libs.sh pcg
+          build_target.sh pcg
 
       - name: Build kernels
         run: |
-          build_libs.sh kernels
+          build_target.sh kernels
 
       - name: Build substitutions
         run: |
-          build_libs.sh substitutions
+          build_target.sh substitutions
 
       - name: Build compiler
         run: |
-          build_libs.sh compiler
+          build_target.sh compiler
 
       - name: Build substitution-generator
         run: |
-          build_libs.sh substitution-generator
+          build_target.sh substitution-generator
 
       - name: Build local-execution
         run: |
-          build_libs.sh local-execution
+          build_target.sh local-execution
+
+      - name: Build models
+        run: |
+          build_target.sh models
+
+      - name: Build substitution-to-dot
+        run: |
+          build_target.sh substitution-to-dot
+
+      - name: Build export-model-arch
+        run: |
+          build_target.sh export-model-arch
 
       - name: Test utils
         run: |
-          test_libs.sh utils
+          test_target.sh utils
 
       - name: Test op-attrs
         run: |
-          test_libs.sh op-attrs
+          test_target.sh op-attrs
 
       - name: Test pcg
         run: |
-          test_libs.sh pcg
+          test_target.sh pcg
 
       - name: Test substitutions
         run: |
-          test_libs.sh substitutions
+          test_target.sh substitutions
 
-      # - name: Test compiler
-      #   run: |
-      #     test_libs.sh compiler
+      - name: Test compiler
+        run: |
+          test_target.sh compiler
 
       - name: Test substitution-generator
         run: |
-          test_libs.sh substitution-generator
+          test_target.sh substitution-generator
+
+      - name: Test local-execution
+        run: |
+          test_target.sh local-execution
+
+      - name: Test models
+        run: |
+          test_target.sh models
 
       - name: Generate code coverage
         run: |

.proj.toml

@@ -11,16 +11,22 @@ build_targets = [
   "substitutions",
   "compiler",
   "substitution-generator",
-  "local-execution", 
+  "local-execution",
+  "models",
+  "export-model-arch",
+  "substitution-to-dot",
 ]
 
 test_targets = [
+  # "kernels-tests",
   "utils-tests",
   "op-attrs-tests",
   "pcg-tests",
   "substitutions-tests",
   "compiler-tests",
   "substitution-generator-tests",
+  "local-execution-tests",
+  "models-tests",
 ]
 
 [cmake_flags_extra]

CMakeLists.txt

@@ -75,6 +75,7 @@ option(FF_BUILD_SUBSTITUTION_TOOL "build substitution conversion tool" OFF)
 option(FF_BUILD_VISUALIZATION_TOOL "build substitution visualization tool" ON)
 option(FF_BUILD_SP_IZATION_BENCHMARKING "build sp-ization benchmarking" ON)
 option(FF_BUILD_ARG_PARSER "build command line argument parser" OFF)
+option(FF_BUILD_BIN_EXPORT_MODEL_ARCH "build export-model-arch utility" ON)
 
 set(FF_CUDA_ARCH "autodetect" CACHE STRING "Target CUDA Arch")
 if (FF_CUDA_ARCH STREQUAL "")

bin/CMakeLists.txt

@@ -7,7 +7,7 @@ if(FF_BUILD_SUBSTITUTION_TOOL)
 endif()
 
 if(FF_BUILD_VISUALIZATION_TOOL)
-  add_subdirectory(substitutions-to-dot)
+  add_subdirectory(substitution-to-dot)
 endif()
 
 if(FF_BUILD_SP_IZATION_BENCHMARKING)
@@ -17,3 +17,7 @@ endif()
 if(FF_BUILD_ARG_PARSER)
   add_subdirectory(arg_parser)
 endif()
+
+if(FF_BUILD_BIN_EXPORT_MODEL_ARCH)
+  add_subdirectory(export-model-arch)
+endif()

bin/export-model-arch/CMakeLists.txt

@@ -0,0 +1,12 @@
+ff_add_executable(
+  NAME
+    export-model-arch
+  SRC_PATTERNS
+    src/*.cc
+  PRIVATE_INCLUDE
+    include/
+  DEPS
+    utils
+    models
+    compiler
+)

bin/export-model-arch/include/export_model_arch/json_sp_model_export.struct.toml

@@ -0,0 +1,25 @@
+namespace = "FlexFlow"
+name = "JsonSPModelExport"
+features = [
+  "eq",
+  "hash",
+  "fmt",
+  "json", 
+]
+
+includes = [
+  "pcg/file_format/v1/v1_computation_graph.dtg.h",
+  "pcg/file_format/v1/v1_binary_sp_decomposition/v1_binary_sp_decomposition.dtg.h",
+]
+
+src_includes = [
+  "pcg/file_format/v1/v1_binary_sp_decomposition/json.h",
+]
+
+[[fields]]
+name = "sp_decomposition"
+type = "::FlexFlow::V1BinarySPDecomposition"
+
+[[fields]]
+name = "computation_graph"
+type = "::FlexFlow::V1ComputationGraph"