Merge branch 'repo-refactor' into bob-hip-refactor-st

.editorconfig

@@ -0,0 +1,22 @@
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+end_of_line = lf
+insert_final_newline = true
+
+[{CMakeLists.txt,*.cmake}]
+indent_style = space
+indent_size = 2
+
+[*.{cc,h,cu,cpp}]
+indent_style = space
+indent_size = 2
+
+[*.py]
+indent_style = space
+indent_size = 4
+
+[*.toml]
+indent_style = space
+indent_size = 2

.flake/pkgs/hpp2plantuml.nix

@@ -0,0 +1,11 @@
+{buildPythonPackage, fetchPypi}:
+
+buildPythonPackage rec {
+  pname = "hpp2plantuml";
+  version = "0.8.5";
+  format = "wheel";
+  src = fetchPypi {
+    inherit pname version format;
+    sha256 = "sha256-PfTJmBypI21AAK3sMojygQfrhnRqcMmVCW4dxGfDfQg=";
+  };
+}

.gitattributes

@@ -0,0 +1,2 @@
+*.dtg.cc linguist-generated=true
+*.dtg.h linguist-generated=true

.github/workflows/helpers/cmake_cuda.sh

@@ -23,6 +23,7 @@ IFS=" " read -r -a FLAGS <<< "$CMAKE_FLAGS"
         -DCMAKE_C_COMPILER_LAUNCHER=ccache \
         -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
         -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \
+        -DFF_USE_CODE_COVERAGE=ON \
         "${FLAGS[@]}"
 
 # vim: set tabstop=2 shiftwidth=2 expandtab:

.github/workflows/per-lib-check.yml

@@ -72,22 +72,56 @@ jobs:
         run: |
           build_libs.sh kernels
 
-      - name: Build substitutions
-        run: |
-          build_libs.sh substitutions
+      # - name: Build substitutions
+      #   run: |
+      #     build_libs.sh substitutions
+
+      # - name: Build compiler
+      #   run: |
+      #     build_libs.sh compiler
 
-      - name: Build compiler
+      - name: Build substitution-generator
         run: |
-          build_libs.sh compiler
+          build_libs.sh substitution-generator
 
       - name: Test utils
         run: |
           test_libs.sh utils
 
-      - name: Test substitutions
+      - name: Test op-attrs
         run: |
-          test_libs.sh substitutions
+          test_libs.sh op-attrs
 
-      - name: Test compiler
+      - name: Test pcg
         run: |
-          test_libs.sh compiler
+          test_libs.sh pcg
+
+      # - name: Test substitutions
+      #   run: |
+      #     test_libs.sh substitutions
+
+      # - name: Test compiler
+      #   run: |
+      #     test_libs.sh compiler
+
+      - name: Test substitution-generator
+        run: |
+          test_libs.sh substitution-generator
+
+      - name: Generate code coverage
+        run: |
+          echo "gitwork: $GITHUB_WORKSPACE"
+          lcov --capture --directory . --output-file main_coverage.info
+          lcov --extract main_coverage.info "$GITHUB_WORKSPACE/lib/*" --output-file main_coverage.info
+          lcov --remove main_coverage.info "$GITHUB_WORKSPACE/lib/*.dtg.h" "$GITHUB_WORKSPACE/lib/*.dtg.cc"  --output-file main_coverage.info
+          lcov --list main_coverage.info
+      
+      - name: Upload code coverage
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: main_coverage.info
+          flags: unittests
+          name: codecov-umbrella
+          fail_ci_if_error: false
+          verbose: true

.proj.toml

@@ -7,13 +7,18 @@ build_targets = [
   "utils",
   "op-attrs",
   "kernels",
-  "substitutions",
-  "compiler",
+  "pcg",
+  # "substitutions",
+  # "compiler",
+  "substitution-generator",
 ]
 test_targets = [
   "utils-tests",
-  "substitutions-tests",
-  "compiler-tests",
+  "op-attrs-tests",
+  "pcg-tests",
+  # "substitutions-tests",
+  # "compiler-tests",
+  "substitution-generator-tests",
 ]
 
 [cmake_flags_extra]

CMakeLists.txt

@@ -37,6 +37,7 @@ option(FF_USE_PREBUILT_LEGION "Enable use of Legion pre-compiled library, if ava
 option(FF_USE_ALL_PREBUILT_LIBRARIES "Enable use of all pre-compiled libraries, if available" OFF)
 option(FF_USE_PYTHON "Enable Python" ON)
 option(FF_BUILD_FROM_PYPI "Build from pypi" OFF)
+option(FF_USE_CODE_COVERAGE "Enable code coverage" OFF)
 
 set(FF_GASNET_CONDUITS aries udp mpi ibv ucx)
 set(FF_GASNET_CONDUIT "mpi" CACHE STRING "Select GASNet conduit ${FF_GASNET_CONDUITS}")
@@ -71,7 +72,7 @@ option(FF_BUILD_SPLIT_TEST_2 "build split test 2 example" OFF)
 option(FF_BUILD_ALL_EXAMPLES "build all examples. Overrides others" OFF)
 option(FF_BUILD_UNIT_TESTS "build non-operator unit tests" OFF)
 option(FF_BUILD_SUBSTITUTION_TOOL "build substitution conversion tool" OFF)
-option(FF_BUILD_VISUALIZATION_TOOL "build substitution visualization tool" OFF)
+option(FF_BUILD_VISUALIZATION_TOOL "build substitution visualization tool" ON)
 option(FF_BUILD_ARG_PARSER "build command line argument parser" OFF)
 
 set(FF_CUDA_ARCH "autodetect" CACHE STRING "Target CUDA Arch")
@@ -86,6 +87,10 @@ endif()
 include(cuda)
 include(cudnn)
 include(nccl)
+if (FF_USE_CODE_COVERAGE)
+  include(CodeCoverage)
+  append_coverage_compiler_flags()
+endif()
 # set_property(CACHE FF_GPU_BACKEND PROPERTY STRINGS ${FF_GPU_BACKENDS})
 
 include(json)

bin/CMakeLists.txt

@@ -7,7 +7,7 @@ if(FF_BUILD_SUBSTITUTION_TOOL)
 endif()
 
 if(FF_BUILD_VISUALIZATION_TOOL)
-  add_subdirectory(substitutions_to_dot)
+  add_subdirectory(substitutions-to-dot)
 endif()
 
 if(FF_BUILD_ARG_PARSER)

bin/substitutions-to-dot/CMakeLists.txt

@@ -0,0 +1,8 @@
+ff_add_executable(
+  NAME
+    substitution-to-dot
+  SRC_PATTERNS
+    *.cc
+  DEPS
+    substitution-generator
+)

bin/substitutions_to_dot/substitution_to_dot.cc → bin/substitutions-to-dot/substitution_to_dot.cc

@@ -1,12 +1,9 @@
-#include "ffc/substitution_loader.h"
-#include "op-meta/ffconst.h"
-#include "tl/optional.hpp"
+#include "substitution-generator/json.h"
 #include "utils/dot_file.h"
 #include <cassert>
 #include <iostream>
 
-using namespace FlexFlow::substitution_loader;
-using FlexFlow::opmeta::get_operator_type_name;
+using namespace FlexFlow;
 
 enum class NodeType {
   SRC,
@@ -29,7 +26,7 @@ int main(int argc, char **argv) {
 
   RuleCollection rule_collection = load_rule_collection_from_path(json_path);
 
-  tl::optional<Rule> found = tl::nullopt;
+  std::optional<Rule> found = std::nullopt;
   for (Rule const &r : rule_collection.rules) {
     if (r.name == rule_name) {
       found = r;
@@ -88,10 +85,7 @@ int main(int argc, char **argv) {
     Operator const &o = r.srcOp[i];
     Node srcOpNode = {NodeType::SRC, i, 0};
     {
-      dot.add_node(
-          srcOpNode,
-          label_map(FlexFlow::opmeta::get_operator_type_name(o.op_type),
-                    srcOpNode));
+      dot.add_node(srcOpNode, label_map(fmt::to_string(o.op_type), srcOpNode));
       dot.add_node_to_subgraph(srcOpNode, src_body_subgraph);
     }
 
@@ -115,10 +109,7 @@ int main(int argc, char **argv) {
     Operator const &o = r.dstOp[j];
     Node dstOpNode = {NodeType::DST, j, 0};
     {
-      dot.add_node(
-          dstOpNode,
-          label_map(FlexFlow::opmeta::get_operator_type_name(o.op_type),
-                    dstOpNode));
+      dot.add_node(dstOpNode, label_map(fmt::to_string(o.op_type), dstOpNode));
       dot.add_node_to_subgraph(dstOpNode, dst_body_subgraph);
     }