Merge branch 'master' into rti-DNET

.github/workflows/c-embedded.yml

@@ -25,5 +25,5 @@ jobs:
 #     uses: ./.github/workflows/c-flexpret-tests.yml
 
       # Run the C Patmos integration tests.
-  patmos:
-    uses: ./.github/workflows/c-patmos-tests.yml
+#  patmos:
+#    uses: ./.github/workflows/c-patmos-tests.yml

.github/workflows/c-flexpret-tests.yml

@@ -45,7 +45,7 @@ jobs:
         if: ${{ inputs.runtime-ref }}
       - name: Run FlexPRET smoke tests
         run: |
-          cd "$FP_DIR" && source env.bash && cd -
+          source $FP_DIR/env.bash
           ./gradlew core:integrationTest \
             --tests org.lflang.tests.runtime.CFlexPRETTest.* \
             core:integrationTestCodeCoverageReport

.github/workflows/lsp-tests.yml

@@ -69,7 +69,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.12"
       - name: Run language server Python tests without PyLint
         run: ./gradlew core:integrationTest --tests org.lflang.tests.lsp.LspTests.pythonValidationTestSyntaxOnly core:integrationTestCodeCoverageReport
       - name: Install pylint

.github/workflows/py-tests.yml

@@ -36,7 +36,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.12'
       - name: Install dependencies OS X
         run: |
           brew install coreutils

core/src/main/java/org/lflang/federated/extensions/CExtensionUtils.java

@@ -313,6 +313,7 @@ public static void generateCMakeInclude(
         "add_compile_definitions(LF_PACKAGE_DIRECTORY=\"" + fileConfig.srcPkgPath + "\")");
     cmakeIncludeCode.pr(
         "add_compile_definitions(LF_SOURCE_GEN_DIRECTORY=\"" + fileConfig.getSrcGenPath() + "\")");
+    cmakeIncludeCode.pr("add_compile_definitions(LF_FILE_SEPARATOR=\"" + File.separator + "\")");
     try (var srcWriter = Files.newBufferedWriter(cmakeIncludePath)) {
       srcWriter.write(cmakeIncludeCode.getCode());
     }

core/src/main/java/org/lflang/federated/extensions/PythonExtension.java

@@ -85,6 +85,11 @@ protected String generateSerializationIncludes(
     return code.getCode();
   }
 
+  @Override
+  public String getNetworkBufferType() {
+    return "PyObject*";
+  }
+
   @Override
   public String generateNetworkSenderBody(
       VarRef sendingPort,

core/src/main/java/org/lflang/federated/launcher/FedLauncherGenerator.java

@@ -429,6 +429,7 @@ private String getDistCode(Path remoteBase, FederateInstance federate) {
     String logDirectory = "~/" + remoteBase + "/" + fileConfig.name + "/log";
     String remoteBuildLogFileName = logDirectory + "/build.log";
     String buildShellFileName = "build_" + federate.name + ".sh";
+    String tarFileName = federate.name + ".tar.gz";
     return String.join(
         "\n",
         "echo \"Making directory "
@@ -447,18 +448,26 @@ private String getDistCode(Path remoteBase, FederateInstance federate) {
             + "; \\",
         "    date >> " + remoteBuildLogFileName + ";",
         "'",
-        "pushd " + fileConfig.getSrcGenPath() + "/" + federate.name + " > /dev/null",
-        "echo \"**** Copying source files to host "
-            + getUserHost(federate.user, federate.host)
-            + "\"",
-        "scp -r * "
+        "pushd " + fileConfig.getSrcGenPath() + " > /dev/null",
+        "echo \"**** Bundling source files into " + tarFileName + "\"",
+        "tar -czf " + tarFileName + " --exclude build " + federate.name,
+        "echo \"**** Copying tarfile to host " + getUserHost(federate.user, federate.host) + "\"",
+        "scp -r "
+            + tarFileName
+            + " "
             + getUserHost(federate.user, federate.host)
             + ":"
             + remoteBase
             + "/"
             + fileConfig.name
             + "/"
-            + federate.name,
+            + tarFileName,
+        "rm " + tarFileName,
+        "ssh " + getUserHost(federate.user, federate.host) + " '\\",
+        "    cd ~/" + remoteBase + "/" + fileConfig.name + "; \\",
+        "    tar -xzf " + tarFileName + "; \\",
+        "    rm " + tarFileName + ";",
+        "'",
         "popd > /dev/null",
         "echo \"**** Generating and executing compile.sh on host "
             + getUserHost(federate.user, federate.host)
@@ -517,26 +526,6 @@ private String getDistCode(Path remoteBase, FederateInstance federate) {
             + "'");
   }
 
-  /** Return the body of a shell script file to compile the specified federate. */
-  private String getCompileScript(Path remoteBase, FederateInstance federate) {
-    String baseDir = "~/" + remoteBase + "/" + fileConfig.name;
-    return String.join(
-        "\n",
-        "#!/bin/bash -l", // The -l argument makes this a login shell so PATH etc are inherited.
-        // FIXME: Put copied files in subdirectory federate.name
-        "cd " + remoteBase + "/fed-gen/" + fileConfig.name + "/src-gen/" + federate.name,
-        "rm -rf build",
-        "mkdir -p ~/" + remoteBase + "/log",
-        // >> appends stdout to the specified file, and 2>&1 appends stderr to the same file.
-        "mkdir -p build && cd build && cmake .. && make >> "
-            + baseDir
-            + "/"
-            + federate.name
-            + ".log 2>&1",
-        "mkdir -p ~/" + remoteBase + "/bin;\\",
-        "mv " + federate.name + " ~/" + remoteBase + "/bin;'");
-  }
-
   private String getUserHost(Object user, Object host) {
     if (user == null) {
       return host.toString();

core/src/main/java/org/lflang/generator/c/CCmakeGenerator.java

@@ -69,7 +69,7 @@ public class CCmakeGenerator {
 
   private final FileConfig fileConfig;
   private final List<String> additionalSources;
-  private final SetUpMainTarget setUpMainTarget;
+  private SetUpMainTarget setUpMainTarget;
   private final String installCode;
 
   public CCmakeGenerator(FileConfig fileConfig, List<String> additionalSources) {
@@ -90,6 +90,15 @@ public CCmakeGenerator(
     this.installCode = installCode;
   }
 
+  /**
+   * Set the code generator for the CMake main target.
+   *
+   * @param setUpMainTarget
+   */
+  public void setCmakeGenerator(SetUpMainTarget setUpMainTarget) {
+    this.setUpMainTarget = setUpMainTarget;
+  }
+
   /**
    * Generate the contents of a CMakeLists.txt that builds the provided LF C 'sources'. Any error
    * will be reported in the 'errorReporter'.
@@ -228,10 +237,14 @@ CodeBuilder generateCMakeCode(
         break;
       case PATMOS:
         cMakeCode.newLine();
+        cMakeCode.pr("SET(CMAKE_SYSTEM_NAME patmos)");
+        cMakeCode.pr("SET(CMAKE_SYSTEM_PROCESSOR patmos)");
         cMakeCode.pr("# Include toolchain file and set project");
         cMakeCode.pr(
             "find_program(CLANG_EXECUTABLE NAMES patmos-clang REQUIRED DOC \"Path to the clang"
                 + " front-end.\")");
+        cMakeCode.pr("set(CMAKE_C_FLAGS_INIT \"-O2 -DNDEBUG\")");
+
         cMakeCode.pr("set(CMAKE_C_COMPILER ${CLANG_EXECUTABLE})");
         cMakeCode.pr(
             "set(CMAKE_C_FLAGS_RELEASE \"-O2 -DNDEBUG\")"); // patmos-clang cannot compiler -O3

core/src/main/java/org/lflang/generator/c/CGenerator.java

@@ -102,7 +102,6 @@
 import org.lflang.target.property.TracingProperty;
 import org.lflang.target.property.WorkersProperty;
 import org.lflang.target.property.type.PlatformType.Platform;
-import org.lflang.target.property.type.SchedulerType.Scheduler;
 import org.lflang.util.ArduinoUtil;
 import org.lflang.util.FileUtil;
 import org.lflang.util.FlexPRETUtil;
@@ -320,7 +319,7 @@ public class CGenerator extends GeneratorBase {
 
   private final CTypes types;
 
-  private final CCmakeGenerator cmakeGenerator;
+  protected CCmakeGenerator cmakeGenerator;
 
   protected CGenerator(
       LFGeneratorContext context,
@@ -741,31 +740,6 @@ else if (term.getParameter() != null)
     return result.toString();
   }
 
-  /** Set the scheduler type in the target config as needed. */
-  private void pickScheduler() {
-    // Don't use a scheduler that does not prioritize reactions based on deadlines
-    // if the program contains a deadline (handler). Use the GEDF_NP scheduler instead.
-    if (!targetConfig.get(SchedulerProperty.INSTANCE).prioritizesDeadline()) {
-      // Check if a deadline is assigned to any reaction
-      if (hasDeadlines(reactors)) {
-        if (!targetConfig.isSet(SchedulerProperty.INSTANCE)) {
-          SchedulerProperty.INSTANCE.override(targetConfig, Scheduler.GEDF_NP);
-        }
-      }
-    }
-  }
-
-  private boolean hasDeadlines(List<Reactor> reactors) {
-    for (Reactor reactor : reactors) {
-      for (Reaction reaction : allReactions(reactor)) {
-        if (reaction.getDeadline() != null) {
-          return true;
-        }
-      }
-    }
-    return false;
-  }
-
   /**
    * Copy all files or directories listed in the target property {@code files}, {@code
    * cmake-include}, and {@code _fed_setup} into the src-gen folder of the main .lf file
@@ -2033,7 +2007,6 @@ protected boolean setUpGeneralParameters() {
       CompileDefinitionsProperty.INSTANCE.update(targetConfig, Map.of("MODAL_REACTORS", "TRUE"));
     }
     if (!targetConfig.get(SingleThreadedProperty.INSTANCE)) {
-      pickScheduler();
       CompileDefinitionsProperty.INSTANCE.update(
           targetConfig,
           Map.of(

core/src/main/java/org/lflang/generator/python/PythonGenerator.java

@@ -65,6 +65,7 @@
 import org.lflang.target.Target;
 import org.lflang.target.property.DockerProperty;
 import org.lflang.target.property.ProtobufsProperty;
+import org.lflang.target.property.PythonVersionProperty;
 import org.lflang.util.FileUtil;
 import org.lflang.util.LFCommand;
 
@@ -82,14 +83,17 @@
  *
  * @author Soroush Bateni
  */
-public class PythonGenerator extends CGenerator {
+public class PythonGenerator extends CGenerator implements CCmakeGenerator.SetUpMainTarget {
 
   // Used to add statements that come before reactor classes and user code
   private final CodeBuilder pythonPreamble = new CodeBuilder();
 
   // Used to add module requirements to setup.py (delimited with ,)
   private final List<String> pythonRequiredModules = new ArrayList<>();
 
+  /** Indicator that we have already generated top-level preambles. */
+  private Set<Model> generatedTopLevelPreambles = new HashSet<Model>();
+
   private final PythonTypes types;
 
   public PythonGenerator(LFGeneratorContext context) {
@@ -104,8 +108,9 @@ public PythonGenerator(LFGeneratorContext context) {
                 "lib/python_tag.c",
                 "lib/python_time.c",
                 "lib/pythontarget.c"),
-            PythonGenerator::setUpMainTarget,
+            null, // Temporarily, because can't pass this.
             generateCmakeInstall(context.getFileConfig())));
+    cmakeGenerator.setCmakeGenerator(this);
   }
 
   private PythonGenerator(
@@ -186,6 +191,7 @@ public String generatePythonCode(String pyModuleName) {
         "\n",
         "import os",
         "import sys",
+        "print(\"******* Using Python version: %s.%s.%s\" % sys.version_info[:3])",
         "sys.path.append(os.path.dirname(__file__))",
         "# List imported names, but do not use pylint's --extension-pkg-allow-list option",
         "# so that these names will be assumed present without having to compile and install.",
@@ -270,7 +276,12 @@ protected String generateTopLevelPreambles(Reactor ignored) {
       models.add((Model) ASTUtils.toDefinition(this.mainDef.getReactorClass()).eContainer());
     }
     for (Model m : models) {
-      pythonPreamble.pr(PythonPreambleGenerator.generatePythonPreambles(m.getPreambles()));
+      // In the generated Python code, unlike C, all reactors go into the same file.
+      // Therefore, we do not need to generate this if it has already been generated.
+      if (!generatedTopLevelPreambles.contains(m)) {
+        generatedTopLevelPreambles.add(m);
+        pythonPreamble.pr(PythonPreambleGenerator.generatePythonPreambles(m.getPreambles()));
+      }
     }
     return PythonPreambleGenerator.generateCIncludeStatements(
         targetConfig, targetLanguageIsCpp(), hasModalReactors);
@@ -486,9 +497,6 @@ protected void generateUserPreamblesForReactor(Reactor reactor, CodeBuilder src)
   @Override
   protected void generateReactorClassHeaders(
       TypeParameterizedReactor tpr, String headerName, CodeBuilder header, CodeBuilder src) {
-    header.pr(
-        PythonPreambleGenerator.generateCIncludeStatements(
-            targetConfig, targetLanguageIsCpp(), hasModalReactors));
     super.generateReactorClassHeaders(tpr, headerName, header, src);
   }
 
@@ -639,15 +647,27 @@ protected void additionalPostProcessingForModes() {
     PythonModeGenerator.generateResetReactionsIfNeeded(reactors);
   }
 
-  private static String setUpMainTarget(
-      boolean hasMain, String executableName, Stream<String> cSources) {
+  public String getCmakeCode(boolean hasMain, String executableName, Stream<String> cSources) {
+    // According to https://cmake.org/cmake/help/latest/module/FindPython.html#hints, the following
+    // should work to select the version of Python given in your virtual environment.
+    // However, this does not work for me (macOS Sequoia 15.0.1).
+    // As a consequence, the python-version target property can be used to specify the exact Python
+    // version.
+    var pythonVersion =
+        "3.10.0"; // Allows 3.10 or later. Change to "3.10.0...<3.11.0" to require 3.10 by default.
+    if (targetConfig.isSet(PythonVersionProperty.INSTANCE)) {
+      pythonVersion = targetConfig.get(PythonVersionProperty.INSTANCE) + " EXACT";
+    }
     return ("""
             set(CMAKE_POSITION_INDEPENDENT_CODE ON)
             add_compile_definitions(_PYTHON_TARGET_ENABLED)
             add_subdirectory(core)
             set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR})
             set(LF_MAIN_TARGET <pyModuleName>)
-            find_package(Python 3.10.0...<3.11.0 REQUIRED COMPONENTS Interpreter Development)
+            set(Python_FIND_VIRTUALENV FIRST)
+            set(Python_FIND_STRATEGY LOCATION)
+            set(Python_FIND_FRAMEWORK LAST)
+            find_package(Python <pyVersion> REQUIRED COMPONENTS Interpreter Development)
             Python_add_library(
                 ${LF_MAIN_TARGET}
                 MODULE
@@ -667,7 +687,8 @@ private static String setUpMainTarget(
 target_link_libraries(${LF_MAIN_TARGET} PRIVATE ${Python_LIBRARIES})
 target_compile_definitions(${LF_MAIN_TARGET} PUBLIC MODULE_NAME=<pyModuleName>)
 """)
-        .replace("<pyModuleName>", generatePythonModuleName(executableName));
+        .replace("<pyModuleName>", generatePythonModuleName(executableName))
+        .replace("<pyVersion>", pythonVersion);
     // The use of fileConfig.name will break federated execution, but that's fine
   }
 
@@ -677,6 +698,9 @@ private static String generateCmakeInstall(FileConfig fileConfig) {
     // need to replace '\' with '\\' on Windwos for proper escaping in cmake
     final var pyMainName = pyMainPath.toString().replace("\\", "\\\\");
     return """
+  if (NOT DEFINED CMAKE_INSTALL_BINDIR)
+    set(CMAKE_INSTALL_BINDIR "bin")
+  endif()
   if(WIN32)
     file(GENERATE OUTPUT <fileName>.bat CONTENT
       "@echo off

core/src/main/java/org/lflang/generator/python/PythonPortGenerator.java

@@ -177,7 +177,6 @@ public static String generatePythonListForContainedBank(
         "    }",
         "    /* Release the thread. No Python API allowed beyond this point. */",
         "    PyGILState_Release(gstate);",
-        "    Py_FinalizeEx();",
         "    exit(1);",
         "}",
         "for (int i = 0; i < " + generateWidthVariable(reactorName) + "; i++) {",
@@ -193,7 +192,6 @@ public static String generatePythonListForContainedBank(
         "        }",
         "        /* Release the thread. No Python API allowed beyond this point. */",
         "        PyGILState_Release(gstate);",
-        "        Py_FinalizeEx();",
         "        exit(1);",
         "    }",
         "}");

core/src/main/java/org/lflang/generator/python/PythonPreambleGenerator.java

@@ -44,8 +44,8 @@ public static String generateCDefineDirectives(
   public static String generateCIncludeStatements(
       TargetConfig targetConfig, boolean CCppMode, boolean hasModalReactors) {
     CodeBuilder code = new CodeBuilder();
-    code.pr(CPreambleGenerator.generateIncludeStatements(targetConfig, CCppMode));
     code.pr("#include \"pythontarget.h\"");
+    code.pr(CPreambleGenerator.generateIncludeStatements(targetConfig, CCppMode));
     if (hasModalReactors) {
       code.pr("#include \"include/modal_models/definitions.h\"");
     }

core/src/main/java/org/lflang/generator/python/PythonReactionGenerator.java

@@ -102,10 +102,16 @@ private static String generateCPythonFunctionCaller(
                 + "."
                 + pythonFunctionName
                 + "\");",
+            "PyObject *arglist = Py_BuildValue(\"("
+                + "O".repeat(pyObjects.size())
+                + ")\""
+                + pyObjectsJoined
+                + ");",
             "PyObject *rValue = PyObject_CallObject(",
             "    self->" + cpythonFunctionName + ", ",
-            "    Py_BuildValue(\"(" + "O".repeat(pyObjects.size()) + ")\"" + pyObjectsJoined + ")",
+            "    arglist",
             ");",
+            "Py_DECREF(arglist);",
             "if (rValue == NULL) {",
             "    lf_print_error(\"FATAL: Calling reaction "
                 + reactorDeclName