Merge pull request #24 from discipl/filler

Added integration with flintfiller

.github/workflows/release.yml

@@ -15,10 +15,9 @@ jobs:
         uses: actions/checkout@v2
       - name: Fetch Tags
         run: git fetch --prune --unshallow --tags -f
-      - name: Setup Java 11
-        uses: actions/setup-java@v1
+      - uses: actions/setup-java@v1
         with:
-          java-version: '11'
+          java-version: 11
       - name: Build Plugin
         run: ./gradlew initProject buildFlintPlugin
         env:
@@ -29,6 +28,12 @@ jobs:
           run: ./gradlew buildFlintIde
         env:
           GRADLE_OPTS: -Dorg.gradle.daemon=false
+      - name: Upload Test Results
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: test-results
+          path: code/java/FlintParser/build/reports/tests/test
       - name: Get Release Logs
         id: get_release_logs
         run: |

.github/workflows/test.yml

@@ -0,0 +1,34 @@
+name: Tests
+on:
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ ubuntu-16.04, macos-latest, windows-latest ]
+    name: Run Tests
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - uses: actions/setup-java@v1
+        with:
+          java-version: 11
+      - name: Test
+        uses: GabrielBB/xvfb-action@v1
+        with:
+          run: ./gradlew initProject test
+        env:
+          GRADLE_OPTS: -Dorg.gradle.daemon=false
+      - name: Tests failed
+        if: ${{ matrix.os == 'ubuntu-16.04' && failure() }}
+        run: sudo apt-get install lynx && if [ -f ./code/build/reports/html/junit-noframes.html ]; then lynx -dump ./code/build/reports/html/junit-noframes.html; fi && if [ -f ./code/java/FlintParser/build/reports/tests/test/index.html ]; then lynx -dump ./code/java/FlintParser/build/reports/tests/test/index.html; fi && if [ -f ./code/java/FlintParser/build/reports/tests/test/classes/org.discipl.flint.flintfiller.FlintFillerTest.html ]; then lynx -dump ./code/java/FlintParser/build/reports/tests/test/classes/org.discipl.flint.flintfiller.FlintFillerTest.html; fi
+      - name: Upload Test Results
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: test-results-${{ matrix.os }}
+          path: code/java/FlintParser/build/reports/tests/test

.gitignore

@@ -76,6 +76,7 @@ libs
 platform-libs
 artifacts
 static-resources/generated
+static-resources/temp
 
 workspace.xml
 junitvmwatcher*.properties

build.gradle

@@ -4,7 +4,6 @@ buildscript {
             url = uri("https://projects.itemis.de/nexus/content/repositories/mbeddr")
         }
         mavenCentral()
-        mavenLocal()
     }
     dependencies {
         classpath "de.itemis.mps:mps-gradle-plugin:$mps_gradle_plugin_version"
@@ -34,6 +33,9 @@ ext {
     current_platform = currentPlatform()
     generateTask = null
 
+    flint_filler_release_version = "v0.0.3"
+    flint_filler_executables_dir = new File(generatedResourcesDir, "flintfiller")
+
     javafx_version = "14"
     mbeddr_version = "2019.3.+"
     mps_version = "2019.3.4"
@@ -67,13 +69,17 @@ dependencies {
         mpsplatformlib "org.openjfx:javafx-swing:$javafx_version:$platform"
         mpsplatformlib "org.openjfx:javafx-web:$javafx_version:$platform"
     }
+
     mpslib(project("code:java:FlintParser")) {
         exclude group: 'org.openjfx'
     }
     mpslib 'com.googlecode.json-simple:json-simple:1.1.1'
+    mpslib 'commons-io:commons-io:2.7'
+
     artifact "com.mbeddr:platform:$mbeddr_version"
     mps("com.jetbrains:mps:$mps_version")
     antLib("org.apache.ant:ant-junit:1.10.1")
+
     mpsGenLib "org.openjfx:javafx-base:$javafx_version:$current_platform"
     mpsGenLib "org.openjfx:javafx-controls:$javafx_version:$current_platform"
     mpsGenLib "org.openjfx:javafx-graphics:$javafx_version:$current_platform"
@@ -85,8 +91,6 @@ dependencies {
 ext["itemis.mps.gradle.ant.defaultScriptArgs"] = ["-Dmps_home=$mpsDir.path".toString()]
 ext["itemis.mps.gradle.ant.defaultScriptClasspath"] = project.configurations.antLib.fileCollection({ true })
 
-project.configurations.antLib.fileCollection({ true }).forEach { println it }
-
 node {
     // Version of node to use.
     version = node_version
@@ -141,7 +145,7 @@ task resolveLibs(type: Copy) {
 }
 
 task cleanResources(type: Delete) {
-    delete(generatedResourcesDir)
+    delete fileTree(dir: "$generatedResourcesDir", exclude: "**/flintfiller/*")
 }
 
 task fetchComplianceByDesign(type: Copy) {
@@ -154,6 +158,7 @@ task fetchResources() {
     group "dependancies"
     dependsOn cleanResources
     finalizedBy fetchComplianceByDesign
+    finalizedBy "fetchFlintFillerExecutables"
 }
 
 task cleanPlatformLibs(type: Delete) {
@@ -317,3 +322,30 @@ task htmlReport(type: de.itemis.mps.gradle.RunAntScript, dependsOn: collectXmlRe
     group "verification"
     script = file("$codeDir/html-report.xml")
 }
+
+def downloadFillerFile(name) {
+    def f = new File(flint_filler_executables_dir, name)
+    f.getParentFile().mkdirs();
+    if (f.exists()) {
+        f.delete()
+    }
+    new URL("https://github.com/discipl/flintfiller/releases/download/${flint_filler_release_version}/${name}").withInputStream{ i -> f.withOutputStream{ it << i }}
+    return f
+}
+
+
+task fetchFlintFillerExecutables() {
+    doLast {
+        def files = []
+        files.add(downloadFillerFile("flintfiller-linux"))
+        files.add(downloadFillerFile("flintfiller-macos"))
+        files.add(downloadFillerFile("flintfiller-windows.exe"))
+        if (!OperatingSystem.current().isWindows()) {
+            files.each { file ->
+                exec {
+                    commandLine "chmod", "+x", file.path
+                }
+            }
+        }
+    }
+}

code/java/FlintParser/build.gradle

@@ -14,13 +14,17 @@ dependencies {
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8"
     implementation 'com.google.code.gson:gson:2.8.6'
     implementation 'io.gsonfire:gson-fire:1.8.4'
+
     implementation "org.openjfx:javafx-base:$javafx_version:$current_platform"
     implementation "org.openjfx:javafx-controls:$javafx_version:$current_platform"
     implementation "org.openjfx:javafx-graphics:$javafx_version:$current_platform"
     implementation "org.openjfx:javafx-media:$javafx_version:$current_platform"
     implementation "org.openjfx:javafx-swing:$javafx_version:$current_platform"
     implementation "org.openjfx:javafx-web:$javafx_version:$current_platform"
 
+    implementation 'org.apache.commons:commons-lang3:3.11'
+    implementation 'org.apache.commons:commons-exec:1.3'
+
     testImplementation 'org.junit.jupiter:junit-jupiter-api:5.6.1'
     testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.6.1'
     testImplementation 'org.hamcrest:hamcrest-library:2.2'
@@ -34,5 +38,12 @@ compileTestKotlin {
 }
 
 test {
+    environment "FLINT_FILLER_DIR", flint_filler_executables_dir.path
+    environment "FLINT_FILLER_OUTPUT_DIR", new File(buildDir, "tmp").path
+    environment "FLINT_FILLER_TEST_FILE", new File(projectDir, "src/test/resources/flint-filler-test-file.xml").path
     useJUnitPlatform()
 }
+
+if (!new File(flint_filler_executables_dir, "flintfiller-linux").exists()) {
+    tasks.getByName("test").dependsOn  ":fetchFlintFillerExecutables"
+}

code/java/FlintParser/src/main/kotlin/org/discipl/flint/flintfiller/FlintFiller.kt

@@ -0,0 +1,53 @@
+package org.discipl.flint.flintfiller
+
+import org.apache.commons.exec.CommandLine
+import org.apache.commons.exec.DefaultExecutor
+import org.apache.commons.exec.PumpStreamHandler
+import org.apache.commons.lang3.SystemUtils
+import java.nio.file.Files
+import java.nio.file.Path
+
+
+class FlintFiller(private val pathToFillerDir: String, private val outputDir: String) {
+    fun run(file: String, onCommandOutput: (String) -> Unit = {}): String {
+        Files.createDirectories(Path.of(outputDir))
+        val command = "${pathToFillerDir}/${osSpecificFlintFiller()} ${fillerArgs(file, outputDir)}"
+        val cmdLine: CommandLine = CommandLine.parse(command)
+
+        val outputStream = StringOutputStream()
+        val streamHandler = PumpStreamHandler(outputStream)
+
+        val executor = DefaultExecutor()
+        executor.streamHandler = streamHandler
+
+        try {
+            val exitCode = executor.execute(cmdLine)
+            if (exitCode != 0) throw Exception("Bad exit code")
+            onCommandOutput("command output:\n${outputStream.value()}")
+            return Files.readString(Path.of(outputDir).resolve("flintFrame.json"))
+        } catch (e: Exception) {
+            throw Exception(
+                "Something went wrong while running flint filler\ncommand output:\n${outputStream.value()}",
+                e
+            )
+        }
+    }
+
+    private fun osSpecificFlintFiller(): String {
+        return when {
+            SystemUtils.IS_OS_WINDOWS -> "flintfiller-windows.exe"
+            SystemUtils.IS_OS_LINUX -> "flintfiller-linux"
+            SystemUtils.IS_OS_MAC_OSX -> "flintfiller-macos"
+            else -> throw NotImplementedError("Os ${SystemUtils.OS_NAME} is not supported")
+        }
+    }
+
+    private fun fillerArgs(inputFile: String, outputDir: String): String {
+        val inputFileLine = "-x ${inputFile}"
+        val dictFile = "-d ${outputDir}/dict.json"
+        val dataFrameFile = "-df ${outputDir}/dataFrame.csv"
+        val postTaggedDataFrameFile = "-pt ${outputDir}/postTaggedDataFrame_.csv"
+        val flintFrame = "-fo ${outputDir}/flintFrame.json"
+        return "$inputFileLine $dictFile $dataFrameFile $postTaggedDataFrameFile $flintFrame"
+    }
+}

code/java/FlintParser/src/main/kotlin/org/discipl/flint/flintfiller/StringOutputStream.kt

@@ -0,0 +1,15 @@
+package org.discipl.flint.flintfiller
+
+import java.io.OutputStream
+
+class StringOutputStream : OutputStream() {
+    private val sb = StringBuilder()
+
+    override fun write(x: Int) {
+        sb.append(x.toChar())
+    }
+
+    fun value(): String {
+        return sb.toString()
+    }
+}

code/java/FlintParser/src/main/kotlin/org/discipl/flint/typeselector/LiteralExpressionTypeSelector.kt

@@ -12,7 +12,7 @@ import org.discipl.flint.StringLiteralExpression
 class LiteralExpressionTypeSelector : TypeSelector<LiteralExpression<*>> {
     override fun getClassForElement(readElement: JsonElement): Class<out LiteralExpression<*>> {
         val operand: JsonPrimitive = (readElement as? JsonObject)?.get("operand") as? JsonPrimitive
-            ?: throw IllegalArgumentException("Not an literal expression")
+            ?: throw IllegalArgumentException("Not an literal expression: $readElement")
         return when {
             operand.isBoolean -> BooleanLiteralExpression::class.java
             operand.isNumber -> NumberLiteralExpression::class.java

code/java/FlintParser/src/main/kotlin/org/discipl/flint/ui/FlintWebViewJfxPanel.kt

@@ -26,7 +26,7 @@ class FlintWebViewJfxPanel(assetFilePath: Path, private val logger: JSLogger) :
             val scene = Scene(root)
 
             webEngine.loadWorker.stateProperty()
-                .addListener { observable: ObservableValue<out Worker.State?>?, oldValue: Worker.State?, newValue: Worker.State? ->
+                .addListener { _: ObservableValue<out Worker.State?>?, _: Worker.State?, _: Worker.State? ->
                     val window: JSObject = webEngine.executeScript("window") as JSObject
                     window.setMember("java", logger)
                     webEngine.executeScript(
@@ -45,7 +45,6 @@ console.error = function(message, e) {
             val children = root.children
             children.add(this.webView)
             this.scene = scene
-            load(null, null)
         }
     }
 

code/java/FlintParser/src/test/kotlin/org/discipl/flint/flintfiller/FlintFillerTest.kt

@@ -0,0 +1,21 @@
+package org.discipl.flint.flintfiller
+
+import org.hamcrest.MatcherAssert.assertThat
+import org.hamcrest.Matchers.`is`
+import org.hamcrest.Matchers.notNullValue
+import org.junit.jupiter.api.Test
+import java.nio.file.Path
+
+internal class FlintFillerTest {
+    private val filler: FlintFiller = FlintFiller(
+        System.getenv("FLINT_FILLER_DIR"),
+        System.getenv("FLINT_FILLER_OUTPUT_DIR")
+    )
+
+    @Test
+    fun run() {
+        val flintModel = filler.run(System.getenv("FLINT_FILLER_TEST_FILE"))
+        println(flintModel.substring(0..400))
+        assertThat(flintModel, `is`(notNullValue()))
+    }
+}