Added juridecompose support (#43)

* Added juridecompose support and did some hacking to temporarily fix issues with api * Bumped version to 0.5.9 * Bumped kotlin version
discipl · Nov 23, 2021 · 6d295b7 · 6d295b7
1 parent bd634db
commit 6d295b7
Show file tree

Hide file tree

Showing 20 changed files with 20,466 additions and 17,900 deletions.
diff --git a/build.gradle b/build.gradle
@@ -65,7 +65,7 @@ configurations {
 repositories {
     mavenLocal()
     mavenCentral()
-    jcenter()
+    maven { url = uri("https://jcenter.bintray.com/") }
     if (System.getenv('CI') != null) {
         maven {
             url = uri("https://itemis.blackstardlb.nl")

diff --git a/code/java/FlintParser/build.gradle b/code/java/FlintParser/build.gradle
@@ -1,5 +1,5 @@
 plugins {
-    id 'org.jetbrains.kotlin.jvm' version '1.5.30'
+    id 'org.jetbrains.kotlin.jvm' version '1.6.0'
 }
 
 group 'org.discipl'

diff --git a/code/java/FlintSources/build.gradle b/code/java/FlintSources/build.gradle
@@ -1,13 +1,13 @@
 plugins {
-    id 'org.jetbrains.kotlin.jvm' version '1.5.30'
+    id 'org.jetbrains.kotlin.jvm' version '1.6.0'
 }
 
 group 'org.discipl'
 version '1.0-SNAPSHOT'
 
 repositories {
     mavenCentral()
-    jcenter()
+    maven { url = uri("https://jcenter.bintray.com/") }
 }
 
 ext {

diff --git a/...ava/FlintSources/src/main/kotlin/org/discipl/flint/sources/clients/AsyncTextLineClient.kt b/...ava/FlintSources/src/main/kotlin/org/discipl/flint/sources/clients/AsyncTextLineClient.kt
@@ -26,4 +26,5 @@ interface AsyncTextLine {
     val parent: String?
     val next: String?
     val textNodeType: String?
+    val number: String?
 }
diff --git a/...urces/src/main/kotlin/org/discipl/flint/sources/clients/nsx/NsxAsyncTextLineClientImpl.kt b/...urces/src/main/kotlin/org/discipl/flint/sources/clients/nsx/NsxAsyncTextLineClientImpl.kt
@@ -4,6 +4,7 @@ import io.ktor.client.*
 import io.ktor.client.request.*
 import io.ktor.client.request.forms.*
 import io.ktor.http.*
+import io.ktor.util.*
 import io.ktor.utils.io.core.*
 import kotlinx.coroutines.runBlocking
 import org.discipl.flint.sources.clients.AsyncTextLine
@@ -25,10 +26,10 @@ class NsxAsyncTextLineClientImpl(private val httpClient: HttpClient) : AsyncText
             "publicatieparsings",
             formData {
                 append("file", csv.toFile().readBytes(), Headers.build {
-                    append(HttpHeaders.ContentType, ContentType.Text.CSV)
+                    append(HttpHeaders.ContentType, ContentType.Text.CSV.toString())
                     append(HttpHeaders.ContentDisposition, "filename=${csv.fileName}")
                 })
-                append("documentStructure","EUR-LEX")
+                append("documentStructure", "EUR-LEX")
             }
         )
         result.id
@@ -58,15 +59,22 @@ class NsxAsyncTextLineClientImpl(private val httpClient: HttpClient) : AsyncText
     }
 
     private class AnAsyncTextLine(nsxTextLine: NsxTextLine) : AsyncTextLine {
+        private fun NsxTextLine.isPrefixLine(): Boolean {
+            if (this.structuurkenmerk == null) return false
+            if (this.structuurkenmerk.value?.startsWith("Onderdeel") == true && this.structuurkenmerk.number?.toIntOrNull() != null) return false
+            return true
+        }
+
         override val structure: String = nsxTextLine.fixedStructure
         override val type: String? = nsxTextLine.type
-        override val teken: String? = nsxTextLine.listIndex
+        override val teken: String? = nsxTextLine.listIndex ?: if (nsxTextLine.isPrefixLine()) nsxTextLine.structuurkenmerk?.number else null
         override val bibliographicIdentifierString: String? = nsxTextLine.bibliographicIdentifierString
         override val text: String = nsxTextLine.text
         override val id: String = nsxTextLine.iri
         override val textNodeType: String = nsxTextLine.textNodeType
         override val parent: String? = nsxTextLine.parent
         override val next: String? = nsxTextLine.next
+        override val number: String? = nsxTextLine.structuurkenmerk?.number
         override fun toString(): String {
             return "AnAsyncTextLine(structure='$structure', type=$type, teken=$teken, bibliographicIdentifierString=$bibliographicIdentifierString, text='$text', id='$id', textNodeType='$textNodeType', parent='$parent')"
         }

diff --git a/...ces/src/main/kotlin/org/discipl/flint/sources/clients/nsx/NsxTextLinesForVersionResult.kt b/...ces/src/main/kotlin/org/discipl/flint/sources/clients/nsx/NsxTextLinesForVersionResult.kt
@@ -1,7 +1,6 @@
 package org.discipl.flint.sources.clients.nsx
 
 import com.google.gson.annotations.SerializedName
-import java.util.*
 
 data class NsxTextLinesForVersionResult(
     @SerializedName("@graph")
@@ -17,20 +16,26 @@ data class NsxTextLine (
     val bibliographicIdentifierString: String?,
     @SerializedName("textnodeType")
     val textNodeType: String,
-    val stuctureCategory: String?,
+    val structureCategory: String?,
     val structure: String?,
+    @SerializedName("structuralIndex")
     val structuurkenmerk: StructuurKenmerk?,
     @SerializedName("@type")
     val type: String?,
     val next: String?
 ) {
     val fixedStructure: String get() {
         if (structure != null) return structure
-        if (stuctureCategory == "Artikel") {
+        if (structureCategory == "Artikel") {
             if (structuurkenmerk != null ) {
                 return "/${structuurkenmerk.name.replace("_", "")}"
             }
         }
+        if (structureCategory.isNullOrEmpty()) {
+            if (structuurkenmerk != null ) {
+                return "/${structuurkenmerk.name}"
+            }
+        }
         return ""
     }
 }
@@ -40,5 +45,6 @@ data class StructuurKenmerk(
     val number: String?,
     val name: String,
     @SerializedName("@type")
-    val type: String?
+    val type: String?,
+    val value: String?
 )
diff --git a/...src/main/kotlin/org/discipl/flint/sources/clients/triply/TriplyAsyncTextLineClientImpl.kt b/...src/main/kotlin/org/discipl/flint/sources/clients/triply/TriplyAsyncTextLineClientImpl.kt
@@ -45,5 +45,6 @@ class TriplyAsyncTextLineClientImpl(private val textLineClient: TextLineClient)
         override val parent: String? = textLine.parent
         override val next: String? = null
         override val textNodeType: String? = null
+        override val number: String? = null
     }
 }
diff --git a/code/java/FlintSources/src/main/kotlin/org/discipl/flint/sources/models/Article.kt b/code/java/FlintSources/src/main/kotlin/org/discipl/flint/sources/models/Article.kt
@@ -2,9 +2,14 @@ package org.discipl.flint.sources.models
 
 class Article(override val id: String) : IHasParts {
     private val _parts = mutableListOf<Part>()
-    val name get() = (_parts.first { it is ArticleTitle } as ArticleTitle).text
-    val nameLineNumber get() = (_parts.first { it is ArticleTitle } as ArticleTitle).lineNr
+    val name get() = articleTitle.text
+    val nameLineNumber get() = articleTitle.lineNr
     val articleTextParts get() = _parts.filter { it !is ArticleTitle }
+
+    private val articleTitle: ArticleTitle
+        get() = _parts.firstNotNullOfOrNull { it as? SpecifiedArticleTitle }
+            ?: _parts.firstNotNullOf { it as? UnspecifiedArticleTitle }
+
     override val parts: List<Part> get() = _parts
 
     override fun addPart(part: Part) {

diff --git a/code/java/FlintSources/src/main/kotlin/org/discipl/flint/sources/models/ArticleTitle.kt b/code/java/FlintSources/src/main/kotlin/org/discipl/flint/sources/models/ArticleTitle.kt
@@ -1,6 +1,21 @@
 package org.discipl.flint.sources.models
 
-class ArticleTitle(override val lineNr: Int, override val id: String, override val text: String) : Line {
+abstract class ArticleTitle : Line {
+    override fun toString(): String {
+        return text
+    }
+}
+
+class SpecifiedArticleTitle(override val lineNr: Int, override val id: String, override val text: String) :
+    ArticleTitle() {
+    override fun toString(): String {
+        return text
+    }
+}
+
+
+class UnspecifiedArticleTitle(override val lineNr: Int, override val id: String, override val text: String) :
+    ArticleTitle() {
     override fun toString(): String {
         return text
     }

diff --git a/code/java/FlintSources/src/main/kotlin/org/discipl/flint/sources/services/Constants.kt b/code/java/FlintSources/src/main/kotlin/org/discipl/flint/sources/services/Constants.kt
@@ -3,5 +3,5 @@ package org.discipl.flint.sources.services
 import java.util.concurrent.Callable
 
 object Strings {
-    var baseUrl: Callable<String> = Callable { "http://calculemus-app.d3e4884570654486a16f.westeurope.aksapp.io/calculemus/calculemusComp/v1" }
+    var baseUrl: Callable<String> = Callable { "http://calculemus-app.d3e4884570654486a16f.westeurope.aksapp.io/calculemus/normTexts/v1" }
 }
diff --git a/...ources/src/main/kotlin/org/discipl/flint/sources/transformers/AsyncTextLineTransformer.kt b/...ources/src/main/kotlin/org/discipl/flint/sources/transformers/AsyncTextLineTransformer.kt
@@ -5,7 +5,7 @@ import org.discipl.flint.sources.models.*
 
 class AsyncTextLineTransformer {
     private fun AsyncTextLine.toArticleTitle(regelNr: Int): ArticleTitle {
-        return ArticleTitle(
+        return SpecifiedArticleTitle(
             regelNr,
             this.id,
             this.text
@@ -81,30 +81,63 @@ class AsyncTextLineTransformer {
         }
     }
 
-    fun toNode(line: AsyncTextLine, textLines: List<AsyncTextLine>): AsyncTextLineNode {
+    fun toNode(line: AsyncTextLine, textLines: MutableList<AsyncTextLine>): AsyncTextLineNode {
+        textLines.remove(line)
         return AsyncTextLineNode(
             line,
             line.getFirstChild(textLines)?.let { toNode(it, textLines) },
             line.getNext(textLines)?.let { toNode(it, textLines) }
         )
     }
 
+    fun firstLooseSibling(textLines: List<AsyncTextLine>): AsyncTextLine? {
+        val nexts = textLines.filter { it.next != null }.map { it.next }
+        return textLines.firstOrNull { !nexts.contains(it.id) }
+    }
+
+    fun toNodeWithLose(
+        first: AsyncTextLine,
+        remaining: MutableList<AsyncTextLine>,
+        textLines: MutableList<AsyncTextLine>
+    ): AsyncTextLineNode {
+        textLines.remove(first)
+        remaining.remove(first)
+        val next = first.getNext(remaining) ?: firstLooseSibling(remaining)
+        remaining.remove(next)
+        textLines.remove(next)
+        return AsyncTextLineNode(
+            first,
+            first.getFirstChild(textLines)?.let { toNode(it, textLines) },
+            next?.let { toNodeWithLose(it, remaining, textLines) }
+        )
+    }
+
     fun toArticleList(textLines: List<AsyncTextLine>): List<Article> {
-        val topOfTree = textLines.first { it.getParent(textLines) == null }
-        val topNode = toNode(topOfTree, textLines)
+        val looseNodes = textLines.filter { it.getParent(textLines) == null }
+        val topNode = toNodeWithLose(firstLooseSibling(looseNodes)!!, looseNodes.toMutableList(), textLines.toMutableList())
         val sortedTextLines = topNode.iterator().asSequence().toList().map { it.value }
-
         fun toPartAndAddChildren(asyncTextLine: AsyncTextLine, regelNr: Int): Part {
             val part: Part = asyncTextLine.toPart(sortedTextLines, regelNr)
+            if (part is Article) {
+                part.addPart(
+                    UnspecifiedArticleTitle(
+                        regelNr,
+                        asyncTextLine.id,
+                        asyncTextLine.structure.replace("/", "")
+                    )
+                )
+                if (asyncTextLine.text.isNotBlank()) {
+                    part.addPart(asyncTextLine.toSimpleLine(regelNr))
+                }
+            }
             if (part is IHasParts) {
                 asyncTextLine.getChildren(sortedTextLines).map { toPartAndAddChildren(it, regelNr) }
                     .forEach { part.addPart(it) }
             }
-
             return part
         }
 
-        return sortedTextLines.filter { isArticle(it) }.map { toPartAndAddChildren(it, sortedTextLines.indexOf(it)) }
+        return sortedTextLines.filter { isArticle(it) }.sortedBy { it.number?.toIntOrNull() ?: sortedTextLines.indexOf(it) }.map { toPartAndAddChildren(it, it.number?.toIntOrNull() ?: sortedTextLines.indexOf(it)) }
             .filterIsInstance<Article>()
     }
 }

diff --git a/...lintSources/src/main/kotlin/org/discipl/flint/sources/transformers/TextLineTransformer.kt b/...lintSources/src/main/kotlin/org/discipl/flint/sources/transformers/TextLineTransformer.kt
@@ -92,7 +92,7 @@ class TextLineTransformer {
     }
 
     private fun TextLine.toArticleTitle(): ArticleTitle {
-        return ArticleTitle(
+        return SpecifiedArticleTitle(
             this.regelNr,
             this.id,
             this.text

diff --git a/...intSources/src/test/kotlin/org/discipl/flint/sources/clients/NsxTextLineClientImplTest.kt b/...intSources/src/test/kotlin/org/discipl/flint/sources/clients/NsxTextLineClientImplTest.kt
@@ -1,14 +1,9 @@
 package org.discipl.flint.sources.clients
 
 import org.discipl.flint.sources.di.*
-import org.discipl.flint.sources.services.Strings
 import org.junit.jupiter.api.Assertions.*
 import org.junit.jupiter.api.Test
-import org.koin.core.component.KoinComponent
-import org.koin.core.context.KoinContext
-import java.nio.file.Paths
 import java.util.*
-import java.util.concurrent.Callable
 
 internal class AsyncTextLineClientImplTest {
     private val textLineClient: AsyncTextLineClient = TestSourceLoader.asyncTextLineClient
@@ -189,25 +184,25 @@ internal class AsyncTextLineClientImplTest {
         )
 
         assertEquals(1499, results.size)
-        val first = results.first()
+        val textLine = results[1415]
         assertEquals(
             "Verwerking en vrijheid van meningsuiting en van informatie",
-            first.text
+            textLine.text
         )
         assertEquals(
-            "https://calculemus.org/429661c1-48d5-4b59-900e-708124d85678",
-            first.id
+            "https://calculemus.org/767d39c9-d325-455f-9222-60da821dfd1b",
+            textLine.id
         )
-        assertEquals("", first.structure)
+        assertEquals("", textLine.structure)
         assertEquals(
             null,
-            first.bibliographicIdentifierString
+            textLine.bibliographicIdentifierString
         )
         assertEquals(
-            "https://calculemus.org/3364277f-93b2-4572-a7fc-71ddf05f3420",
-            first.parent
+            "https://calculemus.org/f3e456a3-3de8-493e-9602-7b3ed373c92a",
+            textLine.parent
         )
-        assertEquals(null, first.teken)
-        assertEquals("TextNode", first.type)
+        assertEquals(null, textLine.teken)
+        assertEquals("TextNode", textLine.type)
     }
 }
diff --git a/code/java/FlintSources/src/test/kotlin/org/discipl/flint/sources/di/NsxResponseHandler.kt b/code/java/FlintSources/src/test/kotlin/org/discipl/flint/sources/di/NsxResponseHandler.kt
@@ -47,6 +47,11 @@ val defaultResult by lazy {
     getResourceAsString("parseRequestResult.json")
 }
 
+val juriDecomposeResult by lazy {
+    getResourceAsString("juriDecomposeResponse.json")
+}
+
+
 val cov19Result by lazy {
     getResourceAsString("cov19parseResult2.json")
 }