Merge pull request #106 from Starchasers/fix/plaintext-encoding

fix plaintext files encoding

spring-app/build.gradle.kts

@@ -30,6 +30,8 @@ dependencies {
     implementation("commons-fileupload:commons-fileupload:1.3.3")
     implementation("io.jsonwebtoken:jjwt:0.9.1")
     runtimeOnly("com.h2database:h2:1.4.200")
+    implementation("com.ibm.icu:icu4j:67.1")
+    runtimeOnly("com.h2database:h2:1.4.200")
     runtimeOnly("mysql:mysql-connector-java")
     runtimeOnly("org.mariadb.jdbc:mariadb-java-client")
     runtimeOnly(files("../gatsby-app/build/artifact/gatsby-app.jar"))

spring-app/src/main/kotlin/pl/starchasers/up/service/CharsetDetectionService.kt

@@ -0,0 +1,20 @@
+package pl.starchasers.up.service
+
+import com.ibm.icu.text.CharsetDetector
+import org.springframework.stereotype.Service
+import java.io.InputStream
+
+interface CharsetDetectionService {
+    fun detect(inputStream: InputStream): String
+}
+
+@Service
+class CharsetDetectionServiceImpl : CharsetDetectionService {
+    override fun detect(inputStream: InputStream): String {
+        val detector = CharsetDetector()
+        detector.setDeclaredEncoding("utf-8")
+        detector.setText(inputStream)
+        return detector.detect().name
+    }
+
+}

spring-app/src/main/kotlin/pl/starchasers/up/service/FileService.kt

@@ -39,7 +39,8 @@ interface FileService {
 class FileServiceImpl(
         private val fileStorageService: FileStorageService,
         private val fileEntryRepository: FileEntryRepository,
-        private val configurationService: ConfigurationService
+        private val configurationService: ConfigurationService,
+        private val charsetDetectionService: CharsetDetectionService
 ) : FileService {
 
     @Value("\${up.max-file-size}")
@@ -55,6 +56,11 @@ class FileServiceImpl(
             size: Long,
             user: User?
     ): UploadCompleteResponseDTO {
+        val actualContentType = when {
+            contentType.isBlank() -> "application/octet-stream"
+            contentType == "text/plain" -> "text/plain; charset=" + charsetDetectionService.detect(tmpFile)
+            else -> contentType
+        }
         val personalLimit: Long = user?.maxTemporaryFileSize?.value
                 ?: configurationService.getConfigurationOption(ANONYMOUS_MAX_FILE_SIZE).toLong()
 
@@ -67,7 +73,7 @@ class FileServiceImpl(
         val fileEntry = FileEntry(0,
                 key,
                 filename,
-                contentType.let { if (contentType.isBlank()) "application/octet-stream" else contentType },
+                actualContentType,
                 null,
                 false,
                 Timestamp.valueOf(LocalDateTime.now()),

spring-app/src/test/kotlin/pl/starchasers/up/controller/UploadControllerTest.kt

@@ -52,7 +52,7 @@ internal class UploadControllerTest : MockMvcTestBase() {
         fun `Given valid request, should upload and store file`() {
             val exampleTextFile = MockMultipartFile("file",
                     "exampleTextFile.txt",
-                    "text/plain",
+                    "text/plain; charset=UTF-8",
                     "example content".toByteArray())
 
             mockMvc.multipart(path = uploadFileRequestPath,
@@ -68,7 +68,7 @@ internal class UploadControllerTest : MockMvcTestBase() {
             }
 
             fileEntryRepository.findAll()[0].let { fileEntry ->
-                assertEquals("text/plain", fileEntry.contentType)
+                assertEquals("text/plain; charset=UTF-8", fileEntry.contentType)
                 assertEquals(false, fileEntry.encrypted)
                 assertEquals("exampleTextFile.txt", fileEntry.filename)
                 assertEquals(null, fileEntry.password)
@@ -158,20 +158,22 @@ internal class UploadControllerTest : MockMvcTestBase() {
     ) : MockMvcTestBase() {
         private val content = "example content"
 
+        private fun createFile(contentType: String, fileContent:String=content): String = fileService.createFile(
+                fileContent.byteInputStream(),
+                "fileName.txt",
+                contentType,
+                fileContent.byteInputStream().readAllBytes().size.toLong(),
+                null
+        ).key
+
         @Test
         @DocumentResponse
         fun `Given valid key, should return raw file`() {
-            val key = fileService.createFile(
-                    content.byteInputStream(),
-                    "fileName.txt",
-                    "text/plain",
-                    content.byteInputStream().readAllBytes().size.toLong(),
-                    null
-            ).key
+            val key = createFile("application/octet-stream")
 
             mockMvc.get(path = Path("/u/$key")) {
                 responseJsonPath("$").equalsValue("example content")
-                responseHeader(HttpHeaders.CONTENT_TYPE).equals("text/plain")
+                responseHeader(HttpHeaders.CONTENT_TYPE).equals("application/octet-stream")
                 responseHeader(HttpHeaders.CONTENT_LENGTH).equals("${content.length}")
             }
 
@@ -180,13 +182,7 @@ internal class UploadControllerTest : MockMvcTestBase() {
         @Test
         fun `Given valid Range header, should return 206`() {
             val contentSize = content.byteInputStream().readAllBytes().size.toLong()
-            val key = fileService.createFile(
-                    content.byteInputStream(),
-                    "fileName.txt",
-                    "text/plain",
-                    contentSize,
-                    null
-            ).key
+            val key = createFile("text/plain")
 
             val headers = HttpHeaders()
             headers.set(HttpHeaders.RANGE, "bytes=0-")
@@ -201,12 +197,7 @@ internal class UploadControllerTest : MockMvcTestBase() {
         @Test
         fun `Given invalid Range header, should return 200`() {
             val contentSize = content.byteInputStream().readAllBytes().size.toLong()
-            val key = fileService.createFile(
-                    content.byteInputStream(),
-                    "fileName.txt",
-                    "text/plain",
-                    contentSize,
-                    null).key
+            val key = createFile("text/plain")
 
             val headers = HttpHeaders()
             headers.set(HttpHeaders.RANGE, "mb=-1024")
@@ -222,6 +213,29 @@ internal class UploadControllerTest : MockMvcTestBase() {
                 isError(HttpStatus.NOT_FOUND)
             }
         }
+
+        @Test
+        fun `Given unspecified text file encoding, should guess based on content`() {
+            val key = createFile("text/plain", fileContent = "Ā ā Ă অ আ ই ঈ উ")
+
+            mockMvc.get(path = Path("/u/$key")) {
+                isSuccess()
+                responseJsonPath("$").equalsValue("Ā ā Ă অ আ ই ঈ উ")
+                responseHeader("Content-Type").equals("text/plain; charset=UTF-8")
+            }
+        }
+
+        @Test
+        fun `Given specified text file encoding, should preserve it`() {
+            val contentType = "text/plain; charset=us-ascii"
+            val key = createFile(contentType)
+
+            mockMvc.get(path = Path("/u/$key")) {
+                isSuccess()
+                responseJsonPath("$").equalsValue("example content")
+                responseHeader("Content-Type").equals(contentType)
+            }
+        }
     }
 
     @Transactional
@@ -300,15 +314,14 @@ internal class UploadControllerTest : MockMvcTestBase() {
     @Nested
     @TestInstance(TestInstance.Lifecycle.PER_CLASS)
     inner class GetFileDetails(
-            @Autowired val fileService: FileService,
-            @Autowired val fileEntryRepository: FileEntryRepository
+            @Autowired val fileService: FileService
     ) : MockMvcTestBase() {
 
         private fun getRequestPath(key: String): Path = Path("/api/u/$key/details")
         private val content = "example content"
         private lateinit var fileKey: String
         private val filename: String = "filename.txt"
-        private val contentType: String = "text/plain"
+        private val contentType: String = "text/plain; charset=UTF-8"
 
         @BeforeAll
         fun setup() {