From 8f6aa8bb5c476f4dff260b0b1b108c6d4a809bfa Mon Sep 17 00:00:00 2001 From: Hubert Balcerzak Date: Fri, 19 Jun 2020 03:09:17 +0200 Subject: [PATCH 1/4] fix encoding for pastes --- gatsby-app/src/components/useFileUpload.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gatsby-app/src/components/useFileUpload.js b/gatsby-app/src/components/useFileUpload.js index b6208880..274c45ce 100644 --- a/gatsby-app/src/components/useFileUpload.js +++ b/gatsby-app/src/components/useFileUpload.js @@ -63,7 +63,7 @@ const useFileUpload = () => { data.append('file', blob) } else { const file = new File([event.clipboardData.getData('text')], 'paste.txt', { - type: 'text/plain', + type: 'text/plain; charset=utf-8', }) data.append('file', file) } From ac0a78cbf2ad4f1d47e4f8907632031e5b595747 Mon Sep 17 00:00:00 2001 From: Hubert Balcerzak Date: Fri, 19 Jun 2020 16:08:20 +0200 Subject: [PATCH 2/4] default utf-8 encoding for text files --- .../controller/AnonymousUploadController.kt | 3 +- .../AnonymousUploadControllerTest.kt | 55 ++++++++++++------- 2 files changed, 38 insertions(+), 20 deletions(-) diff --git a/spring-app/src/main/kotlin/pl/starchasers/up/controller/AnonymousUploadController.kt b/spring-app/src/main/kotlin/pl/starchasers/up/controller/AnonymousUploadController.kt index b90735a7..63abf2d1 100644 --- a/spring-app/src/main/kotlin/pl/starchasers/up/controller/AnonymousUploadController.kt +++ b/spring-app/src/main/kotlin/pl/starchasers/up/controller/AnonymousUploadController.kt @@ -52,7 +52,8 @@ class AnonymousUploadController(private val fileStorageService: FileStorageServi @GetMapping("/u/{fileKey}") fun getAnonymousUpload(@PathVariable fileKey: String, request: HttpServletRequest, response: HttpServletResponse) { val (fileEntry, stream) = fileStorageService.getStoredFileRaw(fileKey) - response.contentType = fileEntry.contentType + response.contentType = if (fileEntry.contentType == "text/plain") "text/plain; charset=utf-8" + else fileEntry.contentType response.addHeader("Content-Disposition", ContentDisposition diff --git a/spring-app/src/test/kotlin/pl/starchasers/up/controller/AnonymousUploadControllerTest.kt b/spring-app/src/test/kotlin/pl/starchasers/up/controller/AnonymousUploadControllerTest.kt index 50ca62b8..d629b267 100644 --- a/spring-app/src/test/kotlin/pl/starchasers/up/controller/AnonymousUploadControllerTest.kt +++ b/spring-app/src/test/kotlin/pl/starchasers/up/controller/AnonymousUploadControllerTest.kt @@ -123,19 +123,23 @@ internal class AnonymousUploadControllerTest() : MockMvcTestBase() { ) : MockMvcTestBase() { private val content = "example content" - @Test - @DocumentResponse - fun `Given valid key, should return raw file`() { - val key = fileService.createFile( + private fun createFile(contentType: String = "text/plain; charset=utf-8"): String { + return fileService.createFile( content.byteInputStream(), "fileName.txt", - "text/plain", + contentType, content.byteInputStream().readAllBytes().size.toLong() ).key + } + + @Test + @DocumentResponse + fun `Given valid key, should return raw file`() { + val key = createFile() mockMvc.get(path = Path("/u/$key")) { responseJsonPath("$").equalsValue("example content") - responseHeader("Content-Type").equals("text/plain") + responseHeader("Content-Type").equals("text/plain; charset=utf-8") } } @@ -143,19 +147,14 @@ internal class AnonymousUploadControllerTest() : MockMvcTestBase() { @Test fun `Given valid Range header, should return 206`() { val contentSize = content.byteInputStream().readAllBytes().size.toLong() - val key = fileService.createFile( - content.byteInputStream(), - "fileName.txt", - "text/plain", - contentSize - ).key + val key = createFile() val headers = HttpHeaders() headers.set(HttpHeaders.RANGE, "bytes=0-") mockMvc.get(path = Path("/u/$key"), headers = headers) { status(HttpStatus.PARTIAL_CONTENT) - responseHeader(HttpHeaders.CONTENT_RANGE).equals("bytes 0-${contentSize-1}/$contentSize") + responseHeader(HttpHeaders.CONTENT_RANGE).equals("bytes 0-${contentSize - 1}/$contentSize") responseHeader(HttpHeaders.CONTENT_LENGTH).equals("$contentSize") } } @@ -163,12 +162,7 @@ internal class AnonymousUploadControllerTest() : MockMvcTestBase() { @Test fun `Given invalid Range header, should return 200`() { val contentSize = content.byteInputStream().readAllBytes().size.toLong() - val key = fileService.createFile( - content.byteInputStream(), - "fileName.txt", - "text/plain", - contentSize - ).key + val key = createFile() val headers = HttpHeaders() headers.set(HttpHeaders.RANGE, "mb=-1024") @@ -184,6 +178,29 @@ internal class AnonymousUploadControllerTest() : MockMvcTestBase() { isError(HttpStatus.NOT_FOUND) } } + + @Test + fun `Given unspecified text file encoding, should default to utf-8`() { + val key = createFile("text/plain") + + mockMvc.get(path = Path("/u/$key")) { + isSuccess() + responseJsonPath("$").equalsValue("example content") + responseHeader("Content-Type").equals("text/plain; charset=utf-8") + } + } + + @Test + fun `Given specified text file encoding, should preserve it`() { + val contentType = "text/plain; charset=us-ascii" + val key = createFile(contentType) + + mockMvc.get(path = Path("/u/$key")) { + isSuccess() + responseJsonPath("$").equalsValue("example content") + responseHeader("Content-Type").equals(contentType) + } + } } @Transactional From 6b381dec98c335cb2cb653806ee0320670ca38b4 Mon Sep 17 00:00:00 2001 From: Hubert Balcerzak Date: Fri, 16 Oct 2020 02:53:13 +0200 Subject: [PATCH 3/4] charset guessing --- spring-app/build.gradle.kts | 2 ++ .../up/service/CharsetDetectionService.kt | 20 ++++++++++++ .../pl/starchasers/up/service/FileService.kt | 10 ++++-- .../up/controller/UploadControllerTest.kt | 31 +++++++------------ 4 files changed, 41 insertions(+), 22 deletions(-) create mode 100644 spring-app/src/main/kotlin/pl/starchasers/up/service/CharsetDetectionService.kt diff --git a/spring-app/build.gradle.kts b/spring-app/build.gradle.kts index 8ed34f38..1a496156 100644 --- a/spring-app/build.gradle.kts +++ b/spring-app/build.gradle.kts @@ -30,6 +30,8 @@ dependencies { implementation("commons-fileupload:commons-fileupload:1.3.3") implementation("io.jsonwebtoken:jjwt:0.9.1") runtimeOnly("com.h2database:h2:1.4.200") + implementation("com.ibm.icu:icu4j:67.1") + runtimeOnly("com.h2database:h2:1.4.200") runtimeOnly("mysql:mysql-connector-java") runtimeOnly("org.mariadb.jdbc:mariadb-java-client") runtimeOnly(files("../gatsby-app/build/artifact/gatsby-app.jar")) diff --git a/spring-app/src/main/kotlin/pl/starchasers/up/service/CharsetDetectionService.kt b/spring-app/src/main/kotlin/pl/starchasers/up/service/CharsetDetectionService.kt new file mode 100644 index 00000000..a29c62b7 --- /dev/null +++ b/spring-app/src/main/kotlin/pl/starchasers/up/service/CharsetDetectionService.kt @@ -0,0 +1,20 @@ +package pl.starchasers.up.service + +import com.ibm.icu.text.CharsetDetector +import org.springframework.stereotype.Service +import java.io.InputStream + +interface CharsetDetectionService { + fun detect(inputStream: InputStream): String +} + +@Service +class CharsetDetectionServiceImpl : CharsetDetectionService { + override fun detect(inputStream: InputStream): String { + val detector = CharsetDetector() + detector.setDeclaredEncoding("utf-8") + detector.setText(inputStream) + return detector.detect().name + } + +} \ No newline at end of file diff --git a/spring-app/src/main/kotlin/pl/starchasers/up/service/FileService.kt b/spring-app/src/main/kotlin/pl/starchasers/up/service/FileService.kt index c7c9ffe7..aeb12e5c 100644 --- a/spring-app/src/main/kotlin/pl/starchasers/up/service/FileService.kt +++ b/spring-app/src/main/kotlin/pl/starchasers/up/service/FileService.kt @@ -39,7 +39,8 @@ interface FileService { class FileServiceImpl( private val fileStorageService: FileStorageService, private val fileEntryRepository: FileEntryRepository, - private val configurationService: ConfigurationService + private val configurationService: ConfigurationService, + private val charsetDetectionService: CharsetDetectionService ) : FileService { @Value("\${up.max-file-size}") @@ -55,6 +56,11 @@ class FileServiceImpl( size: Long, user: User? ): UploadCompleteResponseDTO { + val actualContentType = when { + contentType.isBlank() -> "application/octet-stream" + contentType == "text/plain" -> "text/plain; charset=" + charsetDetectionService.detect(tmpFile) + else -> contentType + } val personalLimit: Long = user?.maxTemporaryFileSize?.value ?: configurationService.getConfigurationOption(ANONYMOUS_MAX_FILE_SIZE).toLong() @@ -67,7 +73,7 @@ class FileServiceImpl( val fileEntry = FileEntry(0, key, filename, - contentType.let { if (contentType.isBlank()) "application/octet-stream" else contentType }, + actualContentType, null, false, Timestamp.valueOf(LocalDateTime.now()), diff --git a/spring-app/src/test/kotlin/pl/starchasers/up/controller/UploadControllerTest.kt b/spring-app/src/test/kotlin/pl/starchasers/up/controller/UploadControllerTest.kt index be262da6..a97b619b 100644 --- a/spring-app/src/test/kotlin/pl/starchasers/up/controller/UploadControllerTest.kt +++ b/spring-app/src/test/kotlin/pl/starchasers/up/controller/UploadControllerTest.kt @@ -158,16 +158,18 @@ internal class UploadControllerTest : MockMvcTestBase() { ) : MockMvcTestBase() { private val content = "example content" + private fun createFile(contentType: String): String = fileService.createFile( + content.byteInputStream(), + "fileName.txt", + contentType, + content.byteInputStream().readAllBytes().size.toLong(), + null + ).key + @Test @DocumentResponse fun `Given valid key, should return raw file`() { - val key = fileService.createFile( - content.byteInputStream(), - "fileName.txt", - "text/plain", - content.byteInputStream().readAllBytes().size.toLong(), - null - ).key + val key = createFile("text/plain") mockMvc.get(path = Path("/u/$key")) { responseJsonPath("$").equalsValue("example content") @@ -180,13 +182,7 @@ internal class UploadControllerTest : MockMvcTestBase() { @Test fun `Given valid Range header, should return 206`() { val contentSize = content.byteInputStream().readAllBytes().size.toLong() - val key = fileService.createFile( - content.byteInputStream(), - "fileName.txt", - "text/plain", - contentSize, - null - ).key + val key = createFile("text/plain") val headers = HttpHeaders() headers.set(HttpHeaders.RANGE, "bytes=0-") @@ -201,12 +197,7 @@ internal class UploadControllerTest : MockMvcTestBase() { @Test fun `Given invalid Range header, should return 200`() { val contentSize = content.byteInputStream().readAllBytes().size.toLong() - val key = fileService.createFile( - content.byteInputStream(), - "fileName.txt", - "text/plain", - contentSize, - null).key + val key = createFile("text/plain") val headers = HttpHeaders() headers.set(HttpHeaders.RANGE, "mb=-1024") From 3691fc8ecb4ef786cc271fa8294987f4c2de8cb6 Mon Sep 17 00:00:00 2001 From: Hubert Balcerzak Date: Fri, 16 Oct 2020 03:11:01 +0200 Subject: [PATCH 4/4] test adjustments --- .../up/controller/UploadController.kt | 3 +-- .../up/controller/UploadControllerTest.kt | 27 +++++++++---------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/spring-app/src/main/kotlin/pl/starchasers/up/controller/UploadController.kt b/spring-app/src/main/kotlin/pl/starchasers/up/controller/UploadController.kt index 6502db54..e76fde3b 100644 --- a/spring-app/src/main/kotlin/pl/starchasers/up/controller/UploadController.kt +++ b/spring-app/src/main/kotlin/pl/starchasers/up/controller/UploadController.kt @@ -58,8 +58,7 @@ class UploadController(private val fileStorageService: FileStorageService, @GetMapping("/u/{fileKey}") fun getAnonymousUpload(@PathVariable fileKey: String, request: HttpServletRequest, response: HttpServletResponse) { val (fileEntry, stream) = fileStorageService.getStoredFileRaw(fileKey) - response.contentType = if (fileEntry.contentType == "text/plain") "text/plain; charset=utf-8" - else fileEntry.contentType + response.contentType = fileEntry.contentType response.addHeader(HttpHeaders.CONTENT_DISPOSITION, ContentDisposition diff --git a/spring-app/src/test/kotlin/pl/starchasers/up/controller/UploadControllerTest.kt b/spring-app/src/test/kotlin/pl/starchasers/up/controller/UploadControllerTest.kt index a97b619b..6f07be92 100644 --- a/spring-app/src/test/kotlin/pl/starchasers/up/controller/UploadControllerTest.kt +++ b/spring-app/src/test/kotlin/pl/starchasers/up/controller/UploadControllerTest.kt @@ -52,7 +52,7 @@ internal class UploadControllerTest : MockMvcTestBase() { fun `Given valid request, should upload and store file`() { val exampleTextFile = MockMultipartFile("file", "exampleTextFile.txt", - "text/plain", + "text/plain; charset=UTF-8", "example content".toByteArray()) mockMvc.multipart(path = uploadFileRequestPath, @@ -68,7 +68,7 @@ internal class UploadControllerTest : MockMvcTestBase() { } fileEntryRepository.findAll()[0].let { fileEntry -> - assertEquals("text/plain", fileEntry.contentType) + assertEquals("text/plain; charset=UTF-8", fileEntry.contentType) assertEquals(false, fileEntry.encrypted) assertEquals("exampleTextFile.txt", fileEntry.filename) assertEquals(null, fileEntry.password) @@ -158,22 +158,22 @@ internal class UploadControllerTest : MockMvcTestBase() { ) : MockMvcTestBase() { private val content = "example content" - private fun createFile(contentType: String): String = fileService.createFile( - content.byteInputStream(), + private fun createFile(contentType: String, fileContent:String=content): String = fileService.createFile( + fileContent.byteInputStream(), "fileName.txt", contentType, - content.byteInputStream().readAllBytes().size.toLong(), + fileContent.byteInputStream().readAllBytes().size.toLong(), null ).key @Test @DocumentResponse fun `Given valid key, should return raw file`() { - val key = createFile("text/plain") + val key = createFile("application/octet-stream") mockMvc.get(path = Path("/u/$key")) { responseJsonPath("$").equalsValue("example content") - responseHeader(HttpHeaders.CONTENT_TYPE).equals("text/plain; charset=utf-8") + responseHeader(HttpHeaders.CONTENT_TYPE).equals("application/octet-stream") responseHeader(HttpHeaders.CONTENT_LENGTH).equals("${content.length}") } @@ -215,13 +215,13 @@ internal class UploadControllerTest : MockMvcTestBase() { } @Test - fun `Given unspecified text file encoding, should default to utf-8`() { - val key = createFile("text/plain") + fun `Given unspecified text file encoding, should guess based on content`() { + val key = createFile("text/plain", fileContent = "Ā ā Ă অ আ ই ঈ উ") mockMvc.get(path = Path("/u/$key")) { isSuccess() - responseJsonPath("$").equalsValue("example content") - responseHeader("Content-Type").equals("text/plain; charset=utf-8") + responseJsonPath("$").equalsValue("Ā ā Ă অ আ ই ঈ উ") + responseHeader("Content-Type").equals("text/plain; charset=UTF-8") } } @@ -314,15 +314,14 @@ internal class UploadControllerTest : MockMvcTestBase() { @Nested @TestInstance(TestInstance.Lifecycle.PER_CLASS) inner class GetFileDetails( - @Autowired val fileService: FileService, - @Autowired val fileEntryRepository: FileEntryRepository + @Autowired val fileService: FileService ) : MockMvcTestBase() { private fun getRequestPath(key: String): Path = Path("/api/u/$key/details") private val content = "example content" private lateinit var fileKey: String private val filename: String = "filename.txt" - private val contentType: String = "text/plain" + private val contentType: String = "text/plain; charset=UTF-8" @BeforeAll fun setup() {