diff --git a/gatsby-app/src/components/useFileUpload.js b/gatsby-app/src/components/useFileUpload.js new file mode 100644 index 00000000..e69de29b diff --git a/spring-app/build.gradle.kts b/spring-app/build.gradle.kts index 8ed34f38..1a496156 100644 --- a/spring-app/build.gradle.kts +++ b/spring-app/build.gradle.kts @@ -30,6 +30,8 @@ dependencies { implementation("commons-fileupload:commons-fileupload:1.3.3") implementation("io.jsonwebtoken:jjwt:0.9.1") runtimeOnly("com.h2database:h2:1.4.200") + implementation("com.ibm.icu:icu4j:67.1") + runtimeOnly("com.h2database:h2:1.4.200") runtimeOnly("mysql:mysql-connector-java") runtimeOnly("org.mariadb.jdbc:mariadb-java-client") runtimeOnly(files("../gatsby-app/build/artifact/gatsby-app.jar")) diff --git a/spring-app/src/main/kotlin/pl/starchasers/up/service/CharsetDetectionService.kt b/spring-app/src/main/kotlin/pl/starchasers/up/service/CharsetDetectionService.kt new file mode 100644 index 00000000..a29c62b7 --- /dev/null +++ b/spring-app/src/main/kotlin/pl/starchasers/up/service/CharsetDetectionService.kt @@ -0,0 +1,20 @@ +package pl.starchasers.up.service + +import com.ibm.icu.text.CharsetDetector +import org.springframework.stereotype.Service +import java.io.InputStream + +interface CharsetDetectionService { + fun detect(inputStream: InputStream): String +} + +@Service +class CharsetDetectionServiceImpl : CharsetDetectionService { + override fun detect(inputStream: InputStream): String { + val detector = CharsetDetector() + detector.setDeclaredEncoding("utf-8") + detector.setText(inputStream) + return detector.detect().name + } + +} \ No newline at end of file diff --git a/spring-app/src/main/kotlin/pl/starchasers/up/service/FileService.kt b/spring-app/src/main/kotlin/pl/starchasers/up/service/FileService.kt index c7c9ffe7..aeb12e5c 100644 --- a/spring-app/src/main/kotlin/pl/starchasers/up/service/FileService.kt +++ b/spring-app/src/main/kotlin/pl/starchasers/up/service/FileService.kt @@ -39,7 +39,8 @@ interface FileService { class FileServiceImpl( private val fileStorageService: FileStorageService, private val fileEntryRepository: FileEntryRepository, - private val configurationService: ConfigurationService + private val configurationService: ConfigurationService, + private val charsetDetectionService: CharsetDetectionService ) : FileService { @Value("\${up.max-file-size}") @@ -55,6 +56,11 @@ class FileServiceImpl( size: Long, user: User? ): UploadCompleteResponseDTO { + val actualContentType = when { + contentType.isBlank() -> "application/octet-stream" + contentType == "text/plain" -> "text/plain; charset=" + charsetDetectionService.detect(tmpFile) + else -> contentType + } val personalLimit: Long = user?.maxTemporaryFileSize?.value ?: configurationService.getConfigurationOption(ANONYMOUS_MAX_FILE_SIZE).toLong() @@ -67,7 +73,7 @@ class FileServiceImpl( val fileEntry = FileEntry(0, key, filename, - contentType.let { if (contentType.isBlank()) "application/octet-stream" else contentType }, + actualContentType, null, false, Timestamp.valueOf(LocalDateTime.now()), diff --git a/spring-app/src/test/kotlin/pl/starchasers/up/controller/UploadControllerTest.kt b/spring-app/src/test/kotlin/pl/starchasers/up/controller/UploadControllerTest.kt index 3f460ee4..6f07be92 100644 --- a/spring-app/src/test/kotlin/pl/starchasers/up/controller/UploadControllerTest.kt +++ b/spring-app/src/test/kotlin/pl/starchasers/up/controller/UploadControllerTest.kt @@ -52,7 +52,7 @@ internal class UploadControllerTest : MockMvcTestBase() { fun `Given valid request, should upload and store file`() { val exampleTextFile = MockMultipartFile("file", "exampleTextFile.txt", - "text/plain", + "text/plain; charset=UTF-8", "example content".toByteArray()) mockMvc.multipart(path = uploadFileRequestPath, @@ -68,7 +68,7 @@ internal class UploadControllerTest : MockMvcTestBase() { } fileEntryRepository.findAll()[0].let { fileEntry -> - assertEquals("text/plain", fileEntry.contentType) + assertEquals("text/plain; charset=UTF-8", fileEntry.contentType) assertEquals(false, fileEntry.encrypted) assertEquals("exampleTextFile.txt", fileEntry.filename) assertEquals(null, fileEntry.password) @@ -158,20 +158,22 @@ internal class UploadControllerTest : MockMvcTestBase() { ) : MockMvcTestBase() { private val content = "example content" + private fun createFile(contentType: String, fileContent:String=content): String = fileService.createFile( + fileContent.byteInputStream(), + "fileName.txt", + contentType, + fileContent.byteInputStream().readAllBytes().size.toLong(), + null + ).key + @Test @DocumentResponse fun `Given valid key, should return raw file`() { - val key = fileService.createFile( - content.byteInputStream(), - "fileName.txt", - "text/plain", - content.byteInputStream().readAllBytes().size.toLong(), - null - ).key + val key = createFile("application/octet-stream") mockMvc.get(path = Path("/u/$key")) { responseJsonPath("$").equalsValue("example content") - responseHeader(HttpHeaders.CONTENT_TYPE).equals("text/plain") + responseHeader(HttpHeaders.CONTENT_TYPE).equals("application/octet-stream") responseHeader(HttpHeaders.CONTENT_LENGTH).equals("${content.length}") } @@ -180,13 +182,7 @@ internal class UploadControllerTest : MockMvcTestBase() { @Test fun `Given valid Range header, should return 206`() { val contentSize = content.byteInputStream().readAllBytes().size.toLong() - val key = fileService.createFile( - content.byteInputStream(), - "fileName.txt", - "text/plain", - contentSize, - null - ).key + val key = createFile("text/plain") val headers = HttpHeaders() headers.set(HttpHeaders.RANGE, "bytes=0-") @@ -201,12 +197,7 @@ internal class UploadControllerTest : MockMvcTestBase() { @Test fun `Given invalid Range header, should return 200`() { val contentSize = content.byteInputStream().readAllBytes().size.toLong() - val key = fileService.createFile( - content.byteInputStream(), - "fileName.txt", - "text/plain", - contentSize, - null).key + val key = createFile("text/plain") val headers = HttpHeaders() headers.set(HttpHeaders.RANGE, "mb=-1024") @@ -222,6 +213,29 @@ internal class UploadControllerTest : MockMvcTestBase() { isError(HttpStatus.NOT_FOUND) } } + + @Test + fun `Given unspecified text file encoding, should guess based on content`() { + val key = createFile("text/plain", fileContent = "Ā ā Ă অ আ ই ঈ উ") + + mockMvc.get(path = Path("/u/$key")) { + isSuccess() + responseJsonPath("$").equalsValue("Ā ā Ă অ আ ই ঈ উ") + responseHeader("Content-Type").equals("text/plain; charset=UTF-8") + } + } + + @Test + fun `Given specified text file encoding, should preserve it`() { + val contentType = "text/plain; charset=us-ascii" + val key = createFile(contentType) + + mockMvc.get(path = Path("/u/$key")) { + isSuccess() + responseJsonPath("$").equalsValue("example content") + responseHeader("Content-Type").equals(contentType) + } + } } @Transactional @@ -300,15 +314,14 @@ internal class UploadControllerTest : MockMvcTestBase() { @Nested @TestInstance(TestInstance.Lifecycle.PER_CLASS) inner class GetFileDetails( - @Autowired val fileService: FileService, - @Autowired val fileEntryRepository: FileEntryRepository + @Autowired val fileService: FileService ) : MockMvcTestBase() { private fun getRequestPath(key: String): Path = Path("/api/u/$key/details") private val content = "example content" private lateinit var fileKey: String private val filename: String = "filename.txt" - private val contentType: String = "text/plain" + private val contentType: String = "text/plain; charset=UTF-8" @BeforeAll fun setup() {