From 0a00d7b0d98196f9c779b9017165be069aa77e37 Mon Sep 17 00:00:00 2001 From: Guilhem Bonnefille Date: Thu, 6 Feb 2020 19:18:46 +0100 Subject: [PATCH 1/6] Add support for fixed ContentLength Some servers do not support chunked message. Sending whole *part* in a single message would help to interact with such servers. --- .../java/io/tus/java/client/TusUploader.java | 50 +++++++++++++++++-- .../io/tus/java/client/TestTusUploader.java | 46 +++++++++++++++++ 2 files changed, 92 insertions(+), 4 deletions(-) diff --git a/src/main/java/io/tus/java/client/TusUploader.java b/src/main/java/io/tus/java/client/TusUploader.java index 282d18e..016e437 100644 --- a/src/main/java/io/tus/java/client/TusUploader.java +++ b/src/main/java/io/tus/java/client/TusUploader.java @@ -54,7 +54,7 @@ public TusUploader(TusClient client, TusUpload upload, URL uploadURL, TusInputSt setChunkSize(2 * 1024 * 1024); } - private void openConnection() throws IOException, ProtocolException { + private void openConnection(boolean chunked) throws IOException, ProtocolException { // Only open a connection, if we have none open. if(connection != null) { return; @@ -77,7 +77,8 @@ private void openConnection() throws IOException, ProtocolException { } connection.setDoOutput(true); - connection.setChunkedStreamingMode(0); + if (chunked) + connection.setChunkedStreamingMode(0); try { output = connection.getOutputStream(); } catch(java.net.ProtocolException pe) { @@ -159,6 +160,47 @@ public int getRequestPayloadSize() { return requestPayloadSize; } + /** + * Upload a part of the file by reading a chunk from the InputStream and writing + * it to the HTTP request's body. If the number of available bytes is lower than the chunk's + * size, all available bytes will be uploaded and nothing more. + * The size of the read chunk can be obtained using {@link #getRequestPayloadSize()} and changed + * using {@link #setRequestPayloadSize(int)}. + * In order to obtain the new offset, use {@link #getOffset()} after this method returns. + * + * @return Number of bytes read and written. + * @throws IOException Thrown if an exception occurs while reading from the source or writing + * to the HTTP request. + */ + public int upload() throws IOException, ProtocolException { + openConnection(false); + + setChunkSize(bytesRemainingForRequest); + + int bytesRead = input.read(buffer, bytesRemainingForRequest); + if(bytesRead == -1) { + // No bytes were read since the input stream is empty + return -1; + } + + // FIXME connection.setRequestProperty("Content-Length", Long.toString(bytesRead)); + + // Do not write the entire buffer to the stream since the array will + // be filled up with 0x00s if the number of read bytes is lower then + // the chunk's size. + output.write(buffer, 0, bytesRead); + output.flush(); + + offset += bytesRead; + bytesRemainingForRequest -= bytesRead; + + if(bytesRemainingForRequest <= 0) { + finishConnection(); + } + + return bytesRead; + } + /** * Upload a part of the file by reading a chunk from the InputStream and writing * it to the HTTP request's body. If the number of available bytes is lower than the chunk's @@ -174,7 +216,7 @@ public int getRequestPayloadSize() { * to the HTTP request. */ public int uploadChunk() throws IOException, ProtocolException { - openConnection(); + openConnection(true); int bytesToRead = Math.min(getChunkSize(), bytesRemainingForRequest); @@ -226,7 +268,7 @@ public int uploadChunk() throws IOException, ProtocolException { * to the HTTP request. */ @Deprecated public int uploadChunk(int chunkSize) throws IOException, ProtocolException { - openConnection(); + openConnection(true); byte[] buf = new byte[chunkSize]; int bytesRead = input.read(buf, chunkSize); diff --git a/src/test/java/io/tus/java/client/TestTusUploader.java b/src/test/java/io/tus/java/client/TestTusUploader.java index e3c5f5f..6f7807b 100644 --- a/src/test/java/io/tus/java/client/TestTusUploader.java +++ b/src/test/java/io/tus/java/client/TestTusUploader.java @@ -63,6 +63,52 @@ public void testTusUploader() throws IOException, ProtocolException { uploader.finish(); } + @Test + public void testTusUploaderFixedLength() throws IOException, ProtocolException { + byte[] content = "hello world".getBytes(); + + mockServer.when(new HttpRequest() + .withPath("/files/fixed") + .withHeader("Tus-Resumable", TusClient.TUS_VERSION) + .withHeader("Upload-Offset", "5") + .withHeader("Content-Type", "application/offset+octet-stream") + .withHeader(isOpenJDK6 ? "": "Expect: 100-continue") + .withBody(Arrays.copyOfRange(content, 5, 10))) + .respond(new HttpResponse() + .withStatusCode(204) + .withHeader("Tus-Resumable", TusClient.TUS_VERSION) + .withHeader("Upload-Offset", "10")); + mockServer.when(new HttpRequest() + .withPath("/files/fixed") + .withHeader("Tus-Resumable", TusClient.TUS_VERSION) + .withHeader("Upload-Offset", "10") + .withHeader("Content-Type", "application/offset+octet-stream") + .withHeader(isOpenJDK6 ? "": "Expect: 100-continue") + .withBody(Arrays.copyOfRange(content, 10, 11))) + .respond(new HttpResponse() + .withStatusCode(204) + .withHeader("Tus-Resumable", TusClient.TUS_VERSION) + .withHeader("Upload-Offset", "11")); + + TusClient client = new TusClient(); + URL uploadUrl = new URL(mockServerURL + "/fixed"); + TusInputStream input = new TusInputStream(new ByteArrayInputStream(content)); + long offset = 5; + + TusUpload upload = new TusUpload(); + + TusUploader uploader = new TusUploader(client, upload, uploadUrl, input, offset); + + uploader.setRequestPayloadSize(5); + assertEquals(uploader.getRequestPayloadSize(), 5); + + assertEquals(5, uploader.upload()); + assertEquals(1, uploader.upload()); + assertEquals(-1, uploader.upload()); + assertEquals(11, uploader.getOffset()); + uploader.finish(); + } + @Test public void testTusUploaderClientUploadFinishedCalled() throws IOException, ProtocolException { From 9c518a4267b0965dd7e91e02562a790e9625123b Mon Sep 17 00:00:00 2001 From: Guilhem Bonnefille Date: Mon, 10 Feb 2020 13:33:56 +0100 Subject: [PATCH 2/6] Remove old FIXME The Java HTTP client library is able to automatically set the Content-Length. --- src/main/java/io/tus/java/client/TusUploader.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/io/tus/java/client/TusUploader.java b/src/main/java/io/tus/java/client/TusUploader.java index 016e437..c974966 100644 --- a/src/main/java/io/tus/java/client/TusUploader.java +++ b/src/main/java/io/tus/java/client/TusUploader.java @@ -183,8 +183,6 @@ public int upload() throws IOException, ProtocolException { return -1; } - // FIXME connection.setRequestProperty("Content-Length", Long.toString(bytesRead)); - // Do not write the entire buffer to the stream since the array will // be filled up with 0x00s if the number of read bytes is lower then // the chunk's size. From a23d56ae901cb5f6ec869797a2dc614019249f6e Mon Sep 17 00:00:00 2001 From: Guilhem Bonnefille Date: Wed, 12 Feb 2020 12:32:23 +0100 Subject: [PATCH 3/6] Control for chunked transfer encoding from client --- .../java/io/tus/java/client/TusClient.java | 34 +++++++++++ .../java/io/tus/java/client/TusUploader.java | 61 +++++-------------- .../io/tus/java/client/TestTusUploader.java | 11 ++-- 3 files changed, 55 insertions(+), 51 deletions(-) diff --git a/src/main/java/io/tus/java/client/TusClient.java b/src/main/java/io/tus/java/client/TusClient.java index 3bda521..8d29174 100644 --- a/src/main/java/io/tus/java/client/TusClient.java +++ b/src/main/java/io/tus/java/client/TusClient.java @@ -21,6 +21,7 @@ public class TusClient { private URL uploadCreationURL; private boolean resumingEnabled; private boolean removeFingerprintOnSuccessEnabled; + private boolean chunkedTransferEncoding = true; private TusURLStore urlStore; private Map headers; private int connectTimeout = 5000; @@ -115,6 +116,39 @@ public boolean removeFingerprintOnSuccessEnabled() { return removeFingerprintOnSuccessEnabled; } + /** + * Enable chunked transfer encoding. + * + * @see https://en.wikipedia.org/wiki/Chunked_transfer_encoding + * @see #disableChunkedTransferEncoding() + */ + public void enableChunkedTransferEncoding() { + chunkedTransferEncoding = true; + } + + /** + * Disable chunked transfer encoding. + * + * @see https://en.wikipedia.org/wiki/Chunked_transfer_encoding + * @see #enableChunkedTransferEncoding() + */ + public void disableChunkedTransferEncoding() { + chunkedTransferEncoding = false; + } + + /** + * Get the current status of chunked transfer encoding. + * + * @see https://en.wikipedia.org/wiki/Chunked_transfer_encoding + * @see #enableChunkedTransferEncoding() + * @see #disableChunkedTransferEncoding() + * + * @return True if chunked transfer encoding has been enabled. + */ + public boolean chunkedTransferEncoding() { + return chunkedTransferEncoding; + } + /** * Set headers which will be added to every HTTP requestes made by this TusClient instance. diff --git a/src/main/java/io/tus/java/client/TusUploader.java b/src/main/java/io/tus/java/client/TusUploader.java index c974966..968b3c3 100644 --- a/src/main/java/io/tus/java/client/TusUploader.java +++ b/src/main/java/io/tus/java/client/TusUploader.java @@ -54,7 +54,7 @@ public TusUploader(TusClient client, TusUpload upload, URL uploadURL, TusInputSt setChunkSize(2 * 1024 * 1024); } - private void openConnection(boolean chunked) throws IOException, ProtocolException { + private void openConnection() throws IOException, ProtocolException { // Only open a connection, if we have none open. if(connection != null) { return; @@ -77,8 +77,8 @@ private void openConnection(boolean chunked) throws IOException, ProtocolExcepti } connection.setDoOutput(true); - if (chunked) - connection.setChunkedStreamingMode(0); + if (client.chunkedTransferEncoding()) + connection.setChunkedStreamingMode(0); try { output = connection.getOutputStream(); } catch(java.net.ProtocolException pe) { @@ -160,45 +160,6 @@ public int getRequestPayloadSize() { return requestPayloadSize; } - /** - * Upload a part of the file by reading a chunk from the InputStream and writing - * it to the HTTP request's body. If the number of available bytes is lower than the chunk's - * size, all available bytes will be uploaded and nothing more. - * The size of the read chunk can be obtained using {@link #getRequestPayloadSize()} and changed - * using {@link #setRequestPayloadSize(int)}. - * In order to obtain the new offset, use {@link #getOffset()} after this method returns. - * - * @return Number of bytes read and written. - * @throws IOException Thrown if an exception occurs while reading from the source or writing - * to the HTTP request. - */ - public int upload() throws IOException, ProtocolException { - openConnection(false); - - setChunkSize(bytesRemainingForRequest); - - int bytesRead = input.read(buffer, bytesRemainingForRequest); - if(bytesRead == -1) { - // No bytes were read since the input stream is empty - return -1; - } - - // Do not write the entire buffer to the stream since the array will - // be filled up with 0x00s if the number of read bytes is lower then - // the chunk's size. - output.write(buffer, 0, bytesRead); - output.flush(); - - offset += bytesRead; - bytesRemainingForRequest -= bytesRead; - - if(bytesRemainingForRequest <= 0) { - finishConnection(); - } - - return bytesRead; - } - /** * Upload a part of the file by reading a chunk from the InputStream and writing * it to the HTTP request's body. If the number of available bytes is lower than the chunk's @@ -214,9 +175,17 @@ public int upload() throws IOException, ProtocolException { * to the HTTP request. */ public int uploadChunk() throws IOException, ProtocolException { - openConnection(true); - - int bytesToRead = Math.min(getChunkSize(), bytesRemainingForRequest); + openConnection(); + + final int bytesToRead; + if (client.chunkedTransferEncoding()) + bytesToRead = Math.min(getChunkSize(), bytesRemainingForRequest); + else { + // No chunked tranfer encoding, so we upload a single chunk. + bytesToRead = getChunkSize(); + // The connection should be closed after single write. + bytesRemainingForRequest = 0; + } int bytesRead = input.read(buffer, bytesToRead); if(bytesRead == -1) { @@ -266,7 +235,7 @@ public int uploadChunk() throws IOException, ProtocolException { * to the HTTP request. */ @Deprecated public int uploadChunk(int chunkSize) throws IOException, ProtocolException { - openConnection(true); + openConnection(); byte[] buf = new byte[chunkSize]; int bytesRead = input.read(buf, chunkSize); diff --git a/src/test/java/io/tus/java/client/TestTusUploader.java b/src/test/java/io/tus/java/client/TestTusUploader.java index 6f7807b..2a53415 100644 --- a/src/test/java/io/tus/java/client/TestTusUploader.java +++ b/src/test/java/io/tus/java/client/TestTusUploader.java @@ -91,6 +91,7 @@ public void testTusUploaderFixedLength() throws IOException, ProtocolException { .withHeader("Upload-Offset", "11")); TusClient client = new TusClient(); + client.disableChunkedTransferEncoding(); URL uploadUrl = new URL(mockServerURL + "/fixed"); TusInputStream input = new TusInputStream(new ByteArrayInputStream(content)); long offset = 5; @@ -99,12 +100,12 @@ public void testTusUploaderFixedLength() throws IOException, ProtocolException { TusUploader uploader = new TusUploader(client, upload, uploadUrl, input, offset); - uploader.setRequestPayloadSize(5); - assertEquals(uploader.getRequestPayloadSize(), 5); + uploader.setChunkSize(5); + assertEquals(uploader.getChunkSize(), 5); - assertEquals(5, uploader.upload()); - assertEquals(1, uploader.upload()); - assertEquals(-1, uploader.upload()); + assertEquals(5, uploader.uploadChunk()); + assertEquals(1, uploader.uploadChunk()); + assertEquals(-1, uploader.uploadChunk()); assertEquals(11, uploader.getOffset()); uploader.finish(); } From a5754cf7c46853e242f7df699a2e31b2280ff575 Mon Sep 17 00:00:00 2001 From: Guilhem Bonnefille Date: Wed, 12 Feb 2020 12:33:32 +0100 Subject: [PATCH 4/6] Add a finalization reply --- src/test/java/io/tus/java/client/TestTusUploader.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/test/java/io/tus/java/client/TestTusUploader.java b/src/test/java/io/tus/java/client/TestTusUploader.java index 2a53415..e15692b 100644 --- a/src/test/java/io/tus/java/client/TestTusUploader.java +++ b/src/test/java/io/tus/java/client/TestTusUploader.java @@ -89,6 +89,16 @@ public void testTusUploaderFixedLength() throws IOException, ProtocolException { .withStatusCode(204) .withHeader("Tus-Resumable", TusClient.TUS_VERSION) .withHeader("Upload-Offset", "11")); + mockServer.when(new HttpRequest() + .withPath("/files/fixed") + .withHeader("Tus-Resumable", TusClient.TUS_VERSION) + .withHeader("Upload-Offset", "11") + .withHeader("Content-Type", "application/offset+octet-stream") + .withHeader(isOpenJDK6 ? "": "Expect: 100-continue")) + .respond(new HttpResponse() + .withStatusCode(204) + .withHeader("Tus-Resumable", TusClient.TUS_VERSION) + .withHeader("Upload-Offset", "11")); TusClient client = new TusClient(); client.disableChunkedTransferEncoding(); From 9940d3c89b1e5c76d6bf03887ef73ba7641910cf Mon Sep 17 00:00:00 2001 From: Guilhem Bonnefille Date: Wed, 6 May 2020 17:17:16 +0200 Subject: [PATCH 5/6] Fix naming convention --- src/main/java/io/tus/java/client/TusClient.java | 10 +++++----- src/main/java/io/tus/java/client/TusUploader.java | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/io/tus/java/client/TusClient.java b/src/main/java/io/tus/java/client/TusClient.java index 8d29174..1b23e76 100644 --- a/src/main/java/io/tus/java/client/TusClient.java +++ b/src/main/java/io/tus/java/client/TusClient.java @@ -21,7 +21,7 @@ public class TusClient { private URL uploadCreationURL; private boolean resumingEnabled; private boolean removeFingerprintOnSuccessEnabled; - private boolean chunkedTransferEncoding = true; + private boolean chunkedTransferEncodingEnabled = true; private TusURLStore urlStore; private Map headers; private int connectTimeout = 5000; @@ -123,7 +123,7 @@ public boolean removeFingerprintOnSuccessEnabled() { * @see #disableChunkedTransferEncoding() */ public void enableChunkedTransferEncoding() { - chunkedTransferEncoding = true; + chunkedTransferEncodingEnabled = true; } /** @@ -133,7 +133,7 @@ public void enableChunkedTransferEncoding() { * @see #enableChunkedTransferEncoding() */ public void disableChunkedTransferEncoding() { - chunkedTransferEncoding = false; + chunkedTransferEncodingEnabled = false; } /** @@ -145,8 +145,8 @@ public void disableChunkedTransferEncoding() { * * @return True if chunked transfer encoding has been enabled. */ - public boolean chunkedTransferEncoding() { - return chunkedTransferEncoding; + public boolean chunkedTransferEncodingEnabled() { + return chunkedTransferEncodingEnabled; } diff --git a/src/main/java/io/tus/java/client/TusUploader.java b/src/main/java/io/tus/java/client/TusUploader.java index 968b3c3..ef6d447 100644 --- a/src/main/java/io/tus/java/client/TusUploader.java +++ b/src/main/java/io/tus/java/client/TusUploader.java @@ -77,7 +77,7 @@ private void openConnection() throws IOException, ProtocolException { } connection.setDoOutput(true); - if (client.chunkedTransferEncoding()) + if (client.chunkedTransferEncodingEnabled()) connection.setChunkedStreamingMode(0); try { output = connection.getOutputStream(); @@ -178,7 +178,7 @@ public int uploadChunk() throws IOException, ProtocolException { openConnection(); final int bytesToRead; - if (client.chunkedTransferEncoding()) + if (client.chunkedTransferEncodingEnabled()) bytesToRead = Math.min(getChunkSize(), bytesRemainingForRequest); else { // No chunked tranfer encoding, so we upload a single chunk. From 9daad1e74be78703c1b43de21b70b64466bc469d Mon Sep 17 00:00:00 2001 From: Guilhem Bonnefille Date: Wed, 6 May 2020 17:21:38 +0200 Subject: [PATCH 6/6] Fix code convention --- src/main/java/io/tus/java/client/TusUploader.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/io/tus/java/client/TusUploader.java b/src/main/java/io/tus/java/client/TusUploader.java index ef6d447..973d250 100644 --- a/src/main/java/io/tus/java/client/TusUploader.java +++ b/src/main/java/io/tus/java/client/TusUploader.java @@ -77,8 +77,9 @@ private void openConnection() throws IOException, ProtocolException { } connection.setDoOutput(true); - if (client.chunkedTransferEncodingEnabled()) + if (client.chunkedTransferEncodingEnabled()) { connection.setChunkedStreamingMode(0); + } try { output = connection.getOutputStream(); } catch(java.net.ProtocolException pe) { @@ -178,9 +179,9 @@ public int uploadChunk() throws IOException, ProtocolException { openConnection(); final int bytesToRead; - if (client.chunkedTransferEncodingEnabled()) + if (client.chunkedTransferEncodingEnabled()) { bytesToRead = Math.min(getChunkSize(), bytesRemainingForRequest); - else { + } else { // No chunked tranfer encoding, so we upload a single chunk. bytesToRead = getChunkSize(); // The connection should be closed after single write.