From 289a46aea04578b55d80c241f6f14209cbb8eed2 Mon Sep 17 00:00:00 2001 From: Jakob Vogel Date: Thu, 23 Feb 2023 03:14:06 +0100 Subject: [PATCH 1/6] =?UTF-8?q?Reports=20content-length=20of=20streaming?= =?UTF-8?q?=20data=20=F0=9F=93=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This avoids warnings by the AWS client software. #209 SIRI-721 Signed-off-by: Jakob Vogel --- src/test/java/BaseAWSSpec.groovy | 95 +++++++++----------------------- 1 file changed, 25 insertions(+), 70 deletions(-) diff --git a/src/test/java/BaseAWSSpec.groovy b/src/test/java/BaseAWSSpec.groovy index a0a5533..6e1d5c7 100644 --- a/src/test/java/BaseAWSSpec.groovy +++ b/src/test/java/BaseAWSSpec.groovy @@ -8,6 +8,7 @@ import com.amazonaws.HttpMethod import com.amazonaws.services.s3.AmazonS3Client +import com.amazonaws.services.s3.Headers import com.amazonaws.services.s3.model.AmazonS3Exception import com.amazonaws.services.s3.model.DeleteObjectsRequest import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest @@ -126,11 +127,7 @@ abstract class BaseAWSSpec extends BaseSpecification { client.createBucket(bucketName) } and: - client.putObject( - bucketName, - key, - new ByteArrayInputStream("Test".getBytes(Charsets.UTF_8)), - new ObjectMetadata()) + putObjectWithContent(bucketName, key, "Test") def content = new String( ByteStreams.toByteArray(client.getObject(bucketName, key).getObjectContent()), Charsets.UTF_8) @@ -159,16 +156,8 @@ abstract class BaseAWSSpec extends BaseSpecification { } client.createBucket(bucketName) and: - client.putObject( - bucketName, - key1, - new ByteArrayInputStream("Eins".getBytes(Charsets.UTF_8)), - new ObjectMetadata()) - client.putObject( - bucketName, - key2, - new ByteArrayInputStream("Zwei".getBytes(Charsets.UTF_8)), - new ObjectMetadata()) + putObjectWithContent(bucketName, key1, "Eins") + putObjectWithContent(bucketName, key2, "Zwei") then: def listing = client.listObjects(bucketName) def summaries = listing.getObjectSummaries() @@ -191,21 +180,9 @@ abstract class BaseAWSSpec extends BaseSpecification { } client.createBucket(bucketName) and: - client.putObject( - bucketName, - key1, - new ByteArrayInputStream("Eins".getBytes(Charsets.UTF_8)), - new ObjectMetadata()) - client.putObject( - bucketName, - key2, - new ByteArrayInputStream("Zwei".getBytes(Charsets.UTF_8)), - new ObjectMetadata()) - client.putObject( - bucketName, - key3, - new ByteArrayInputStream("Drei".getBytes(Charsets.UTF_8)), - new ObjectMetadata()) + putObjectWithContent(bucketName, key1, "Eins") + putObjectWithContent(bucketName, key2, "Zwei") + putObjectWithContent(bucketName, key3, "Drei") then: def listing = client.listObjects(bucketName, DEFAULT_KEY + '/') def summaries = listing.getObjectSummaries() @@ -224,11 +201,7 @@ abstract class BaseAWSSpec extends BaseSpecification { client.createBucket(bucketName) } and: - client.putObject( - bucketName, - key, - new ByteArrayInputStream("Test".getBytes(Charsets.UTF_8)), - new ObjectMetadata()) + putObjectWithContent(bucketName, key, "Test") client.deleteBucket(bucketName) client.getObject(bucketName, key) then: @@ -340,11 +313,7 @@ abstract class BaseAWSSpec extends BaseSpecification { client.createBucket(bucketName) } and: - client.putObject( - bucketName, - key, - new ByteArrayInputStream("Test".getBytes(Charsets.UTF_8)), - new ObjectMetadata()) + putObjectWithContent(bucketName, key, "Test") def content = new String( ByteStreams.toByteArray(client.getObject(bucketName, key).getObjectContent()), Charsets.UTF_8) @@ -374,21 +343,9 @@ abstract class BaseAWSSpec extends BaseSpecification { def key3 = DEFAULT_KEY + "/Drei" def client = getClient() when: - client.putObject( - bucketName, - key1, - new ByteArrayInputStream("Eins".getBytes(Charsets.UTF_8)), - new ObjectMetadata()) - client.putObject( - bucketName, - key2, - new ByteArrayInputStream("Zwei".getBytes(Charsets.UTF_8)), - new ObjectMetadata()) - client.putObject( - bucketName, - key3, - new ByteArrayInputStream("Drei".getBytes(Charsets.UTF_8)), - new ObjectMetadata()) + putObjectWithContent(bucketName, key1, "Eins") + putObjectWithContent(bucketName, key2, "Zwei") + putObjectWithContent(bucketName, key3, "Drei") def result = client.deleteObjects(new DeleteObjectsRequest(bucketName).withKeys(key1, key2)) then: result.getDeletedObjects().size() == 2 @@ -411,21 +368,9 @@ abstract class BaseAWSSpec extends BaseSpecification { def key3 = DEFAULT_KEY + "/Drei" def client = getClient() when: - client.putObject( - bucketName, - key1, - new ByteArrayInputStream("Eins".getBytes(Charsets.UTF_8)), - new ObjectMetadata()) - client.putObject( - bucketName, - key2, - new ByteArrayInputStream("Zwei".getBytes(Charsets.UTF_8)), - new ObjectMetadata()) - client.putObject( - bucketName, - key3, - new ByteArrayInputStream("Drei".getBytes(Charsets.UTF_8)), - new ObjectMetadata()) + putObjectWithContent(bucketName, key1, "Eins") + putObjectWithContent(bucketName, key2, "Zwei") + putObjectWithContent(bucketName, key3, "Drei") def result = client.listObjectsV2(new ListObjectsV2Request().withBucketName(bucketName).withPrefix(key1)) then: result.getKeyCount() == 2 @@ -437,4 +382,14 @@ abstract class BaseAWSSpec extends BaseSpecification { client.deleteObject(bucketName, key2) client.deleteObject(bucketName, key3) } + + private void putObjectWithContent(String bucketName, String key, String content) { + def client = getClient() + def data = content.getBytes(Charsets.UTF_8) + + def metadata = new ObjectMetadata() + metadata.setHeader(Headers.CONTENT_LENGTH, new Long(data.length)) + + client.putObject(bucketName, key, new ByteArrayInputStream(data), metadata) + } } From 0e9390da02ae47ae142f9e2f1e00d43bd3d01b50 Mon Sep 17 00:00:00 2001 From: Jakob Vogel Date: Thu, 23 Feb 2023 03:18:38 +0100 Subject: [PATCH 2/6] =?UTF-8?q?Moves=20helper=20method=20up=20=F0=9F=91=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jakob Vogel #209 --- src/test/java/BaseAWSSpec.groovy | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/test/java/BaseAWSSpec.groovy b/src/test/java/BaseAWSSpec.groovy index 6e1d5c7..9836e3c 100644 --- a/src/test/java/BaseAWSSpec.groovy +++ b/src/test/java/BaseAWSSpec.groovy @@ -32,6 +32,16 @@ abstract class BaseAWSSpec extends BaseSpecification { abstract AmazonS3Client getClient() + private void putObjectWithContent(String bucketName, String key, String content) { + def client = getClient() + def data = content.getBytes(Charsets.UTF_8) + + def metadata = new ObjectMetadata() + metadata.setHeader(Headers.CONTENT_LENGTH, new Long(data.length)) + + client.putObject(bucketName, key, new ByteArrayInputStream(data), metadata) + } + def "HEAD of non-existing bucket as expected"() { given: def bucketName = "does-not-exist" @@ -382,14 +392,4 @@ abstract class BaseAWSSpec extends BaseSpecification { client.deleteObject(bucketName, key2) client.deleteObject(bucketName, key3) } - - private void putObjectWithContent(String bucketName, String key, String content) { - def client = getClient() - def data = content.getBytes(Charsets.UTF_8) - - def metadata = new ObjectMetadata() - metadata.setHeader(Headers.CONTENT_LENGTH, new Long(data.length)) - - client.putObject(bucketName, key, new ByteArrayInputStream(data), metadata) - } } From 3b5ad709947ff0b23226fd13b3c4cccf491e1afc Mon Sep 17 00:00:00 2001 From: Jakob Vogel Date: Thu, 23 Feb 2023 03:25:37 +0100 Subject: [PATCH 3/6] =?UTF-8?q?Uses=20standard=20charsets=20=F0=9F=91=A8?= =?UTF-8?q?=E2=80=8D=E2=9A=96=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jakob Vogel #209 --- src/test/java/BaseAWSSpec.groovy | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/test/java/BaseAWSSpec.groovy b/src/test/java/BaseAWSSpec.groovy index 9836e3c..bac18b8 100644 --- a/src/test/java/BaseAWSSpec.groovy +++ b/src/test/java/BaseAWSSpec.groovy @@ -16,11 +16,11 @@ import com.amazonaws.services.s3.model.ListObjectsV2Request import com.amazonaws.services.s3.model.ObjectMetadata import com.amazonaws.services.s3.model.ResponseHeaderOverrides import com.amazonaws.services.s3.transfer.TransferManagerBuilder -import com.google.common.base.Charsets import com.google.common.io.ByteStreams import com.google.common.io.Files import sirius.kernel.BaseSpecification +import java.nio.charset.StandardCharsets import java.time.Instant import java.time.temporal.ChronoUnit @@ -34,7 +34,7 @@ abstract class BaseAWSSpec extends BaseSpecification { private void putObjectWithContent(String bucketName, String key, String content) { def client = getClient() - def data = content.getBytes(Charsets.UTF_8) + def data = content.getBytes(StandardCharsets.UTF_8) def metadata = new ObjectMetadata() metadata.setHeader(Headers.CONTENT_LENGTH, new Long(data.length)) @@ -112,7 +112,7 @@ abstract class BaseAWSSpec extends BaseSpecification { File file = File.createTempFile("test", "") file.delete() for (int i = 0; i < 10000; i++) { - Files.append("This is a test.", file, Charsets.UTF_8) + Files.append("This is a test.", file, StandardCharsets.UTF_8) } and: def tm = TransferManagerBuilder.standard().withS3Client(client).build() @@ -122,7 +122,7 @@ abstract class BaseAWSSpec extends BaseSpecification { download.deleteOnExit() tm.download(bucketName, key, download).waitForCompletion() then: - Files.toString(file, Charsets.UTF_8) == Files.toString(download, Charsets.UTF_8) + Files.toString(file, StandardCharsets.UTF_8) == Files.toString(download, StandardCharsets.UTF_8) and: client.deleteObject(bucketName, key) } @@ -140,12 +140,12 @@ abstract class BaseAWSSpec extends BaseSpecification { putObjectWithContent(bucketName, key, "Test") def content = new String( ByteStreams.toByteArray(client.getObject(bucketName, key).getObjectContent()), - Charsets.UTF_8) + StandardCharsets.UTF_8) and: GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucketName, key) URLConnection c = new URL(getClient().generatePresignedUrl(request).toString()).openConnection() and: - String downloadedData = new String(ByteStreams.toByteArray(c.getInputStream()), Charsets.UTF_8) + String downloadedData = new String(ByteStreams.toByteArray(c.getInputStream()), StandardCharsets.UTF_8) then: content == "Test" and: @@ -230,7 +230,7 @@ abstract class BaseAWSSpec extends BaseSpecification { withMultipartUploadThreshold(1). withMinimumUploadPartSize(1).build() def meta = new ObjectMetadata() - def message = "Test".getBytes(Charsets.UTF_8) + def message = "Test".getBytes(StandardCharsets.UTF_8) and: if (!client.doesBucketExist(bucketName)) { client.createBucket(bucketName) @@ -242,7 +242,7 @@ abstract class BaseAWSSpec extends BaseSpecification { upload.waitForUploadResult() def content = new String( ByteStreams.toByteArray(client.getObject(bucketName, key).getObjectContent()), - Charsets.UTF_8) + StandardCharsets.UTF_8) def userdata = client.getObjectMetadata(bucketName, key).getUserMetaDataOf("userdata") then: content == "Test" @@ -262,7 +262,7 @@ abstract class BaseAWSSpec extends BaseSpecification { withMultipartUploadThreshold(1). withMinimumUploadPartSize(1).build() def meta = new ObjectMetadata() - def message = "Test".getBytes(Charsets.UTF_8) + def message = "Test".getBytes(StandardCharsets.UTF_8) and: if (!client.doesBucketExist(bucketName)) { client.createBucket(bucketName) @@ -305,7 +305,7 @@ abstract class BaseAWSSpec extends BaseSpecification { GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucketName, key) URLConnection c = new URL(getClient().generatePresignedUrl(request).toString()).openConnection() and: - String downloadedData = new String(ByteStreams.toByteArray(c.getInputStream()), Charsets.UTF_8) + String downloadedData = new String(ByteStreams.toByteArray(c.getInputStream()), StandardCharsets.UTF_8) then: downloadedData == content and: @@ -326,7 +326,7 @@ abstract class BaseAWSSpec extends BaseSpecification { putObjectWithContent(bucketName, key, "Test") def content = new String( ByteStreams.toByteArray(client.getObject(bucketName, key).getObjectContent()), - Charsets.UTF_8) + StandardCharsets.UTF_8) and: GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucketName, key) .withExpiration(Date.from(Instant.now().plus(1, ChronoUnit.HOURS))) @@ -335,7 +335,7 @@ abstract class BaseAWSSpec extends BaseSpecification { .withContentDisposition("inline; filename=\"hello.txt\"")) URLConnection c = new URL(getClient().generatePresignedUrl(request).toString()).openConnection() and: - String downloadedData = new String(ByteStreams.toByteArray(c.getInputStream()), Charsets.UTF_8) + String downloadedData = new String(ByteStreams.toByteArray(c.getInputStream()), StandardCharsets.UTF_8) then: content == "Test" and: From a017e98c05acebb319fac6e19e0c8e366bffc89b Mon Sep 17 00:00:00 2001 From: Jakob Vogel Date: Fri, 24 Feb 2023 01:29:23 +0100 Subject: [PATCH 4/6] =?UTF-8?q?Adds=20helper=20for=20creating=20publicly?= =?UTF-8?q?=20accessible=20buckets=20while=20testing=20=F0=9F=94=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #209 SIRI-721 Signed-off-by: Jakob Vogel --- src/test/java/BaseAWSSpec.groovy | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/test/java/BaseAWSSpec.groovy b/src/test/java/BaseAWSSpec.groovy index bac18b8..1474ba5 100644 --- a/src/test/java/BaseAWSSpec.groovy +++ b/src/test/java/BaseAWSSpec.groovy @@ -32,6 +32,18 @@ abstract class BaseAWSSpec extends BaseSpecification { abstract AmazonS3Client getClient() + private void createPubliclyAccessibleBucket(String bucketName) { + def client = getClient() + client.createBucket(bucketName) + + // we make the bucket now public via our own endpoint; note that this is not proper S3 code + // where you would use ACLs that we do not support in S3 Ninja + def url = new URL("http://localhost:9999/ui/" + bucketName + "/?make-public") + HttpURLConnection connection = (HttpURLConnection) url.openConnection() + connection.getResponseCode() == 200 + connection.disconnect(); + } + private void putObjectWithContent(String bucketName, String key, String content) { def client = getClient() def data = content.getBytes(StandardCharsets.UTF_8) From f45baa6eed9664bb62cd3dbbbba9507bf605d213 Mon Sep 17 00:00:00 2001 From: Jakob Vogel Date: Fri, 24 Feb 2023 01:32:08 +0100 Subject: [PATCH 5/6] =?UTF-8?q?Adds=20test=20case=20checking=20the=20conte?= =?UTF-8?q?nt-length=20when=20responding=20to=20`HEAD`=20=F0=9F=A6=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #209 SIRI-721 Signed-off-by: Jakob Vogel --- src/test/java/BaseAWSSpec.groovy | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/test/java/BaseAWSSpec.groovy b/src/test/java/BaseAWSSpec.groovy index 1474ba5..ab51fa8 100644 --- a/src/test/java/BaseAWSSpec.groovy +++ b/src/test/java/BaseAWSSpec.groovy @@ -404,4 +404,26 @@ abstract class BaseAWSSpec extends BaseSpecification { client.deleteObject(bucketName, key2) client.deleteObject(bucketName, key3) } + + // reported in https://github.com/scireum/s3ninja/issues/209 + def "HEAD reports content length correctly"() { + given: + def bucketName = "public-bucket" + def key = "simple_test" + def content = "I am pointless text content" + def client = getClient() + when: + createPubliclyAccessibleBucket(bucketName) + putObjectWithContent(bucketName, key, content) + then: + def url = new URL("http://localhost:9999/" + bucketName + "/" + key) + HttpURLConnection connection = (HttpURLConnection) url.openConnection() + connection.setRequestMethod("HEAD") + connection.getResponseCode() == 200 + connection.getContentLengthLong() == content.getBytes(StandardCharsets.UTF_8).length + connection.disconnect() + and: + client.deleteObject(bucketName, key) + client.deleteBucket(bucketName) + } } From 86859b929715cbfc82a12839e44bcf42380e718e Mon Sep 17 00:00:00 2001 From: Jakob Vogel Date: Fri, 24 Feb 2023 12:02:44 +0100 Subject: [PATCH 6/6] =?UTF-8?q?Upgrades=20Sirius=20=F0=9F=AA=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #209 SIRI-721 Signed-off-by: Jakob Vogel --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index ee12a15..c7ecbb8 100644 --- a/pom.xml +++ b/pom.xml @@ -16,8 +16,8 @@ http://s3ninja.net - dev-34.6.0 - dev-62.5.2 + dev-34.8.0 + dev-63.0.1