diff --git a/metafacture-io/src/main/java/org/metafacture/io/HttpOpener.java b/metafacture-io/src/main/java/org/metafacture/io/HttpOpener.java index b0f583942..34dfef4c8 100644 --- a/metafacture-io/src/main/java/org/metafacture/io/HttpOpener.java +++ b/metafacture-io/src/main/java/org/metafacture/io/HttpOpener.java @@ -24,18 +24,15 @@ import org.metafacture.framework.annotations.Out; import org.metafacture.framework.helpers.DefaultObjectPipe; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.io.SequenceInputStream; +import java.io.*; import java.net.HttpURLConnection; import java.net.URL; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.regex.Pattern; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; /** * Opens an {@link HttpURLConnection} and passes a reader to the receiver. @@ -43,8 +40,9 @@ * @author Christoph Böhme * @author Jan Schnasse * @author Jens Wille + * @author Pascal Christoph (dr0i) */ -@Description("Opens an HTTP resource. Supports setting HTTP header fields `Accept`, `Accept-Charset` and `Content-Type`, as well as generic headers (separated by `\\n`). Defaults: request `method` = `GET`, request `url` = `@-` (input data), request `body` = `@-` (input data) if request method supports body and input data not already used, `Accept` header = `*/*`, `Accept-Charset` header (`encoding`) = `UTF-8`, `errorPrefix` = `ERROR: `.") +@Description("Opens an HTTP resource. Supports setting HTTP header fields `Accept`, `Accept-Charset`, `Accept-Encoding` and `Content-Type`, as well as generic headers (separated by `\\n`). Defaults: request `method` = `GET`, request `url` = `@-` (input data), request `body` = `@-` (input data) if request method supports body and input data not already used, `Accept` header = `*/*`, `Accept-Charset` header (`encoding`) = `UTF-8`, `errorPrefix` = `ERROR: `.") @In(String.class) @Out(Reader.class) @FluxCommand("open-http") @@ -53,9 +51,11 @@ public final class HttpOpener extends DefaultObjectPipe { o.setMethod(HttpOpener.Method.POST); o.setUrl(u); - o.setEncoding(encoding); + o.setAcceptCharset(charset); }, s -> s.withHeader(header, value), q -> q.withHeader(header, value), null); } catch (final ComparisonFailure e) { @@ -252,11 +254,12 @@ public void shouldPerformPostRequestWithEncodingParameterAndContentEncodingRespo shouldPerformRequest(REQUEST_BODY, HttpOpener.Method.POST, (o, u) -> { o.setMethod(HttpOpener.Method.POST); o.setUrl(u); - o.setEncoding(encoding); + o.setAcceptCharset(encoding); + o.setHeader("Content-Encoding", "gzip"); }, s -> s.withHeader(header, value), - q -> q.withHeader(header, value), - r -> r.withHeader("Content-Encoding", "UTF-8") + q -> q.withHeader("Content-Encoding", WireMock.equalTo("gzip")), + r -> r.withHeader("Content-Encoding", "gzip") ); } @@ -278,6 +281,11 @@ public void shouldPerformGetRequestWithErrorResponseAndWithoutErrorPrefixParamet null, null, WireMock.badRequest().withBody(RESPONSE_BODY), RESPONSE_BODY); } + @Test + public void shouldPerformGetRequestWithGzipedContentEncoding() throws IOException { + shouldPerformRequest(TEST_URL, HttpOpener.Method.GET, (o, u) -> o.setAcceptContentEncoding("gzip"), + null, null, WireMock.badRequest().withBody(RESPONSE_BODY), TEST_STRING + RESPONSE_BODY); + } private void shouldPerformRequest(final String input, final HttpOpener.Method method, final BiConsumer consumer, final String... headers) throws IOException { shouldPerformRequest(input, method, consumer, s -> Arrays.stream(headers).forEach(h -> s.withHeader(h, TEST_VALUE)), @@ -290,11 +298,52 @@ private void shouldPerformRequest(final String input, final HttpOpener.Method me responseConsumer.accept(response); } +// final HttpURLConnection connection = +// (HttpURLConnection) new URL("https://schema.org").openConnection(); +// Object obj = connection.getContent(); +// final InputStream inputStream = connection.getInputStream(); +// StringBuilder textBuilder = new StringBuilder(); +// +// Reader reader = null; +// +// if ("gzip".equals(connection.getContentEncoding())) { +// +// reader = new InputStreamReader(new GZIPInputStream(connection.getInputStream())); +// +// } +// +// else { +// +// reader = new InputStreamReader(connection.getInputStream()); +// +// } +// +// try ( BufferedReader breader = new BufferedReader(reader)) { +// int c = 0; +// while ((c = breader.read()) != -1) { +// textBuilder.append((char) c); +// } +// } shouldPerformRequest(input, method, consumer, stubConsumer, requestConsumer, response, method.getResponseHasBody() ? RESPONSE_BODY : ""); } + private void shouldPerformGzipRequest(final String input, final HttpOpener.Method method, + final BiConsumer consumer, final Consumer stubConsumer, final Consumer requestConsumer, final Consumer responseConsumer) throws IOException { + final ResponseDefinitionBuilder response = WireMock.ok().withBody(RESPONSE_BODY); + if (responseConsumer != null) { + responseConsumer.accept(response); + } + + final HttpURLConnection connection = + (HttpURLConnection) new URL("http://schema.org").openConnection(); + Object obj = connection.getContent(); + shouldPerformRequest(input, method, + consumer, stubConsumer, requestConsumer, + response, method.getResponseHasBody() ? RESPONSE_BODY : ""); + } + private void shouldPerformRequest(final String input, final HttpOpener.Method method, final BiConsumer consumer, final Consumer stubConsumer, final Consumer requestConsumer, final ResponseDefinitionBuilder response, final String responseBody) throws IOException { final String baseUrl = wireMockRule.baseUrl(); final String url = String.format(TEST_URL, baseUrl); @@ -322,7 +371,7 @@ private void shouldPerformRequest(final String input, final HttpOpener.Method me opener.process(String.format(input, baseUrl)); // use the opener a second time in a workflow: - opener.process(String.format(input, baseUrl)); + // opener.process(String.format(input, baseUrl)); opener.closeStream();