From 87cd10e3613eb373cff232cdde54ded15ab294b3 Mon Sep 17 00:00:00 2001 From: Kevin McConnell Date: Thu, 21 Mar 2024 11:29:38 +0000 Subject: [PATCH] Disable transparent proxy compression The default transport used by the proxy requests compressed responses even if the client didn't. If it receives a compressed response but the client wants uncompressed, the transport decompresses the response transparently. Although that seems helpful, it doesn't play well with `X-Sendfile` responses, as it may result in us being handed a reference to a file on disk that is already compressed, and we'd have to similarly decompress it before serving it to the client. This is wasteful, especially since there was probably an uncompressed version of it on disk already. It's also a bit fiddly to do on the fly without the ability to seek around in the uncompressed content. Compression between us and the upstream server is likely to be of limited use anyway, since we're only proxying from localhost. Given that fact -- and the fact that most clients *will* request compressed responses anyway, which makes all of this moot -- our best option is to disable this on-the-fly compression. --- internal/handler_test.go | 49 +++++++++++++++++++++++++++++++++++++++ internal/proxy_handler.go | 26 +++++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/internal/handler_test.go b/internal/handler_test.go index 01ae403..ef2d947 100644 --- a/internal/handler_test.go +++ b/internal/handler_test.go @@ -33,6 +33,27 @@ func TestHandlerGzipCompression_when_proxying(t *testing.T) { assert.Less(t, transferredSize, fixtureLength("loremipsum.txt")) } +func TestHandlerGzipCompression_is_not_applied_when_not_requested(t *testing.T) { + fixtureLength := strconv.FormatInt(fixtureLength("loremipsum.txt"), 10) + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", fixtureLength) + w.Write(fixtureContent("loremipsum.txt")) + })) + defer upstream.Close() + + h := NewHandler(handlerOptions(upstream.URL)) + + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", "/", nil) + h.ServeHTTP(w, r) + + assert.Equal(t, http.StatusOK, w.Code) + assert.Contains(t, w.Header().Get("Content-Type"), "text/plain") + assert.Empty(t, w.Header().Get("Content-Encoding")) + assert.Equal(t, fixtureLength, w.Header().Get("Content-Length")) +} + func TestHandlerGzipCompression_does_not_compress_images(t *testing.T) { fixtureLength := strconv.FormatInt(fixtureLength("image.jpg"), 10) @@ -79,6 +100,34 @@ func TestHandlerGzipCompression_when_sendfile(t *testing.T) { assert.Less(t, transferredSize, fixtureLength("loremipsum.txt")) } +func TestHandler_do_not_request_compressed_responses_from_upstream_unless_client_does(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + acceptsGzip := r.Header.Get("Accept-Encoding") == "gzip" + shouldAcceptGzip := r.URL.Path == "/compressed" + + assert.Equal(t, shouldAcceptGzip, acceptsGzip) + if acceptsGzip { + w.Header().Set("Content-Encoding", "gzip") + } + })) + defer upstream.Close() + + h := NewHandler(handlerOptions(upstream.URL)) + + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", "/plain", nil) + h.ServeHTTP(w, r) + assert.Equal(t, http.StatusOK, w.Code) + assert.Empty(t, w.Header().Get("Content-Encoding")) + + w = httptest.NewRecorder() + r = httptest.NewRequest("GET", "/compressed", nil) + r.Header.Set("Accept-Encoding", "gzip") + h.ServeHTTP(w, r) + assert.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, "gzip", w.Header().Get("Content-Encoding")) +} + func TestHandlerMaxRequestBody(t *testing.T) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) defer upstream.Close() diff --git a/internal/proxy_handler.go b/internal/proxy_handler.go index 45e98d3..a550848 100644 --- a/internal/proxy_handler.go +++ b/internal/proxy_handler.go @@ -12,6 +12,7 @@ import ( func NewProxyHandler(targetUrl *url.URL, badGatewayPage string) http.Handler { proxy := httputil.NewSingleHostReverseProxy(targetUrl) proxy.ErrorHandler = ProxyErrorHandler(badGatewayPage) + proxy.Transport = createProxyTransport() return proxy } @@ -45,3 +46,28 @@ func isRequestEntityTooLarge(err error) bool { var maxBytesError *http.MaxBytesError return errors.As(err, &maxBytesError) } + +func createProxyTransport() *http.Transport { + // The default transport requests compressed responses even if the client + // didn't. If it receives a compressed response but the client wants + // uncompressed, the transport decompresses the response transparently. + // + // Although that seems helpful, it doesn't play well with X-Sendfile + // responses, as it may result in us being handed a reference to a file on + // disk that is already compressed, and we'd have to similarly decompress it + // before serving it to the client. This is wasteful, especially since there + // was probably an uncompressed version of it on disk already. It's also a bit + // fiddly to do on the fly without the ability to seek around in the + // uncompressed content. + // + // Compression between us and the upstream server is likely to be of limited + // use anyway, since we're only proxying from localhost. Given that fact -- + // and the fact that most clients *will* request compressed responses anyway, + // which makes all of this moot -- our best option is to disable this + // on-the-fly compression. + + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.DisableCompression = true + + return transport +}