[WIP] HTTP Compression Support

auto/compression

@@ -0,0 +1,96 @@
+
+# Copyright (C) Alejandro Colomar
+# Copyright (C) Andrew Clayton
+# Copyright (C) NGINX, Inc.
+
+
+NXT_HAVE_ZLIB=no
+NXT_ZLIB_CFLAGS=
+NXT_ZLIB_LIBS=
+
+NXT_HAVE_ZSTD=no
+NXT_ZSTD_CFLAGS=
+NXT_ZSTD_LIBS=
+
+NXT_HAVE_BROTLI=no
+NXT_BROTLI_CFLAGS=
+NXT_BROTLI_LIBS=
+
+
+if [ $NXT_ZLIB = YES ]; then
+    NXT_ZLIB_CFLAGS="$(pkgconf --cflags-only-I zlib 2>/dev/null || echo "")"
+    NXT_ZLIB_LIBS="$(pkgconf --libs zlib 2>/dev/null || echo "-lz")"
+
+    nxt_feature="zlib"
+    nxt_feature_name=NXT_HAVE_ZLIB
+    nxt_feature_run=no
+    nxt_feature_incs=$NXT_ZLIB_CFLAGS
+    nxt_feature_libs=$NXT_ZLIB_LIBS
+    nxt_feature_test="#include <stdio.h>
+
+                      #include <zlib.h>
+
+                      int main(void) {
+                          puts(zlibVersion());
+                          return 0;
+                      }"
+    . auto/feature
+
+    if [ $nxt_found = yes ]; then
+        NXT_HAVE_ZLIB=YES
+        echo " + zlib version: $(pkgconf --modversion zlib)"
+    fi
+fi
+
+
+if [ $NXT_ZSTD = YES ]; then
+    NXT_ZSTD_CFLAGS="$(pkgconf --cflags-only-I libzstd 2>/dev/null || echo "")"
+    NXT_ZSTD_LIBS="$(pkgconf --libs libzstd 2>/dev/null || echo "-lzstd")"
+
+    nxt_feature="zstd"
+    nxt_feature_name=NXT_HAVE_ZSTD
+    nxt_feature_run=no
+    nxt_feature_incs=$NXT_ZSTD_CFLAGS
+    nxt_feature_libs=$NXT_ZSTD_LIBS
+    nxt_feature_test="#include <stdio.h>
+
+                      #include <zstd.h>
+
+                      int main(void) {
+                          printf(\"zstd version: %u\n\", ZSTD_versionNumber());
+                          return 0;
+                      }"
+    . auto/feature
+
+    if [ $nxt_found = yes ]; then
+        NXT_HAVE_ZSTD=YES
+        echo " + zstd version: $(pkgconf --modversion libzstd)"
+    fi
+fi
+
+
+if [ $NXT_BROTLI = YES ]; then
+    NXT_BROTLI_CFLAGS="$(pkgconf --cflags-only-I libbrotlienc 2>/dev/null || echo "")"
+    NXT_BROTLI_LIBS="$(pkgconf --libs libbrotlienc 2>/dev/null || echo "-lbrotlienc")"
+
+    nxt_feature="brotli"
+    nxt_feature_name=NXT_HAVE_BROTLI
+    nxt_feature_run=no
+    nxt_feature_incs=$NXT_BROTLI_CFLAGS
+    nxt_feature_libs=$NXT_BROTLI_LIBS
+    nxt_feature_test="#include <stdio.h>
+
+                      #include <brotli/encode.h>
+
+                      int main(void) {
+                          printf(\"brotli version: %d\n\",
+                                 BrotliEncoderVersion());
+                          return 0;
+                      }"
+    . auto/feature
+
+    if [ $nxt_found = yes ]; then
+        NXT_HAVE_BROTLI=YES
+        echo " + brotli version: $(pkgconf --modversion libbrotlienc)"
+    fi
+fi

auto/help

@@ -50,6 +50,10 @@ cat << END
 
   --openssl            enable OpenSSL library usage
 
+  --zlib               enable zlib compression
+  --zstd               enable zstd compression
+  --brotli             enable brotli compression
+
   --njs                enable njs library usage
 
   --debug              enable debug logging

auto/options

@@ -26,6 +26,10 @@ NXT_GNUTLS=NO
 NXT_CYASSL=NO
 NXT_POLARSSL=NO
 
+NXT_ZLIB=NO
+NXT_ZSTD=NO
+NXT_BROTLI=NO
+
 NXT_NJS=NO
 
 NXT_TEST_BUILD_EPOLL=NO
@@ -111,6 +115,10 @@ do
         --cyassl)                        NXT_CYASSL=YES                      ;;
         --polarssl)                      NXT_POLARSSL=YES                    ;;
 
+        --zlib)                          NXT_ZLIB=YES                        ;;
+        --zstd)                          NXT_ZSTD=YES                        ;;
+        --brotli)                        NXT_BROTLI=YES                      ;;
+
         --njs)                           NXT_NJS=YES                         ;;
 
         --test-build-epoll)              NXT_TEST_BUILD_EPOLL=YES            ;;

auto/sources

@@ -107,6 +107,7 @@ NXT_LIB_SRCS=" \
     src/nxt_http_websocket.c \
     src/nxt_h1proto_websocket.c \
     src/nxt_fs.c \
+    src/nxt_http_compression.c \
 "
 
 
@@ -211,6 +212,21 @@ if [ $NXT_POLARSSL = YES ]; then
 fi
 
 
+if [ "$NXT_HAVE_ZLIB" = "YES" ]; then
+    NXT_LIB_SRCS="$NXT_LIB_SRCS src/nxt_zlib.c"
+fi
+
+
+if [ "$NXT_HAVE_ZSTD" = "YES" ]; then
+    NXT_LIB_SRCS="$NXT_LIB_SRCS src/nxt_zstd.c"
+fi
+
+
+if [ "$NXT_HAVE_BROTLI" = "YES" ]; then
+    NXT_LIB_SRCS="$NXT_LIB_SRCS src/nxt_brotli.c"
+fi
+
+
 if [ "$NXT_REGEX" = "YES" ]; then
     if [ "$NXT_HAVE_PCRE2" = "YES" ]; then
         NXT_LIB_SRCS="$NXT_LIB_SRCS $NXT_LIB_PCRE2_SRCS"

auto/summary

@@ -28,6 +28,9 @@ Unit configuration summary:
   IPv6 support: .............. $NXT_INET6
   Unix domain sockets support: $NXT_UNIX_DOMAIN
   TLS support: ............... $NXT_OPENSSL
+  zlib support: .............. $NXT_ZLIB
+  zstd support: .............. $NXT_ZSTD
+  brotli support: ............ $NXT_BROTLI
   Regex support: ............. $NXT_REGEX
   njs support: ............... $NXT_NJS
 

configure

@@ -127,6 +127,7 @@ NXT_LIBRT=
 . auto/unix
 . auto/os/conf
 . auto/ssltls
+. auto/compression
 
 if [ $NXT_REGEX = YES ]; then
     . auto/pcre
@@ -168,11 +169,13 @@ END
 
 NXT_LIB_AUX_CFLAGS="$NXT_OPENSSL_CFLAGS $NXT_GNUTLS_CFLAGS \\
                     $NXT_CYASSL_CFLAGS $NXT_POLARSSL_CFLAGS \\
-                    $NXT_PCRE_CFLAGS"
+                    $NXT_PCRE_CFLAGS $NXT_ZLIB_CFLAGS $NXT_ZSTD_CFLAGS \\
+                    $NXT_BROTLI_CFLAGS"
 
 NXT_LIB_AUX_LIBS="$NXT_OPENSSL_LIBS $NXT_GNUTLS_LIBS \\
                     $NXT_CYASSL_LIBS $NXT_POLARSSL_LIBS \\
-                    $NXT_PCRE_LIB"
+                    $NXT_PCRE_LIB $NXT_ZLIB_LIBS $NXT_ZSTD_LIBS \\
+                    $NXT_BROTLI_LIBS"
 
 if [ $NXT_NJS != NO ]; then
     . auto/njs

src/nxt_brotli.c

@@ -0,0 +1,86 @@
+/*
+ *
+ */
+
+/* XXX Remove */
+#define _GNU_SOURCE
+#include <unistd.h>
+
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdbool.h>
+
+#include <brotli/encode.h>
+
+#include <nxt_http_compression.h>
+
+static void nxt_brotli_free(const nxt_http_comp_compressor_ctx_t *ctx)
+{
+    BrotliEncoderState *brotli = ctx->brotli_ctx;
+
+    BrotliEncoderDestroyInstance(brotli);
+}
+
+static void nxt_brotli_init(nxt_http_comp_compressor_ctx_t *ctx)
+{
+    BrotliEncoderState **brotli = &ctx->brotli_ctx;
+
+    *brotli = BrotliEncoderCreateInstance(NULL, NULL, NULL);
+    BrotliEncoderSetParameter(*brotli, BROTLI_PARAM_QUALITY, ctx->level);
+
+    printf("%7d %s: brotli compression level [%d]\n", gettid(), __func__,
+           ctx->level);
+}
+
+static size_t nxt_brotli_bound(const nxt_http_comp_compressor_ctx_t *ctx,
+                               size_t in_len)
+{
+    return BrotliEncoderMaxCompressedSize(in_len);
+}
+
+static ssize_t nxt_brotli_compress(nxt_http_comp_compressor_ctx_t *ctx,
+                                   const uint8_t *in_buf, size_t in_len,
+                                   uint8_t *out_buf, size_t out_len, bool last)
+{
+    bool ok;
+    size_t out_bytes;
+    uint8_t *outp;
+    BrotliEncoderState *brotli = ctx->brotli_ctx;
+
+    printf("%7d %s: last/%s\n", gettid(), __func__, last ? "true" : "false");
+    printf("%7d %s: in_len [%lu] out_len [%lu]\n", gettid(),  __func__,
+           in_len, out_len);
+
+    outp = out_buf;
+
+    ok = BrotliEncoderCompressStream(brotli, BROTLI_OPERATION_PROCESS,
+                                     &in_len, &in_buf, &out_bytes, &outp,
+                                     NULL);
+
+    ok = BrotliEncoderCompressStream(brotli, BROTLI_OPERATION_FLUSH,
+                                     &in_len, &in_buf, &out_bytes, &outp,
+                                     NULL);
+
+    printf("%7d %s: in_len [%lu] out_len [%lu] out_bytes [%lu]\n", gettid(),
+           __func__, in_len, out_len, out_bytes);
+    if (last) {
+        ok = BrotliEncoderCompressStream(brotli, BROTLI_OPERATION_FINISH,
+                                         &in_len, &in_buf, &out_bytes, &outp,
+                                         NULL);
+        nxt_brotli_free(ctx);
+    }
+
+    printf("%7d %s: in_len [%lu] out_len [%lu] out_bytes [%lu]\n", gettid(),
+           __func__, in_len, out_len, out_bytes);
+    printf("%7d %s: buf [%p] outp [%p]\n", gettid(), __func__, out_buf, outp);
+
+    return out_len - out_bytes;
+}
+
+const nxt_http_comp_operations_t  nxt_comp_brotli_ops = {
+    .init               = nxt_brotli_init,
+    .bound              = nxt_brotli_bound,
+    .deflate            = nxt_brotli_compress,
+    .free_ctx           = nxt_brotli_free,
+};