From 37b92a7a73670de0a3ed05beb9c0823bd6050048 Mon Sep 17 00:00:00 2001 From: Erik van Oosten Date: Sun, 20 Oct 2024 21:20:34 +0200 Subject: [PATCH 1/3] Add Brotli4J compressor/decompressor --- .../main/scala/zio/compress/Brotli4J.scala | 82 ++++++++++++++++ .../scala/zio/compress/BrotliLogWindow.scala | 22 +++++ .../main/scala/zio/compress/BrotliMode.scala | 19 ++++ .../scala/zio/compress/BrotliQuality.scala | 34 +++++++ .../scala/zio/compress/Brotli4JSpec.scala | 37 ++++++++ build.sbt | 93 +++++++++++-------- 6 files changed, 247 insertions(+), 40 deletions(-) create mode 100644 brotli4j/src/main/scala/zio/compress/Brotli4J.scala create mode 100644 brotli4j/src/main/scala/zio/compress/BrotliLogWindow.scala create mode 100644 brotli4j/src/main/scala/zio/compress/BrotliMode.scala create mode 100644 brotli4j/src/main/scala/zio/compress/BrotliQuality.scala create mode 100644 brotli4j/src/test/scala/zio/compress/Brotli4JSpec.scala diff --git a/brotli4j/src/main/scala/zio/compress/Brotli4J.scala b/brotli4j/src/main/scala/zio/compress/Brotli4J.scala new file mode 100644 index 0000000..8d51d2c --- /dev/null +++ b/brotli4j/src/main/scala/zio/compress/Brotli4J.scala @@ -0,0 +1,82 @@ +package zio.compress + +import com.aayushatharva.brotli4j.Brotli4jLoader +import com.aayushatharva.brotli4j.encoder.{BrotliOutputStream, Encoder} +import com.aayushatharva.brotli4j.decoder.BrotliInputStream +import zio._ +import zio.compress.BrotliMode._ +import zio.compress.JavaIoInterop._ +import zio.stream._ + +object Brotli4JCompressor { + + /** Make a pipeline that accepts a stream of bytes and produces a stream with Brotli compressed bytes. + * + * @param quality + * The compression quality to use, or `None` for the default. + * @param lgwin + * log2(LZ window size) to use, or `None` for the default. + * @param mode + * type of encoding to use, or `None` for the default. + */ + def make( + quality: Option[BrotliQuality] = None, + lgwin: Option[BrotliLogWindow] = None, + mode: Option[BrotliMode] = None, + ): Brotli4JCompressor = + new Brotli4JCompressor(quality, lgwin, mode) +} + +class Brotli4JCompressor private ( + quality: Option[BrotliQuality], + lgwin: Option[BrotliLogWindow], + mode: Option[BrotliMode], +) extends Compressor { + override def compress: ZPipeline[Any, Throwable, Byte, Byte] = + BrotliLoader.ensureAvailability() >>> + viaOutputStreamByte { outputStream => + val brotliMode = mode.map { + case Generic => Encoder.Mode.GENERIC + case Text => Encoder.Mode.TEXT + case Font => Encoder.Mode.FONT + } + val params = new Encoder.Parameters() + .setQuality(quality.map(_.level).getOrElse(-1)) + .setWindow(lgwin.map(_.lgwin).getOrElse(-1)) + .setMode(brotliMode.orNull) + new BrotliOutputStream(outputStream, params) + } +} + +object Brotli4JDecompressor { + + /** Makes a pipeline that accepts a Brotli compressed byte stream and produces a decompressed byte stream. + * + * @param chunkSize + * The maximum chunk size of the outgoing ZStream. Defaults to `ZStream.DefaultChunkSize` (4KiB). + */ + def make( + chunkSize: Int = ZStream.DefaultChunkSize + ): Brotli4JDecompressor = + new Brotli4JDecompressor(chunkSize) +} + +class Brotli4JDecompressor private (chunkSize: Int) extends Decompressor { + override def decompress: ZPipeline[Any, Throwable, Byte, Byte] = + BrotliLoader.ensureAvailability() >>> + viaInputStreamByte(chunkSize) { inputStream => + new BrotliInputStream(inputStream) + } +} + +private object BrotliLoader { + // Trigger loading of the Brotli4j native library + new Brotli4jLoader() + + def ensureAvailability(): ZPipeline[Any, Throwable, Byte, Byte] = + ZPipeline.unwrap { + ZIO + .attemptBlocking(Brotli4jLoader.ensureAvailability()) + .as(ZPipeline.identity[Byte]) + } +} diff --git a/brotli4j/src/main/scala/zio/compress/BrotliLogWindow.scala b/brotli4j/src/main/scala/zio/compress/BrotliLogWindow.scala new file mode 100644 index 0000000..60daa98 --- /dev/null +++ b/brotli4j/src/main/scala/zio/compress/BrotliLogWindow.scala @@ -0,0 +1,22 @@ +package zio.compress + +/** Brotli log Window size. + * + * @param lgwin + * lgwin log2(LZ window size), valid values: 10 to 24 + */ +final case class BrotliLogWindow private (lgwin: Int) + +object BrotliLogWindow { + + /** Makes a valid Brotli log Window size. + * + * @param lgwin + * lgwin log2(LZ window size), valid values: 10 to 24 + * @return + * a [[BrotliLogWindow]] or `None` if the level is not valid + */ + def apply(lgwin: Int): Option[BrotliLogWindow] = + if (10 <= lgwin && lgwin <= 24) Some(new BrotliLogWindow(lgwin)) else None + +} diff --git a/brotli4j/src/main/scala/zio/compress/BrotliMode.scala b/brotli4j/src/main/scala/zio/compress/BrotliMode.scala new file mode 100644 index 0000000..918b28e --- /dev/null +++ b/brotli4j/src/main/scala/zio/compress/BrotliMode.scala @@ -0,0 +1,19 @@ +package zio.compress + +sealed trait BrotliMode + +object BrotliMode { + + /** Default compression mode. In this mode compressor does not know anything in advance about the properties of the + * input. + */ + case object Generic extends BrotliMode + + /** Compression mode for UTF-8 formatted text input. + */ + case object Text extends BrotliMode + + /** Compression mode used in WOFF 2.0. + */ + case object Font extends BrotliMode +} diff --git a/brotli4j/src/main/scala/zio/compress/BrotliQuality.scala b/brotli4j/src/main/scala/zio/compress/BrotliQuality.scala new file mode 100644 index 0000000..c6f61e6 --- /dev/null +++ b/brotli4j/src/main/scala/zio/compress/BrotliQuality.scala @@ -0,0 +1,34 @@ +package zio.compress + +/** Brotli compression level. + * + * @param level + * compression level, valid values: 0 to 11 + */ +final case class BrotliQuality private (level: Int) + +object BrotliQuality { + + /** Makes a Brotli compression level. + * + * @param level + * compression level, valid values: 0 to 11 + * @return + * a [[BrotliQuality]] or `None` if the level is not valid + */ + def apply(level: Int): Option[BrotliQuality] = + if (0 <= level && level <= 11) Some(new BrotliQuality(level)) else None + + val CompressionLevel0 = new BrotliQuality(0) + val CompressionLevel1 = new BrotliQuality(1) + val CompressionLevel2 = new BrotliQuality(2) + val CompressionLevel3 = new BrotliQuality(3) + val CompressionLevel4 = new BrotliQuality(4) + val CompressionLevel5 = new BrotliQuality(5) + val CompressionLevel6 = new BrotliQuality(6) + val CompressionLevel7 = new BrotliQuality(7) + val CompressionLevel8 = new BrotliQuality(8) + val CompressionLevel9 = new BrotliQuality(9) + val CompressionLevel10 = new BrotliQuality(10) + val CompressionLevel11 = new BrotliQuality(11) +} diff --git a/brotli4j/src/test/scala/zio/compress/Brotli4JSpec.scala b/brotli4j/src/test/scala/zio/compress/Brotli4JSpec.scala new file mode 100644 index 0000000..0ffef5f --- /dev/null +++ b/brotli4j/src/test/scala/zio/compress/Brotli4JSpec.scala @@ -0,0 +1,37 @@ +package zio.compress + +import zio._ +import zio.stream._ +import zio.test._ + +import java.nio.charset.StandardCharsets.UTF_8 +import java.util.Base64 + +object Brotli4JSpec extends ZIOSpecDefault { + private val clear = Chunk.fromArray("hello world!".getBytes(UTF_8)) + private val compressed = Chunk.fromArray(Base64.getDecoder.decode("iwWAaGVsbG8gd29ybGQhAw==")) + + override def spec: Spec[TestEnvironment with Scope, Any] = + suite("Brotli4J")( + test("brotli4J decompress") { + for { + obtained <- ZStream + .fromChunk(compressed) + .via(Brotli4JDecompressor.make().decompress) + .runCollect + } yield assertTrue(clear == obtained) + }, + test("brotli4J round trip") { + checkN(10)(Gen.int(40, 5000), Gen.chunkOfBounded(0, 20000)(Gen.byte)) { (chunkSize, genBytes) => + for { + obtained <- ZStream + .fromChunk(genBytes) + .rechunk(chunkSize) + .via(Brotli4JCompressor.make().compress) + .via(Brotli4JDecompressor.make().decompress) + .runCollect + } yield assertTrue(obtained == genBytes) + } + }, + ) +} diff --git a/build.sbt b/build.sbt index 37132cb..3b011cc 100644 --- a/build.sbt +++ b/build.sbt @@ -1,5 +1,6 @@ val V = new { val brotli = "0.1.2" + val brotli4j = "1.17.0" val commonsCompress = "1.27.1" val logbackClassic = "1.5.11" val lz4 = "1.8.0" @@ -80,14 +81,15 @@ lazy val root = publishArtifact := false, ) .aggregate(core.projectRefs: _*) + .aggregate(brotli.projectRefs: _*) + .aggregate(brotli4j.projectRefs: _*) + .aggregate(bzip2.projectRefs: _*) .aggregate(gzip.projectRefs: _*) + .aggregate(lz4.projectRefs: _*) + .aggregate(tar.projectRefs: _*) .aggregate(zip.projectRefs: _*) .aggregate(zip4j.projectRefs: _*) - .aggregate(tar.projectRefs: _*) .aggregate(zstd.projectRefs: _*) - .aggregate(bzip2.projectRefs: _*) - .aggregate(brotli.projectRefs: _*) - .aggregate(lz4.projectRefs: _*) .aggregate(example.projectRefs: _*) .aggregate(docs) @@ -102,35 +104,32 @@ lazy val core = projectMatrix .jvmPlatform(scalaVersions) .jsPlatform(scalaVersions) -lazy val gzip = projectMatrix - .in(file("gzip")) - .dependsOn(core % "compile->compile;test->test") - .settings(commonSettings("gzip")) - .jvmPlatform(scalaVersions) -//.jsPlatform(scalaVersions) - -lazy val zip = projectMatrix - .in(file("zip")) +lazy val brotli = projectMatrix + .in(file("brotli")) .dependsOn(core % "compile->compile;test->test") - .settings(commonSettings("zip")) + .settings(commonSettings("brotli")) + .settings( + libraryDependencies ++= Seq( + "org.brotli" % "dec" % V.brotli + ) + ) .jvmPlatform(scalaVersions) -lazy val zip4j = projectMatrix - .in(file("zip4j")) +lazy val brotli4j = projectMatrix + .in(file("brotli4j")) .dependsOn(core % "compile->compile;test->test") - .settings(commonSettings("zip4j")) + .settings(commonSettings("brotli4j")) .settings( libraryDependencies ++= Seq( - "net.lingala.zip4j" % "zip4j" % V.zip4j + "com.aayushatharva.brotli4j" % "brotli4j" % V.brotli4j ) ) .jvmPlatform(scalaVersions) -lazy val tar = projectMatrix - .in(file("tar")) +lazy val bzip2 = projectMatrix + .in(file("bzip2")) .dependsOn(core % "compile->compile;test->test") - .dependsOn(gzip % "test") - .settings(commonSettings("tar")) + .settings(commonSettings("bzip2")) .settings( libraryDependencies ++= Seq( "org.apache.commons" % "commons-compress" % V.commonsCompress @@ -138,21 +137,30 @@ lazy val tar = projectMatrix ) .jvmPlatform(scalaVersions) -lazy val zstd = projectMatrix - .in(file("zstd")) +lazy val gzip = projectMatrix + .in(file("gzip")) .dependsOn(core % "compile->compile;test->test") - .settings(commonSettings("zstd")) + .settings(commonSettings("gzip")) + .jvmPlatform(scalaVersions) +//.jsPlatform(scalaVersions) + +lazy val lz4 = projectMatrix + .in(file("lz4")) + .dependsOn(core % "compile->compile;test->test") + .settings(commonSettings("lz4")) .settings( + name := "zio-streams-compress-lz4", libraryDependencies ++= Seq( - "com.github.luben" % "zstd-jni" % V.zstdJni - ) + "org.lz4" % "lz4-java" % V.lz4 + ), ) .jvmPlatform(scalaVersions) -lazy val bzip2 = projectMatrix - .in(file("bzip2")) +lazy val tar = projectMatrix + .in(file("tar")) .dependsOn(core % "compile->compile;test->test") - .settings(commonSettings("bzip2")) + .dependsOn(gzip % "test") + .settings(commonSettings("tar")) .settings( libraryDependencies ++= Seq( "org.apache.commons" % "commons-compress" % V.commonsCompress @@ -160,26 +168,31 @@ lazy val bzip2 = projectMatrix ) .jvmPlatform(scalaVersions) -lazy val brotli = projectMatrix - .in(file("brotli")) +lazy val zip = projectMatrix + .in(file("zip")) .dependsOn(core % "compile->compile;test->test") - .settings(commonSettings("brotli")) + .settings(commonSettings("zip")) + .jvmPlatform(scalaVersions) + +lazy val zip4j = projectMatrix + .in(file("zip4j")) + .dependsOn(core % "compile->compile;test->test") + .settings(commonSettings("zip4j")) .settings( libraryDependencies ++= Seq( - "org.brotli" % "dec" % V.brotli + "net.lingala.zip4j" % "zip4j" % V.zip4j ) ) .jvmPlatform(scalaVersions) -lazy val lz4 = projectMatrix - .in(file("lz4")) +lazy val zstd = projectMatrix + .in(file("zstd")) .dependsOn(core % "compile->compile;test->test") - .settings(commonSettings("lz4")) + .settings(commonSettings("zstd")) .settings( - name := "zio-streams-compress-lz4", libraryDependencies ++= Seq( - "org.lz4" % "lz4-java" % V.lz4 - ), + "com.github.luben" % "zstd-jni" % V.zstdJni + ) ) .jvmPlatform(scalaVersions) From 95ee15177b2383f9aa47162b609cd8a64ea5e4c8 Mon Sep 17 00:00:00 2001 From: Erik van Oosten Date: Mon, 21 Oct 2024 14:23:33 +0200 Subject: [PATCH 2/3] Update docs --- docs/index.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/index.md b/docs/index.md index de4354d..02ee2e2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -14,16 +14,21 @@ archive formats with [ZIO Streams](https://zio.dev). In order to use this library, we need to add one of the following line in our `build.sbt` file: ```sbt +libraryDependencies += "dev.zio" %% "zio-streams-compress-brotli" % "@VERSION@" +libraryDependencies += "dev.zio" %% "zio-streams-compress-brotli4j" % "@VERSION@" +libraryDependencies += "dev.zio" %% "zio-streams-compress-bzip2" % "@VERSION@" libraryDependencies += "dev.zio" %% "zio-streams-compress-gzip" % "@VERSION@" +libraryDependencies += "dev.zio" %% "zio-streams-compress-lz4" % "@VERSION@" +libraryDependencies += "dev.zio" %% "zio-streams-compress-tar" % "@VERSION@" libraryDependencies += "dev.zio" %% "zio-streams-compress-zip" % "@VERSION@" libraryDependencies += "dev.zio" %% "zio-streams-compress-zip4j" % "@VERSION@" -libraryDependencies += "dev.zio" %% "zio-streams-compress-tar" % "@VERSION@" -libraryDependencies += "dev.zio" %% "zio-streams-compress-bzip2" % "@VERSION@" libraryDependencies += "dev.zio" %% "zio-streams-compress-zstd" % "@VERSION@" -libraryDependencies += "dev.zio" %% "zio-streams-compress-brotli" % "@VERSION@" -libraryDependencies += "dev.zio" %% "zio-streams-compress-lz4" % "@VERSION@" ``` +For Brotli you can choose between the 'brotli' and the 'brotli4j' version. The first is based on the official Java +library but only does decompression. The second is based on [Brotli4J](https://github.com/hyperxpro/Brotli4j) which does +compression and decompression. + For ZIP files you can choose between the 'zip' and the 'zip4j' version. The first allows you to tweak the compression level, while the second allows you work with password-protected ZIP files. From 5cffc68309b496f457051c5b980dff47b4fc9953 Mon Sep 17 00:00:00 2001 From: Erik van Oosten Date: Mon, 21 Oct 2024 14:25:06 +0200 Subject: [PATCH 3/3] Better names --- .../scala/zio/compress/BrotliQuality.scala | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/brotli4j/src/main/scala/zio/compress/BrotliQuality.scala b/brotli4j/src/main/scala/zio/compress/BrotliQuality.scala index c6f61e6..1e5650f 100644 --- a/brotli4j/src/main/scala/zio/compress/BrotliQuality.scala +++ b/brotli4j/src/main/scala/zio/compress/BrotliQuality.scala @@ -19,16 +19,16 @@ object BrotliQuality { def apply(level: Int): Option[BrotliQuality] = if (0 <= level && level <= 11) Some(new BrotliQuality(level)) else None - val CompressionLevel0 = new BrotliQuality(0) - val CompressionLevel1 = new BrotliQuality(1) - val CompressionLevel2 = new BrotliQuality(2) - val CompressionLevel3 = new BrotliQuality(3) - val CompressionLevel4 = new BrotliQuality(4) - val CompressionLevel5 = new BrotliQuality(5) - val CompressionLevel6 = new BrotliQuality(6) - val CompressionLevel7 = new BrotliQuality(7) - val CompressionLevel8 = new BrotliQuality(8) - val CompressionLevel9 = new BrotliQuality(9) - val CompressionLevel10 = new BrotliQuality(10) - val CompressionLevel11 = new BrotliQuality(11) + val Quality0 = new BrotliQuality(0) + val Quality1 = new BrotliQuality(1) + val Quality2 = new BrotliQuality(2) + val Quality3 = new BrotliQuality(3) + val Quality4 = new BrotliQuality(4) + val Quality5 = new BrotliQuality(5) + val Quality6 = new BrotliQuality(6) + val Quality7 = new BrotliQuality(7) + val Quality8 = new BrotliQuality(8) + val Quality9 = new BrotliQuality(9) + val Quality10 = new BrotliQuality(10) + val Quality11 = new BrotliQuality(11) }