Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

No more make #9

Merged
merged 2 commits into from
Oct 23, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 26 additions & 26 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ Please [open an issue](https://github.com/zio/zio-streams-compress/issues/new) o
on [Discord](https://discord.com/channels/629491597070827530/630498701860929559) if you have suggestions. The API will
stabilize in Jan 2025, followed by a 1.0.0 release.

## Usage
## Installation

In order to use this library, we need to add one of the following line in our `build.sbt` file:

Expand Down Expand Up @@ -46,49 +46,49 @@ Currently only jvm is supported. PRs for scala-js and scala-native are welcome.
// Example.sc
// Run with: scala-cli Example.sc
//> using dep dev.zio:zio-streams-compress-gzip:0.0.1
//> using dep dev.zio:zio-streams-compress-zip:0.0.1
//> using dep dev.zio:zio-streams-compress-tar:0.0.1
//> using dep dev.zio:zio-streams-compress-zip4j:0.0.1

import zio._
import zio.compress.{ArchiveEntry, GzipCompressor, GzipDecompressor, TarUnarchiver, ZipArchiver}
import zio.compress.{ArchiveEntry, GzipCompressor, GzipDecompressor, TarUnarchiver, Zip4JArchiver}
import zio.stream._

import java.nio.charset.StandardCharsets.UTF_8

object ExampleApp extends ZIOAppDefault {
override def run =
override def run: ZIO[Any, Any, Any] =
for {
// Compress a file with GZIP
_ <- ZStream
.fromFileName("file")
.via(GzipCompressor.make().compress)
.run(ZSink.fromFileName("file.gz"))
.fromFileName("file")
.via(GzipCompressor.compress)
.run(ZSink.fromFileName("file.gz"))

// List all items in a gzip tar archive:
_ <- ZStream
.fromFileName("file.tgz")
.via(GzipDecompressor.make().decompress)
.via(TarUnarchiver.make().unarchive)
.mapZIO { case (archiveEntry, contentStream) =>
for {
content <- contentStream.runCollect
_ <- Console.printLine(s"${archiveEntry.name} ${content.length}")
} yield ()
}
.runDrain

// Create a ZIP archive (use the zip4j version for password support)
.fromFileName("file.tgz")
.via(GzipDecompressor.decompress)
.via(TarUnarchiver.unarchive)
.mapZIO { case (archiveEntry, contentStream) =>
for {
content <- contentStream.runCollect
_ <- Console.printLine(s"${archiveEntry.name} ${content.length}")
} yield ()
}
.runDrain

// Create an encrypted ZIP archive
_ <- ZStream(archiveEntry("file1.txt", "Hello world!".getBytes(UTF_8)))
.via(ZipArchiver.make().archive)
.run(ZSink.fromFileName("file.zip"))
.via(Zip4JArchiver(password = Some("it is a secret")).archive)
.run(ZSink.fromFileName("file.zip"))
} yield ()

private def archiveEntry(
name: String,
content: Array[Byte]
): (ArchiveEntry[Some, Any], ZStream[Any, Throwable, Byte]) = {
(ArchiveEntry(name, Some(content.length)), ZStream.fromIterable(content))
}
name: String,
content: Array[Byte],
): (ArchiveEntry[Some, Any], ZStream[Any, Throwable, Byte]) =
(ArchiveEntry(name, Some(content.length.toLong)), ZStream.fromIterable(content))

}
```

Expand Down
8 changes: 6 additions & 2 deletions brotli/src/main/scala/zio/compress/Brotli.scala
Original file line number Diff line number Diff line change
Expand Up @@ -8,22 +8,26 @@ import zio.Trace
//noinspection ScalaFileName
object BrotliDecompressor {

/** Makes a pipeline that accepts a Brotli compressed byte stream and produces a decompressed byte stream.
/** A [[Decompressor]] for Brotli, based on the official Brotli library.
*
* @param customDictionary
* a custom dictionary, or `None` for no custom dictionary
* @param chunkSize
* The maximum chunk size of the outgoing ZStream. Defaults to `ZStream.DefaultChunkSize` (4KiB).
*/
def make(
def apply(
customDictionary: Option[Array[Byte]] = None,
chunkSize: Int = ZStream.DefaultChunkSize,
): BrotliDecompressor =
new BrotliDecompressor(customDictionary, chunkSize)

/** See [[apply]] and [[Decompressor.decompress]]. */
def decompress: ZPipeline[Any, Throwable, Byte, Byte] = apply().decompress
}

//noinspection ScalaFileName
final class BrotliDecompressor private (customDictionary: Option[Array[Byte]], chunkSize: Int) extends Decompressor {

override def decompress(implicit trace: Trace): ZPipeline[Any, Throwable, Byte, Byte] =
// BrotliInputStream.read does its best to read as many bytes as requested; no buffering needed.
viaInputStreamByte(chunkSize) { inputStream =>
Expand Down
2 changes: 1 addition & 1 deletion brotli/src/test/scala/zio/compress/BrotliSpec.scala
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ object BrotliSpec extends ZIOSpecDefault {
for {
obtained <- ZStream
.fromChunk(compressed)
.via(BrotliDecompressor.make().decompress)
.via(BrotliDecompressor.decompress)
.runCollect
} yield assertTrue(clear == obtained)
}
Expand Down
16 changes: 12 additions & 4 deletions brotli4j/src/main/scala/zio/compress/Brotli4J.scala
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import zio.stream._

object Brotli4JCompressor {

/** Make a pipeline that accepts a stream of bytes and produces a stream with Brotli compressed bytes.
/** A [[Compressor]] for Brotli, based on the Brotli4J library.
*
* @param quality
* The compression quality to use, or `None` for the default.
Expand All @@ -19,19 +19,23 @@ object Brotli4JCompressor {
* @param mode
* type of encoding to use, or `None` for the default.
*/
def make(
def apply(
quality: Option[BrotliQuality] = None,
lgwin: Option[BrotliLogWindow] = None,
mode: Option[BrotliMode] = None,
): Brotli4JCompressor =
new Brotli4JCompressor(quality, lgwin, mode)

/** See [[apply]] and [[Compressor.compress]]. */
def compress: ZPipeline[Any, Throwable, Byte, Byte] = apply().compress
}

final class Brotli4JCompressor private (
quality: Option[BrotliQuality],
lgwin: Option[BrotliLogWindow],
mode: Option[BrotliMode],
) extends Compressor {

override def compress(implicit trace: Trace): ZPipeline[Any, Throwable, Byte, Byte] =
BrotliLoader.ensureAvailability() >>>
viaOutputStreamByte { outputStream =>
Expand All @@ -50,18 +54,22 @@ final class Brotli4JCompressor private (

object Brotli4JDecompressor {

/** Makes a pipeline that accepts a Brotli compressed byte stream and produces a decompressed byte stream.
/** A [[Decompressor]] for Brotli, based on the Brotli4J library.
*
* @param chunkSize
* The maximum chunk size of the outgoing ZStream. Defaults to `ZStream.DefaultChunkSize` (4KiB).
*/
def make(
def apply(
chunkSize: Int = ZStream.DefaultChunkSize
): Brotli4JDecompressor =
new Brotli4JDecompressor(chunkSize)

/** See [[apply]] and [[Decompressor.decompress]]. */
def decompress: ZPipeline[Any, Throwable, Byte, Byte] = apply().decompress
}

final class Brotli4JDecompressor private (chunkSize: Int) extends Decompressor {

override def decompress(implicit trace: Trace): ZPipeline[Any, Throwable, Byte, Byte] =
BrotliLoader.ensureAvailability() >>>
viaInputStreamByte(chunkSize) { inputStream =>
Expand Down
6 changes: 3 additions & 3 deletions brotli4j/src/test/scala/zio/compress/Brotli4JSpec.scala
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ object Brotli4JSpec extends ZIOSpecDefault {
for {
obtained <- ZStream
.fromChunk(compressed)
.via(Brotli4JDecompressor.make().decompress)
.via(Brotli4JDecompressor.decompress)
.runCollect
} yield assertTrue(clear == obtained)
},
Expand All @@ -27,8 +27,8 @@ object Brotli4JSpec extends ZIOSpecDefault {
obtained <- ZStream
.fromChunk(genBytes)
.rechunk(chunkSize)
.via(Brotli4JCompressor.make().compress)
.via(Brotli4JDecompressor.make().decompress)
.via(Brotli4JCompressor.compress)
.via(Brotli4JDecompressor.decompress)
.runCollect
} yield assertTrue(obtained == genBytes)
}
Expand Down
11 changes: 1 addition & 10 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -49,15 +49,6 @@ inThisBuild(

def commonSettings(projectName: String) = Seq(
name := s"zio-streams-compress-$projectName",
// Compile / compile / scalacOptions ++=
// optionsOn("2.13")("-Wconf:cat=unused-nowarn:s").value,
// scalacOptions -= "-Xlint:infer-any",
// workaround for bad constant pool issue
// (Compile / doc) := Def.taskDyn {
// val default = (Compile / doc).taskValue
// Def.task(default.value)
// }.value,
// Test / scalaJSLinkerConfig ~= (_.withModuleKind(ModuleKind.CommonJSModule)),
libraryDependencies ++= Seq(
"dev.zio" %%% "zio-test" % V.zio % Test,
"dev.zio" %%% "zio-test-sbt" % V.zio % Test,
Expand Down Expand Up @@ -198,7 +189,7 @@ lazy val zstd = projectMatrix

lazy val example = projectMatrix
.in(file("example"))
.dependsOn(gzip, tar, zip)
.dependsOn(gzip, tar, zip4j)
.settings(commonSettings("example"))
.settings(
publishArtifact := false,
Expand Down
16 changes: 12 additions & 4 deletions bzip2/src/main/scala/zio/compress/Bzip2.scala
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import zio.Trace

object Bzip2Compressor {

/** Make a pipeline that accepts a stream of bytes and produces a stream with Bzip2 compressed bytes.
/** A [[Compressor]] for Bzip2, based on the Apache Commons Compress library.
*
* Note: Bzip2 uses a lot of memory. See
* [[org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream]] for an overview of the required heap
Expand All @@ -16,11 +16,15 @@ object Bzip2Compressor {
* @param blockSize
* the block size to use. Defaults to 900KB.
*/
def make(blockSize: Option[Bzip2BlockSize] = None): Bzip2Compressor =
def apply(blockSize: Option[Bzip2BlockSize] = None): Bzip2Compressor =
new Bzip2Compressor(blockSize)

/** See [[apply]] and [[Compressor.compress]]. */
def compress: ZPipeline[Any, Throwable, Byte, Byte] = apply().compress
}

final class Bzip2Compressor private (blockSize: Option[Bzip2BlockSize]) extends Compressor {

override def compress(implicit trace: Trace): ZPipeline[Any, Throwable, Byte, Byte] =
viaOutputStreamByte { outputStream =>
blockSize match {
Expand All @@ -32,16 +36,20 @@ final class Bzip2Compressor private (blockSize: Option[Bzip2BlockSize]) extends

object Bzip2Decompressor {

/** Makes a pipeline that accepts a Bzip2 compressed byte stream and produces a decompressed byte stream.
/** A [[Decompressor]] for Bzip2, based on the Apache Commons Compress library.
*
* @param chunkSize
* The maximum chunk size of the outgoing ZStream. Defaults to `ZStream.DefaultChunkSize` (4KiB).
*/
def make(chunkSize: Int = ZStream.DefaultChunkSize): Bzip2Decompressor =
def apply(chunkSize: Int = ZStream.DefaultChunkSize): Bzip2Decompressor =
new Bzip2Decompressor(chunkSize)

/** See [[apply]] and [[Decompressor.decompress]]. */
def decompress: ZPipeline[Any, Throwable, Byte, Byte] = apply().decompress
}

final class Bzip2Decompressor private (chunkSize: Int) extends Decompressor {

override def decompress(implicit trace: Trace): ZPipeline[Any, Throwable, Byte, Byte] =
// BrotliInputStream.read does its best to read as many bytes as requested; no buffering needed.
viaInputStreamByte(chunkSize)(new BZip2CompressorInputStream(_))
Expand Down
8 changes: 4 additions & 4 deletions bzip2/src/test/scala/zio/compress/Bzip2Spec.scala
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,15 @@ object Bzip2Spec extends ZIOSpecDefault {
for {
obtained <- ZStream
.fromChunk(clear)
.via(Bzip2Compressor.make().compress)
.via(Bzip2Compressor.compress)
.runCollect
} yield assertTrue(compressed == obtained)
},
test("bzip2 decompress") {
for {
obtained <- ZStream
.fromChunk(compressed)
.via(Bzip2Decompressor.make().decompress)
.via(Bzip2Decompressor.decompress)
.runCollect
} yield assertTrue(clear == obtained)
},
Expand All @@ -38,8 +38,8 @@ object Bzip2Spec extends ZIOSpecDefault {
obtained <- ZStream
.fromChunk(genBytes)
.rechunk(chunkSize)
.via(Bzip2Compressor.make().compress)
.via(Bzip2Decompressor.make().decompress)
.via(Bzip2Compressor.compress)
.via(Bzip2Decompressor.decompress)
.runCollect
} yield assertTrue(obtained == genBytes)
}
Expand Down
2 changes: 2 additions & 0 deletions core/src/main/scala/zio/compress/ArchiveSingleFile.scala
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ final class ArchiveSingleFileCompressor[Size[A] <: Option[A]] private (
archiver: Archiver[Size],
entry: ArchiveEntry[Size, Any],
) extends Compressor {

override def compress(implicit trace: Trace): ZPipeline[Any, Throwable, Byte, Byte] =
ZPipeline.fromFunction { stream =>
ZStream((entry, stream)).via(archiver.archive)
Expand All @@ -30,6 +31,7 @@ object ArchiveSingleFileCompressor {
final class ArchiveSingleFileDecompressor[Size[A] <: Option[A], Underlying] private (
unarchiver: Unarchiver[Size, Underlying]
) extends Decompressor {

override def decompress(implicit trace: Trace): ZPipeline[Any, Throwable, Byte, Byte] =
ZPipeline.fromFunction { stream =>
stream
Expand Down
8 changes: 8 additions & 0 deletions core/src/main/scala/zio/compress/Archiver.scala
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,15 @@ package zio.compress
import zio._
import zio.stream.{ZPipeline, ZStream}

/** An archiver makes pipelines that accept a stream of archive entries, and produce a byte stream of an archive.
*
* @tparam Size
* Either a `Some` when the archive entries require the uncompressed size, or `Option` when the archive entries do
* not require the uncompressed size.
*/
trait Archiver[-Size[A] <: Option[A]] extends Serializable {

/** Makes a pipeline that accepts a stream of archive entries, and produces a byte stream of an archive. */
def archive(implicit
trace: Trace
): ZPipeline[Any, Throwable, (ArchiveEntry[Size, Any], ZStream[Any, Throwable, Byte]), Byte]
Expand Down
10 changes: 8 additions & 2 deletions core/src/main/scala/zio/compress/Compressor.scala
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,17 @@ import zio.stream._
import zio.Trace

trait Compressor extends Serializable {

/** A pipeline that takes a raw byte stream and produces a compressed byte stream. */
def compress(implicit trace: Trace): ZPipeline[Any, Throwable, Byte, Byte]
}

object Compressor {
object Compressor extends Serializable {

/** A compressor that does nothing; it passes all bytes through unchanged. */
def empty: Compressor = new Compressor {
override def compress(implicit trace: Trace): ZPipeline[Any, Nothing, Byte, Byte] = ZPipeline.identity

override def compress(implicit trace: Trace): ZPipeline[Any, Nothing, Byte, Byte] =
ZPipeline.identity
}
}
8 changes: 7 additions & 1 deletion core/src/main/scala/zio/compress/Decompressor.scala
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,17 @@ import zio.Trace
import zio.stream._

trait Decompressor extends Serializable {

/** A pipeline that decompresses a byte stream to an uncompressed byte stream. */
def decompress(implicit trace: Trace): ZPipeline[Any, Throwable, Byte, Byte]
}

object Decompressor {

/** A decompressor that does nothing; it passes all bytes through unchanged. */
def empty: Decompressor = new Decompressor {
override def decompress(implicit trace: Trace): ZPipeline[Any, Nothing, Byte, Byte] = ZPipeline.identity

override def decompress(implicit trace: Trace): ZPipeline[Any, Nothing, Byte, Byte] =
ZPipeline.identity
}
}
10 changes: 10 additions & 0 deletions core/src/main/scala/zio/compress/Unarchiver.scala
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,17 @@ package zio.compress
import zio.Trace
import zio.stream._

/** An unarchiver makes pipelines that accept the byte stream of an archive, and produce a stream of archive entries.
*
* @tparam Size
* Either a `Some` when the archive entries have a known uncompressed size, `None` when the archive entries _do not_
* have the uncompressed size, or `Option` when the archive entries _might_ have the uncompressed size.
* @tparam Underlying
* The archive entries from the underlying archiving library.
*/
trait Unarchiver[Size[A] <: Option[A], Underlying] extends Serializable {

/** Make a pipelines that accepts the byte stream of an archive, and produces a stream of archive entries. */
def unarchive(implicit
trace: Trace
): ZPipeline[Any, Throwable, Byte, (ArchiveEntry[Size, Underlying], ZStream[Any, Throwable, Byte])]
Expand Down
Loading