Init repo

.formatter.exs

@@ -0,0 +1,4 @@
+# Used by "mix format"
+[
+  inputs: ["{mix,.formatter}.exs", "{bench,config,lib,test}/**/*.{ex,exs}"]
+]

.github/workflows/ci.yml

@@ -0,0 +1,40 @@
+name: CI
+on:
+  push:
+    branches:
+      - main
+
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  test:
+    name: Run tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Elixir and Erlang versions
+        uses: erlef/setup-beam@v1
+        id: setup-elixir
+        with:
+          version-type: strict
+          version-file: .tool-versions
+
+      - name: Restore the cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            deps
+            _build
+            dialyzer
+          key: |
+            ${{ runner.os }}-${{ steps.setup-elixir.outputs.elixir-version }}-${{ steps.setup-elixir.outputs.otp-version }}-mixlockhash-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }}
+          restore-keys: |
+            ${{ runner.os }}-${{ steps.setup-elixir.outputs.elixir-version }}-${{ steps.setup-elixir.outputs.otp-version }}-mixlockhash-
+
+      - name: Run CI
+        run: |
+          mix ci

.gitignore

@@ -0,0 +1,32 @@
+# The directory Mix will write compiled artifacts to.
+/_build/
+
+# If you run "mix test --cover", coverage assets end up here.
+/cover/
+
+# The directory Mix downloads your dependencies sources to.
+/deps/
+
+# Where third-party dependencies like ExDoc output generated docs.
+/doc/
+
+# Ignore .fetch files in case you like to edit your project deps locally.
+/.fetch
+
+# If the VM crashes, it generates a dump, let's ignore it too.
+erl_crash.dump
+
+# Also ignore archive artifacts (built via "mix archive.build").
+*.ez
+
+# Ignore package tarball (built via "mix hex.build").
+z_stream-*.tar
+
+# Temporary files, for example, from tests.
+/tmp/
+
+# Ignore Elixir Language Server files
+.elixir_ls
+
+# Dialyzer generated PLT files
+/dialyzer

.tool-versions

@@ -0,0 +1,2 @@
+erlang 26.2.5.1
+elixir 1.17.1-otp-26

README.md

@@ -0,0 +1,35 @@
+# ZStream
+
+Compress and decompress data using various algorithms.
+
+## Installation
+
+Add `z_stream` to your list of dependencies in `mix.exs`:
+
+```elixir
+def deps do
+  [
+    {:z_stream, git: "https://github.com/coherentpath/z_stream", tag: "v*.*.*"}
+  ]
+end
+```
+
+## Usage
+
+Compression algorithms are implemented as modules that adhere to the `ZStream`
+behaviour. Simply pass in the module to the main `ZStream.compress/2` and 
+`ZStream.decompress/2` functions.
+
+```elixir
+alias ZStream.LZ4
+
+data = ["foo", "bar", "baz"]
+data = ZStream.compress(data, LZ4)
+data = ZStream.decompress(data, LZ4)
+Enum.into(data, "")
+```
+
+As of now, the following compression algorithms are available:
+
+- [LZ4](https://lz4.org/) via `ZStream.LZ4`
+- [ZStandard](https://facebook.github.io/zstd/) via `ZStream.Zstandard`

bench/bench.exs

@@ -0,0 +1,92 @@
+alias ZStream.LZ4
+alias ZStream.Zstandard
+
+raw_large_binary = StreamData.binary(length: 128_000)
+raw_large_binary = Enum.take(raw_large_binary, 100)
+
+raw_medium_binary = StreamData.binary(length: 32_000)
+raw_medium_binary = Enum.take(raw_medium_binary, 100)
+
+raw_small_binary = StreamData.binary(length: 8000)
+raw_small_binary = Enum.take(raw_small_binary, 100)
+
+Benchee.run(
+  %{
+    "LZ4.compress large" => fn -> raw_large_binary |> ZStream.compress(LZ4) |> Stream.run() end,
+    "Zstandard.compress large" => fn ->
+      raw_large_binary |> ZStream.compress(Zstandard) |> Stream.run()
+    end
+  },
+  warmup: 4,
+  memory_time: 5
+)
+
+Benchee.run(
+  %{
+    "LZ4.compress medium" => fn -> raw_medium_binary |> ZStream.compress(LZ4) |> Stream.run() end,
+    "Zstandard.compress medium" => fn ->
+      raw_medium_binary |> ZStream.compress(Zstandard) |> Stream.run()
+    end
+  },
+  warmup: 4,
+  memory_time: 5
+)
+
+Benchee.run(
+  %{
+    "LZ4.compress small" => fn -> raw_small_binary |> ZStream.LZ4.compress([]) |> Stream.run() end,
+    "Zstandard.compress small" => fn ->
+      raw_small_binary |> ZStream.compress(Zstandard) |> Stream.run()
+    end
+  },
+  warmup: 4,
+  memory_time: 5
+)
+
+lz4_large_binary = ZStream.compress(raw_large_binary, LZ4)
+zstandard_large_binary = ZStream.compress(raw_large_binary, Zstandard)
+
+lz4_medium_binary = ZStream.compress(raw_medium_binary, LZ4)
+zstandard_medium_binary = ZStream.compress(raw_medium_binary, Zstandard)
+
+lz4_small_binary = ZStream.compress(raw_small_binary, LZ4)
+zstandard_small_binary = ZStream.compress(raw_small_binary, Zstandard)
+
+Benchee.run(
+  %{
+    "LZ4.decompress large" => fn ->
+      lz4_large_binary |> ZStream.decompress(LZ4) |> Stream.run()
+    end,
+    "Zstandard.decompress large" => fn ->
+      zstandard_large_binary |> ZStream.decompress(Zstandard) |> Stream.run()
+    end
+  },
+  warmup: 4,
+  memory_time: 5
+)
+
+Benchee.run(
+  %{
+    "LZ4.decompress medium" => fn ->
+      lz4_medium_binary |> ZStream.decompress(LZ4) |> Stream.run()
+    end,
+    "Zstandard.decompress medium" => fn ->
+      zstandard_medium_binary |> ZStream.decompress(Zstandard) |> Stream.run()
+    end
+  },
+  warmup: 4,
+  memory_time: 5
+)
+
+Benchee.run(
+  %{
+    "LZ4.decompress small" => fn ->
+      lz4_small_binary |> ZStream.decompress(LZ4) |> Stream.run()
+    end,
+    "Zstandard.decompress small" => fn ->
+      zstandard_small_binary |> ZStream.decompress(Zstandard) |> Stream.run()
+    end
+  },
+  warmup: 4,
+  memory_time: 5
+)

lib/z_stream.ex

@@ -0,0 +1,47 @@
+defmodule ZStream do
+  @moduledoc """
+  A module to compress and decompress data using various algorithms.
+  """
+
+  @type t :: module()
+
+  @doc """
+  A callback to compress an enumerable.
+  """
+  @callback compress(Enumerable.t(), keyword()) :: Enumerable.t()
+
+  @doc """
+  A callback to decompress an enumerable.
+  """
+  @callback decompress(Enumerable.t(), keyword()) :: Enumerable.t()
+
+  defmodule CompressionError do
+    @moduledoc """
+    A general compression error.
+    """
+
+    defexception [:message]
+  end
+
+  ################################
+  # Public API
+  ################################
+
+  @doc """
+  Compress an enumerable using the provided implementation.
+  """
+  @spec compress(Enumerable.t(), t()) :: Enumerable.t()
+  @spec compress(Enumerable.t(), t(), keyword()) :: Enumerable.t()
+  def compress(stream, impl, opts \\ []) when is_atom(impl) do
+    impl.compress(stream, opts)
+  end
+
+  @doc """
+  Decompress an enumerable using the provided implementation.
+  """
+  @spec decompress(Enumerable.t(), t()) :: Enumerable.t()
+  @spec decompress(Enumerable.t(), t(), keyword()) :: Enumerable.t()
+  def decompress(stream, impl, opts \\ []) when is_atom(impl) do
+    impl.decompress(stream, opts)
+  end
+end

lib/z_stream/lz4.ex

@@ -0,0 +1,72 @@
+defmodule ZStream.LZ4 do
+  @moduledoc """
+  An implementation of `ZStream` using [LZ4](https://lz4.org/).
+  """
+
+  @behaviour ZStream
+
+  ################################
+  # ZStream Callbacks
+  ################################
+
+  @impl ZStream
+  def compress(stream, _opts) do
+    Stream.transform(
+      stream,
+      fn -> do_compress_start() end,
+      &do_compress/2,
+      &do_compress_end/1,
+      &do_compress_exit/1
+    )
+  end
+
+  @impl ZStream
+  def decompress(stream, _opts) do
+    Stream.transform(
+      stream,
+      fn -> do_decompress_start() end,
+      &do_decompress/2,
+      &do_decompress_exit/1
+    )
+  end
+
+  ################################
+  # Private API
+  ################################
+
+  defp do_compress_start do
+    ref = :lz4f.create_compression_context()
+    {:start, ref}
+  end
+
+  defp do_compress(elem, {:start, ref}) do
+    begin = :lz4f.compress_begin(ref)
+    elem = :lz4f.compress_update(ref, elem)
+    {:erlang.iolist_to_iovec([begin, elem]), ref}
+  end
+
+  defp do_compress(elem, ref) do
+    elem = :lz4f.compress_update(ref, elem)
+    {:erlang.iolist_to_iovec(elem), ref}
+  end
+
+  defp do_compress_end(ref) do
+    finish = :lz4f.compress_end(ref)
+    {:erlang.iolist_to_iovec(finish), ref}
+  end
+
+  defp do_compress_exit(ref) do
+    :lz4f.compress_end(ref)
+  end
+
+  defp do_decompress_start do
+    :lz4f.create_decompression_context()
+  end
+
+  defp do_decompress(elem, ref) do
+    elem = :lz4f.decompress(ref, elem)
+    {:erlang.iolist_to_iovec(elem), ref}
+  end
+
+  defp do_decompress_exit(_ref), do: :ok
+end