diff --git a/c_src/spawner.c b/c_src/spawner.c index 6bb8e3e..df75486 100644 --- a/c_src/spawner.c +++ b/c_src/spawner.c @@ -146,9 +146,19 @@ static int exec_process(char const *bin, char *const *args, int socket, _exit(FORK_EXEC_FAILURE); } - if (strcmp(stderr_str, "consume") == 0) { + if (strcmp(stderr_str, "redirect_to_stdout") == 0) { close(STDERR_FILENO); close(r_cmderr); + close(w_cmderr); + + if (dup2(w_cmdout, STDERR_FILENO) < 0) { + perror("[spawner] failed to redirect stderr to stdout"); + _exit(FORK_EXEC_FAILURE); + } + } else if (strcmp(stderr_str, "consume") == 0) { + close(STDERR_FILENO); + close(r_cmderr); + if (dup2(w_cmderr, STDERR_FILENO) < 0) { perror("[spawner] failed to dup to stderr"); _exit(FORK_EXEC_FAILURE); diff --git a/lib/exile.ex b/lib/exile.ex index 83422d5..3a523cd 100644 --- a/lib/exile.ex +++ b/lib/exile.ex @@ -119,10 +119,18 @@ defmodule Exile do "X 250 X\n" ``` + With stderr set to :redirect_to_stdout + + ``` + iex> Exile.stream!(["sh", "-c", "echo foo; echo bar >> /dev/stderr"], stderr: :redirect_to_stdout) + ...> |> Enum.into("") + "foo\nbar\n" + ``` + With stderr set to :consume ``` - iex> Exile.stream!(["sh", "-c", "echo foo\necho bar >> /dev/stderr"], stderr: :consume) + iex> Exile.stream!(["sh", "-c", "echo foo; echo bar >> /dev/stderr"], stderr: :consume) ...> |> Enum.to_list() [{:stdout, "foo\n"}, {:stderr, "bar\n"}] ``` @@ -130,7 +138,7 @@ defmodule Exile do With stderr set to :disable ``` - iex> Exile.stream!(["sh", "-c", "echo foo\necho bar >> /dev/stderr"], stderr: :disable) + iex> Exile.stream!(["sh", "-c", "echo foo; echo bar >> /dev/stderr"], stderr: :disable) ...> |> Enum.to_list() ["foo\n"] ``` @@ -195,13 +203,16 @@ defmodule Exile do Chunk size can be less than the `max_chunk_size` depending on the amount of data available to be read. Defaults to `65_535` - * `stderr` - different ways to handle stderr stream. possible values `:console`, `:disable`, `:stream`. + * `stderr` - different ways to handle stderr stream. 1. `:console` - stderr output is redirected to console (Default) - 2. `:disable` - stderr output is redirected `/dev/null` suppressing all output - 3. `:consume` - connects stderr for the consumption. The output stream will contain stderr + 2. `:redirect_to_stdout` - stderr output is redirected to stdout + 3. `:disable` - stderr output is redirected `/dev/null` suppressing all output + 4. `:consume` - connects stderr for the consumption. The output stream will contain stderr data along with stdout. Stream data will be either `{:stdout, iodata}` or `{:stderr, iodata}` to differentiate different streams. See example below. + See [`:stderr`](`m:Exile.Process#module-stderr`) for more details and issues associated with them. + * `ignore_epipe` - When set to true, reader can exit early without raising error. Typically writer gets `EPIPE` error on write when program terminate prematurely. With `ignore_epipe` set to true this error will be ignored. This can be used to @@ -221,6 +232,14 @@ defmodule Exile do |> Stream.run() ``` + Stream with stderr redirected to stdout + + ``` + Exile.stream!(["sh", "-c", "echo foo; echo bar >> /dev/stderr"], stderr: :redirect_to_stdout) + |> Stream.map(&IO.write/1) + |> Stream.run() + ``` + Stream with stderr ``` @@ -257,7 +276,7 @@ defmodule Exile do @spec stream!(nonempty_list(String.t()), input: Enum.t() | collectable_func(), exit_timeout: timeout(), - stderr: :console | :disable | :consume, + stderr: :console | :redirect_to_stdout | :disable | :consume, ignore_epipe: boolean(), max_chunk_size: pos_integer() ) :: Exile.Stream.t() @@ -278,7 +297,7 @@ defmodule Exile do @spec stream(nonempty_list(String.t()), input: Enum.t() | collectable_func(), exit_timeout: timeout(), - stderr: :console | :disable | :consume, + stderr: :console | :redirect_to_stdout | :disable | :consume, ignore_epipe: boolean(), max_chunk_size: pos_integer() ) :: Exile.Stream.t() diff --git a/lib/exile/process.ex b/lib/exile/process.ex index 80ebae2..566f8a8 100644 --- a/lib/exile/process.ex +++ b/lib/exile/process.ex @@ -104,25 +104,84 @@ defmodule Exile.Process do ### Pipe Operations - Pipe owner can read or write date to the owned pipe. `:stderr` by - default is connected to console, data written to stderr will appear on - the console. You can enable reading stderr by passing `stderr: :consume` - during process creation. - - Special function `Exile.Process.read_any/2` can be used to read - from either stdout or stderr whichever has the data available. - - All Pipe operations blocks the caller to have blocking as natural - back-pressure and to make the API simple. This is an important - feature of Exile, that is the ability to block caller when the stdio - buffer is full, exactly similar to how programs works on the shell - with pipes between then `cat larg-file | grep "foo"`. Internally it - does not block the Exile process or VM (which is typically the case - with NIF calls). Because of this user can make concurrent read, - write to different pipes from separate processes. Internally Exile - uses asynchronous IO APIs to avoid blocking of VM or VM process. - - Reading from stderr + Only Pipe owner can read or write date to the owned pipe. + All Pipe operations (read/write) blocks the caller as a mechanism + to put back-pressure, and this also makes the API simpler. + This is same as how command-line programs works on the shell, + along with pipes in-between, Example: `cat larg-file | grep "foo"`. + Internally Exile uses asynchronous IO APIs to avoid blocking VM + (by default NIF calls blocks the VM scheduler), + so you can open several pipes and do concurrent IO operations without + blocking VM. + + + ### `stderr` + + by default is `:stderr` is connected to console, data written to + stderr will appear on the console. + + You can change the behavior by setting `:stderr`: + + 1. `:console` - stderr output is redirected to console (Default) + 2. `:redirect_to_stdout` - stderr output is redirected to stdout + 2. `:consume` - stderr output read separately, allowing you to consume it separately from stdout. See below for more details + 4. `:disable` - stderr output is redirected `/dev/null` suppressing all output. See below for more details. + + + ### Using `redirect_to_stdout` + + stderr data will be redirected to stdout. When you read stdout + you will see both stdout & stderr combined and you won't be + able differentiate stdout and stderr separately. + This is similar to `:stderr_to_stdout` option present in + [Ports](https://www.erlang.org/doc/apps/erts/erlang.html#open_port/2). + + > #### Unexpected Behaviors {: .warning} + > + > On many systems, `stdout` and `stderr` are separated. And between + > the source program to Exile, via the kernel, there are several places + > that may buffer data, even temporarily, before Exile is ready + > to read them. There is no enforced ordering of the readiness of + > these independent buffers for Exile to make use of. + > + > This can result in unexpected behavior, including: + > + > * mangled data, for example, UTF-8 characters may be incomplete + > until an additional buffered segment is released on the same + > source + > * raw data, where binary data sent on one source, is incompatible + > with data sent on the other source. + > * interleaved data, where what appears to be synchronous, is not + > + > In short, the two streams might be combined at arbitrary byte position + > leading to above mentioned issue. + > + > Most well-behaved command-line programs are unlikely to exhibit + > this, but you need to be aware of the risk. + > + > A good example of this unexpected behavior is streaming JSON from + > an external tool to Exile, where normal JSON output is expected on + > stdout, and errors or warnings via stderr. In the case of an + > unexpected error, the stdout stream could be incomplete, or the + > stderr message might arrive before the closing data on the stdout + > stream. + + + ### Using `consume` + + stderr data can be consumed separately using + `Exile.Process.read_stderr/2`. Special function + `Exile.Process.read_any/2` can be used to read from either stdout or + stderr whichever has the data available. See the examples for more + details. + + + > #### Unexpected Behaviors {: .warning} + > + > When set, the `stderr` output **MUST** be consumed to + > avoid blocking the external program when stderr buffer is full. + + Reading from stderr using `read_stderr` ``` # write "Hello" to stdout and "World" to stderr @@ -301,12 +360,14 @@ defmodule Exile.Process do These can be accessed in the external program * `stderr` - different ways to handle stderr stream. - possible values `:console`, `:disable`, `:stream`. 1. `:console` - stderr output is redirected to console (Default) - 2. `:disable` - stderr output is redirected `/dev/null` suppressing all output - 3. `:consume` - connects stderr for the consumption. When set to stream the output must be consumed to + 2. `:redirect_to_stdout` - stderr output is redirected to stdout + 3. `:disable` - stderr output is redirected `/dev/null` suppressing all output + 4. `:consume` - connects stderr for the consumption. When set, the stderr output must be consumed to avoid external program from blocking. + See [`:stderr`](#module-stderr) for more details and issues associated with them + Caller of the process will be the owner owner of the Exile Process. And default owner of all opened pipes. diff --git a/lib/exile/process/exec.ex b/lib/exile/process/exec.ex index 4e5dc4b..b783c43 100644 --- a/lib/exile/process/exec.ex +++ b/lib/exile/process/exec.ex @@ -52,7 +52,7 @@ defmodule Exile.Process.Exec do cmd_with_args: nonempty_list(), cd: charlist, env: env, - stderr: :console | :disable | :consume + stderr: :console | :redirect_to_stdout | :disable | :consume }} | {:error, String.t()} def normalize_exec_args(cmd_with_args, opts) do @@ -192,18 +192,19 @@ defmodule Exile.Process.Exec do end end - @spec normalize_stderr(stderr :: :console | :disable | :consume | nil) :: - {:ok, :console | :disable | :consume} | {:error, String.t()} + @spec normalize_stderr(stderr :: :console | :redirect_to_stdout | :disable | :consume | nil) :: + {:ok, :console | :redirect_to_stdout | :disable | :consume} | {:error, String.t()} defp normalize_stderr(stderr) do case stderr do nil -> {:ok, :console} - stderr when stderr in [:console, :disable, :consume] -> + stderr when stderr in [:redirect_to_stdout, :console, :disable, :consume] -> {:ok, stderr} _ -> - {:error, ":stderr must be an atom and one of :console, :disable, :consume"} + {:error, + ":stderr must be an atom and one of :redirect_to_stdout, :console, :disable, :consume"} end end diff --git a/lib/exile/process/state.ex b/lib/exile/process/state.ex index 60f637f..c8b1f5c 100644 --- a/lib/exile/process/state.ex +++ b/lib/exile/process/state.ex @@ -9,7 +9,7 @@ defmodule Exile.Process.State do @type read_mode :: :stdout | :stderr | :stdout_or_stderr - @type stderr_mode :: :console | :disable | :consume + @type stderr_mode :: :console | :redirect_to_stdout | :disable | :consume @type pipes :: %{ stdin: Pipe.t(), diff --git a/lib/exile/stream.ex b/lib/exile/stream.ex index 0a4d23e..0425231 100644 --- a/lib/exile/stream.ex +++ b/lib/exile/stream.ex @@ -297,11 +297,12 @@ defmodule Exile.Stream do nil -> {:ok, :console} - stderr when stderr in [:console, :disable, :consume] -> + stderr when stderr in [:console, :redirect_to_stdout, :disable, :consume] -> {:ok, stderr} _ -> - {:error, ":stderr must be an atom and one of :console, :disable, :consume"} + {:error, + ":stderr must be an atom and one of :console, :redirect_to_stdout, :disable, :consume"} end end diff --git a/mix.lock b/mix.lock index 1537d36..6300303 100644 --- a/mix.lock +++ b/mix.lock @@ -1,14 +1,14 @@ %{ "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, - "credo": {:hex, :credo, "1.7.6", "b8f14011a5443f2839b04def0b252300842ce7388f3af177157c86da18dfbeea", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "146f347fb9f8cbc5f7e39e3f22f70acbef51d441baa6d10169dd604bfbc55296"}, + "credo": {:hex, :credo, "1.7.7", "771445037228f763f9b2afd612b6aa2fd8e28432a95dbbc60d8e03ce71ba4446", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8bc87496c9aaacdc3f90f01b7b0582467b69b4bd2441fe8aae3109d843cc2f2e"}, "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"}, - "elixir_make": {:hex, :elixir_make, "0.8.3", "d38d7ee1578d722d89b4d452a3e36bcfdc644c618f0d063b874661876e708683", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.0", [hex: :certifi, repo: "hexpm", optional: true]}], "hexpm", "5c99a18571a756d4af7a4d89ca75c28ac899e6103af6f223982f09ce44942cc9"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.40", "f3534689f6b58f48aa3a9ac850d4f05832654fe257bf0549c08cc290035f70d5", [:mix], [], "hexpm", "cdb34f35892a45325bad21735fadb88033bcb7c4c296a999bde769783f53e46a"}, + "elixir_make": {:hex, :elixir_make, "0.8.4", "4960a03ce79081dee8fe119d80ad372c4e7badb84c493cc75983f9d3bc8bde0f", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.0", [hex: :certifi, repo: "hexpm", optional: true]}], "hexpm", "6e7f1d619b5f61dfabd0a20aa268e575572b542ac31723293a4c1a567d5ef040"}, "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, - "ex_doc": {:hex, :ex_doc, "0.33.0", "690562b153153c7e4d455dc21dab86e445f66ceba718defe64b0ef6f0bd83ba0", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "3f69adc28274cb51be37d09b03e4565232862a4b10288a3894587b0131412124"}, + "ex_doc": {:hex, :ex_doc, "0.34.1", "9751a0419bc15bc7580c73fde506b17b07f6402a1e5243be9e0f05a68c723368", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "d441f1a86a235f59088978eff870de2e815e290e44a8bd976fe5d64470a4c9d2"}, "excoveralls": {:hex, :excoveralls, "0.18.1", "a6f547570c6b24ec13f122a5634833a063aec49218f6fff27de9df693a15588c", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "d65f79db146bb20399f23046015974de0079668b9abb2f5aac074d078da60b8d"}, "file_system": {:hex, :file_system, "1.0.0", "b689cc7dcee665f774de94b5a832e578bd7963c8e637ef940cd44327db7de2cd", [:mix], [], "hexpm", "6752092d66aec5a10e662aefeed8ddb9531d79db0bc145bb8c40325ca1d8536d"}, - "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"}, + "jason": {:hex, :jason, "1.4.3", "d3f984eeb96fe53b85d20e0b049f03e57d075b5acda3ac8d465c969a2536c17b", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "9a90e868927f7c777689baa16d86f4d0e086d968db5c05d917ccff6d443e58a3"}, "makeup": {:hex, :makeup, "1.1.2", "9ba8837913bdf757787e71c1581c21f9d2455f4dd04cfca785c70bbfff1a76a3", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cce1566b81fbcbd21eca8ffe808f33b221f9eee2cbc7a1706fc3da9ff18e6cac"}, "makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"}, "makeup_erlang": {:hex, :makeup_erlang, "1.0.0", "6f0eff9c9c489f26b69b61440bf1b238d95badae49adac77973cbacae87e3c2e", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "ea7a9307de9d1548d2a72d299058d1fd2339e3d398560a0e46c27dab4891e4d2"}, diff --git a/test/exile_test.exs b/test/exile_test.exs index 5f8e9ee..b4b6ea8 100644 --- a/test/exile_test.exs +++ b/test/exile_test.exs @@ -46,6 +46,53 @@ defmodule ExileTest do assert IO.iodata_to_binary(stderr) == "Hello World\n" end + test "stderr redirect_to_stdout" do + merged_output = + Exile.stream!( + [fixture("write_stderr.sh"), "Hello World"], + stderr: :redirect_to_stdout + ) + |> Enum.to_list() + |> IO.iodata_to_binary() + + assert merged_output == "Hello World\n" + end + + test "order must be preserved when stderr is redirect to stdout" do + merged_output = + Exile.stream!( + ["sh", "-c", "for s in $(seq 1 10); do echo stdout $s; echo stderr $s >&2; done"], + stderr: :redirect_to_stdout, + ignore_epipe: true + ) + |> Enum.to_list() + |> IO.iodata_to_binary() + |> String.trim() + + assert [ + "stdout 1", + "stderr 1", + "stdout 2", + "stderr 2", + "stdout 3", + "stderr 3", + "stdout 4", + "stderr 4", + "stdout 5", + "stderr 5", + "stdout 6", + "stderr 6", + "stdout 7", + "stderr 7", + "stdout 8", + "stderr 8", + "stdout 9", + "stderr 9", + "stdout 10", + "stderr 10" + ] == String.split(merged_output, "\n") + end + test "multiple streams" do script = """ for i in {1..1000}; do