From 6f5475f5a5c69675d577c41c6c169b7e761b0fb6 Mon Sep 17 00:00:00 2001 From: Minigugus <43109623+Minigugus@users.noreply.github.com> Date: Fri, 11 Jun 2021 02:19:04 +0200 Subject: [PATCH] Rewrite and refactor library * Add Response to the core (see #1) * Add middlewares supports * Add better examples * Drop old implementation --- README.md | 113 ++++++------ deno.ts | 160 +++++++++++------ examples/middlewares.ts | 57 ++++++ examples/showcase.ts | 50 ++++++ index.ts | 1 - mock.ts | 25 +++ mod.ts | 5 +- src/bazx.ts | 249 ------------------------- src/consumer.ts | 389 ++++++++++++++++++++++++++++++++++++++++ src/middleware.ts | 50 ++++++ src/mod.ts | 103 +---------- src/provider.ts | 10 ++ src/utils.ts | 62 ------- test/basics.ts | 9 - test/debug.js | 42 ----- test/env.js | 25 --- test/showcase.js | 35 ---- tsconfig.json | 6 +- 18 files changed, 749 insertions(+), 642 deletions(-) create mode 100644 examples/middlewares.ts create mode 100644 examples/showcase.ts delete mode 100644 index.ts create mode 100644 mock.ts delete mode 100644 src/bazx.ts create mode 100644 src/consumer.ts create mode 100644 src/middleware.ts create mode 100644 src/provider.ts delete mode 100644 src/utils.ts delete mode 100644 test/basics.ts delete mode 100644 test/debug.js delete mode 100644 test/env.js delete mode 100644 test/showcase.js diff --git a/README.md b/README.md index 5c606d8..bfcb113 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,8 @@ > 🐚️ [zx](https://github.com/google/zx) on 💊️ steroids -[![asciicast](https://asciinema.org/a/ydfYbBXFyDDyDOeSPormkjEo6.svg)](https://asciinema.org/a/ydfYbBXFyDDyDOeSPormkjEo6) +> You are seeing the code of an upcoming release. The `main` branch contains the latest released code. +> Code on this branch is still under discussion and documentation is not completed yet. ## Main differences with ZX @@ -12,109 +13,111 @@ * **0 dependencies** by default * **Plateform-agnostic** (not limited to *Deno* or *NodeJS*) * Extensible (enables **remote command execution** and thus **browser support**) + * **Middlewares** support (filter allowed commands, hack input/outputs, and much more) * [**Streams**](#streams) at the core for better performances ([no streaming support with zx](https://github.com/google/zx/issues/14#issuecomment-841672494)) - * Tree-shakable (usefull for **bundling**) + * No shell by default (the `|` in ``$`echo |`;`` is not evaluated by default) * Modern (TypeScript, Deno, ES2020) * **MIT** vs Apache license * Library only -### Streams - -Unlike zx, bazx doesn't gather outputs into possibly big strings by default. The main reason for that is to avoid out of memory, especially on embedded devices and with "loud" commands (for instance, ``await $`yes`;`` with zx *will* crash the process, but not with bazx). - ## Support * Deno: 🐥️ (working) * NodeJS: 🥚️ (not started yet) * QuickJS: 🥚️ (not started yet) * Browser: 🥚️ (not started yet) - * Mock: 🥚️ (not started yet) + * Mock: 🐣️ (started) ## Setup ### Deno ```js -import { $ } from 'https://deno.land/x/bazx/mod.ts'; +import $ from 'https://deno.land/x/bazx/deno.ts'; ``` As of now, only the `--allow-run` command is required at runtime. +### Isomorphic (for testing purpose) + +```js +import $ from 'https://deno.land/x/bazx/mock.ts'; +``` + +This implementation doesn't know how to spawn a process and thus always throw. +Intended to be used along with middlewares in tests or sandboxes for instance. + ### (Bring Your Own Runtime) ```ts -// `index.ts` is isomorphic, `mod.ts` is Deno only -import { create } from 'https://deno.land/x/bazx/index.ts'; - -const $ = create({ - exec( - cmd: [string, ...string[]], - streams: { - stdin: ReadableStream, - stdout: WritableStream, - stderr: WritableStream - } - ): PromiseLike<{ success: boolean, code: number }> { - // Create thread here +import { createBazx } from 'https://deno.land/x/bazx/mod.ts'; + +const $ = createBazx(( + cmd: [string, ...string[]], + options: { + cwd?: string, + env?: Record, + stdin?: ReadableStream, + stdout?: WritableStream, + stderr?: WritableStream, + signal?: AbortSignal, } -}, { /* Default options */ }); +): PromiseLike<{ code: number }> => { + // Spawn commands here. + // + // CAUTION: This function is reponsible for closing stdin/stdout/stderr. + // Missing to do so will result in deadlocks. +}, { /* Default options (logging, colors, and so on) */ }); ``` ## Usage -See the [`test`](test/) folder for more complete examples - -### The `$` tagged template function +See the [`examples`](examples/) folder for more examples -Prepare a command. The process will spawn only when the returned object will be awaited (`then` called). The returned object as `stdin`, `stdout` and `stderr` properties, that are streams the user can use to communicated with the process. Also, it is possible to call the `pipe(cmdOrTransfertStream: NativeCommand | PipeCommand | TransformStream)` function to pipe the *this* command into the provided `cmdOrTransfertStream` stream, or to `stdin` stream of the provided command. +### Middlewares -```js -$`echo Never called`; // (never executed - missing `await`) +Middlewares are hooks that runs a process get spawned, so that it can for instance +dynamically hack streams, change the command line, working directories and environment +variabes, never really spawn a process, spawn a process twice, and so on. -await $`echo Hi!`; // $ echo Hi! +For instance, this really simple middleware wraps processes with `time`, so that some process meta are displayed: -const cmd = $`echo Only invoked once`; -(await cmd) === (await cmd); // => true +```typescript +import { BazxMiddleware } from 'https://deno.land/x/bazx/mod.ts'; -await $`env`.pipe($`grep PATH`); // $ env | grep PATH +export const timeWrapperMiddleware: BazxMiddleware = + exec => (cmd, options) => exec(['time', ...cmd], options); ``` -The `$` function can be obtained using the `create` function, or from a third party module that wrap this function. For instance, the [`deno.ts`](deno.ts) module expose a `createDeno` function, which is a wrapper function around `create`. - -See [`test/debug.js`](test/debug.js) for a pratical overview. +Then, it can be applied with the `$.with` function: -### `stdout`, `stderr` and `collect` exports +```typescript +import $ from 'https://deno.land/x/bazx/deno.ts'; -Utilities to read respectively `stdout`, `stderr` and both from the command passed as argument: - -```js -import { stdout } from 'https://deno.land/x/bazx/index.ts'; +const $$ = $.with(timeWrapperMiddleware); -console.log(await stdout($`echo Hello world!`)); +await $$`echo Hello world!`.status -// => { success: true, code: 0, stdout: "Hello world!" } +// $ echo Hello world! +// Hello world! +// 0.00user 0.00system 0:00.00elapsed 100%CPU (0avgtext+0avgdata 1936maxresident)k +// 0inputs+0outputs (0major+73minor)pagefaults 0swaps ``` -### Environment variables - -Environment variables are inherited from the runtime. For instance, with *Deno*, you must use `Deno.env` with the appropriate permission to change environment variables passed to the child process (`Deno.run` inherits `Deno.env` by default). - -Environment variables are still under discussion. Suggestions welcomed :slightly_smiling_face:. - -## WIP - -This project is a work in progress for now; bugs and API change are expected. +As you can may have noticed, only the original command is printed to the user, +not the updated one with `time`. This way, middlewares are fully transparents to the user. -Please fill an issue for any bug you encounter and open a discussion for any question or suggestion. :wink: +Each `.with` call returns a new `$` instance that use the config of the parent (`exec`, `options` and middlewares), +so that multiple `.with` calls can be chained. ### TODO - * [ ] Improve docs + * [ ] Improve docs (README + JSDoc) * [ ] Rollup for NodeJS and browser builds * [ ] Add more runtime support (NodeJS at least) * [ ] Fix bugs (some complex use case doesn't work yet) - * [ ] Dynamic config update (like `set` in bash (enable/disable logs, etc.)) - * [ ] `NO_COLOR` support (for CI/CD) + * [X] Dynamic config update (like `set` in bash (enable/disable logs, etc.)) + * [X] `NO_COLOR` support (for CI/CD) * [ ] Pipelines starting with a stream * [ ] `stderr` pipes * [ ] Add benchmarks, improve perfs (audit WHATWG streams perfs) diff --git a/deno.ts b/deno.ts index 6431c07..2cf9b6a 100644 --- a/deno.ts +++ b/deno.ts @@ -1,62 +1,12 @@ -/// +// Bazx implementation for Deno. -import { create, BazxOptions } from './mod.ts'; +/// -export const $ = createDeno(); +export * from './mod.ts' -export function createDeno(options?: Partial) { - return create({ - async exec(cmd, { stdin, stdout, stderr }) { - const proc = Deno.run({ - cmd, - stdin: stdin ? 'piped' : 'null', - stdout: stdout ? 'piped' : 'null', - stderr: stderr ? 'piped' : 'null' - }); - const procIn = stdin && stdin.pipeTo(streamCopyFromStream(proc.stdin!)); - const procOut = stdout && streamCopyToStream(proc.stdout!).pipeTo(stdout); - const procErr = stderr && streamCopyToStream(proc.stderr!).pipeTo(stderr); - const result = await proc.status(); - await Promise.all([ - procIn, - procOut, - procErr - ]) - proc.close(); - return result; - }, - get stdout() { - return streamCopyFromStream({ write: chunk => Deno.stdout.write(chunk) }); - }, - get stderr() { - const decoder = new TextDecoder(); - const encoder = new TextEncoder(); - return new WritableStream({ - async write(chunk) { - await Deno.stderr.write(encoder.encode(`\x1B[31m${decoder.decode(chunk)}\x1B[0m`)); - } - }, new ByteLengthQueuingStrategy({ - highWaterMark: 16_640 - })); - } - }, options); -} +import type { BazxExec, BazxOptions } from './mod.ts'; -function streamCopyFromStream(writer: Deno.Writer & Partial) { - return new WritableStream({ - async write(chunk) { - await writer.write(chunk); - }, - close() { - writer.close?.(); - }, - abort() { - writer.close?.(); - } - }, new ByteLengthQueuingStrategy({ - highWaterMark: 16_640 - })); -} +import { createBaxz } from './mod.ts'; function streamCopyToStream(reader: Deno.Reader & Deno.Closer) { const buffer = new Uint8Array(16_640); @@ -66,7 +16,7 @@ function streamCopyToStream(reader: Deno.Reader & Deno.Closer) { try { while (controller.desiredSize! > 0) { if ((read = await reader.read(buffer.subarray(0, Math.min(buffer.byteLength, controller.desiredSize ?? Number.MAX_VALUE)))) === null) { - reader.close?.(); + reader.close(); controller.close(); return; } @@ -80,9 +30,103 @@ function streamCopyToStream(reader: Deno.Reader & Deno.Closer) { } }, cancel() { - reader.close?.(); + reader.close(); } }, new ByteLengthQueuingStrategy({ - highWaterMark: 16_640 + highWaterMark: 16640 })); } + +async function pipeReadableStream2Writer( + readable: ReadableStream, + writer: Deno.Writer & Deno.Closer +) { + const reader = readable.getReader(); + try { + let read: ReadableStreamReadResult; + while (!(read = await reader.read()).done) + if (!await writer.write(read.value!)) + break; + await reader.cancel(); + } catch (err) { + if (err instanceof Deno.errors.BrokenPipe) + await reader.releaseLock(); + else + await reader.cancel(err); + } finally { + try { + writer.close(); + } catch (ignored) { } + } +} + +export const exec: BazxExec = async function exec(cmd, { + cwd, + env, + stdin, + stdout, + stderr, + signal +} = {}) { + const process = Deno.run({ + cmd, + cwd, + env, + stdin: stdin ? 'piped' : 'null', + stdout: stdout ? 'piped' : 'null', + stderr: stderr ? 'piped' : 'null', + }); + signal?.addEventListener('abort', () => process.kill?.(9), { once: true }); + try { + const [{ code, signal: exitSignal }] = await Promise.all([ + process.status(), + stdin && pipeReadableStream2Writer(stdin, process.stdin!), + stdout && streamCopyToStream(process.stdout!).pipeTo(stdout), + stderr && streamCopyToStream(process.stderr!).pipeTo(stderr), + ]); + return { code, signal: exitSignal }; + } finally { + process.close(); + } +} + +export const options: BazxOptions = { + highWaterMark: 16640, + noColors: Deno.noColor, + log: chunk => Deno.stdout.writeSync(chunk) +}; + +export const $ = createBaxz(exec, options); + +export default $; + +Deno.test({ + name: 'everything works', + async fn() { + // @ts-ignore + const assert: (expr: unknown, msg: string) => asserts expr = (await import("https://deno.land/std/testing/asserts.ts")).assert; + // @ts-ignore + const { fail, assertEquals } = await import("https://deno.land/std/testing/asserts.ts"); + + const cmd = $`bash -c ${'echo Hello world! $(env | grep WTF) $(pwd)'}` + .cwd('/bin') + .pipe(new TransformStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('Someone said: ')); + } + })) + .env('WTF', 'it works') + .pipe($`sh -c ${'cat - $(realpath $(env | grep WTF))'}`) + .env('WTF', 'not_found') + .cwd('/'); + + try { + await cmd.text(); + fail("Should have thrown since cat should have failed"); + } catch (err) { + assertEquals(`Command ${cmd.command} exited with code 1`, err.message); + assert(err.response instanceof Response, "err.response is defined and is an instance of Response"); + assertEquals('Someone said: Hello world! WTF=it works /bin\n', await err.response.text()); + } + } +}); diff --git a/examples/middlewares.ts b/examples/middlewares.ts new file mode 100644 index 0000000..fa7f043 --- /dev/null +++ b/examples/middlewares.ts @@ -0,0 +1,57 @@ +import $, { filtered } from '../deno.ts'; + +let $safe = $ + .with(filtered( + cmd => ['echo'].includes(cmd[0]) // only the `echo` command is allowed to run + )) + +let text = await $safe`echo Hello world!`.text() + +let alwaysNull = null; +try { + alwaysNull = await $safe`rm -rf /`.status +} catch (err) { + console.error(err); +} + +let mixed$ = await $safe`echo From echo` // using $safe + .pipe($`sh -c ${'cat >&2; echo From sh'}`) // using $ + .text() + +// advanced usage +import { interceptCurl, swapStdoutStderr, shWrapper } from '../deno.ts'; + +let $$ = $ + .with(shWrapper) // last invoked + .with(interceptCurl) // first invoked + +let $swap = $$ + .with(swapStdoutStderr) + +let advanced = await ( + await $safe`echo ${new Date().toISOString()}` // passed as input to `curl` below + .pipe($$`curl http://detectportal.firefox.com/success.txt`) + .env('Content-Type', 'text/plain') // passed to `curl` above + + // transform `curl` output + .pipe(new TransformStream({ + transform(chunk, controller) { + for (let index = 0; index < chunk.length; index++) + if (chunk[index] === 10) + break; + else + chunk[index]++; + controller.enqueue(chunk); + } + })) + + // >&2 works thanks to `shWrapper` and is required to print to stdout because of `swapStdoutStderr` + .pipe($swap`cat - /notfound >&2`) +).text() // not calling `.text()` directly to avoid an error with `cat` that exit with code 1 + +console.table({ + text, + alwaysNull, + mixed$, + advanced +}) diff --git a/examples/showcase.ts b/examples/showcase.ts new file mode 100644 index 0000000..dc88cdf --- /dev/null +++ b/examples/showcase.ts @@ -0,0 +1,50 @@ +import $ from '../deno.ts'; + +let response = await $`echo Hi!`; + +let text = await $`echo Hello world!`.text() + +let [json] = await $`echo ${JSON.stringify(["Hello world!"])}`.json() + +let buffer = await $`echo Hello world!` + .pipe($`gzip`) + .arrayBuffer() + +let env = await $`env` // sees `WTF_A=wtf`, `WTF_B=b` + .env('WTF_A', 'erased') + .env('WTF_A', 'wtf') + .pipe($`sh -c ${ // sees `WTF_A=a`, `WTF_B=b` + 'echo \\"env\\" sees: >&2; grep WTF; echo \\"sh\\" sees: >&2; env | grep WTF' + }`) + .env('WTF_A', 'a') + .env('WTF_B', 'b') + .text() + +let cwd = await $`pwd` // runs in `/bin` + .cwd('/root') + .cwd('/bin') + .pipe($`sh -c ${'cat; pwd'}`) // runs in `/` + .cwd('/') + .text() + +let status = await $`sh -c ${'exit 42'}`.status + +let throws; +try { + throws = await $`grep nothing`.text() +} catch (err) { + if (!(err?.response instanceof Response)) + throw err; + throws = `Nothing found (exit code: ${err.response.status})` +} + +console.table({ + text, + json, + buffer: String(buffer), + env, + cwd, + status, + throws, + response: String(response), +}); diff --git a/index.ts b/index.ts deleted file mode 100644 index 7b02d4e..0000000 --- a/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './src/mod.ts'; diff --git a/mock.ts b/mock.ts new file mode 100644 index 0000000..68a94b0 --- /dev/null +++ b/mock.ts @@ -0,0 +1,25 @@ +// Isomorphic bazx implementation that never runs a command (always throw). +// Intended to be used along with middlewares. + +/// + +export * from './mod.ts' + +import type { BazxExec, BazxOptions } from './mod.ts'; + +import { createBaxz } from './mod.ts'; + +export const exec: BazxExec = async function exec(cmd, { stdin, stdout, stderr } = {}) { + await Promise.all([ + stdin?.cancel(), + stdout?.getWriter().close(), + stderr?.getWriter().close() + ]); + throw new Error(`${cmd[0]}: command not found`); // FIXME Return code 0 instead? +} + +export const options: BazxOptions = {}; + +export const $ = createBaxz(exec, options); + +export default $; diff --git a/mod.ts b/mod.ts index 563d866..3ccdf24 100644 --- a/mod.ts +++ b/mod.ts @@ -1,3 +1,4 @@ -export * from './index.ts'; +// This file expose isomorphic code only. +// Deno users are intented to import the `deno.ts` file instead. -export * from './deno.ts'; +export * from './src/mod.ts'; diff --git a/src/bazx.ts b/src/bazx.ts deleted file mode 100644 index 5ba84ed..0000000 --- a/src/bazx.ts +++ /dev/null @@ -1,249 +0,0 @@ -const RUN = Symbol(); - -export interface ExecutionResult { - success: boolean; - code: number; -} - -export interface BazxContext { - log(cmd: OpaqueCommand, streams: CommandStreamConfig): void; - exec(cmd: [string, ...string[]], streams: CommandStreamConfig): PromiseLike; - throw: boolean; -} - -export abstract class OpaqueCommand implements PromiseLike { - #context: BazxContext; - #completed: PromiseLike | null = null; - - #stdin: TransformStream | null = null; - #stdout: TransformStream | null = null; - #stderr: TransformStream | null = null; - - protected abstract [RUN](exec: BazxContext['exec'], streams: CommandStreamConfig): PromiseLike; - - // abstract readonly stdin: WritableStream; - // abstract readonly stdout: ReadableStream; - // abstract readonly stderr: ReadableStream; - - protected constructor( - context: BazxContext - ) { - this.#context = context; - } - - private get completed(): PromiseLike { - if (this.#completed === null) { - const streams = { - stdin: this.#stdin?.readable || null, - stdout: this.#stdout?.writable || null, - stderr: this.#stderr?.writable || null - }; - this.#context.log(this, streams); - this.#completed = this[RUN](this.#context.exec, streams).then(res => { - if (res.success || !this.#context.throw) - return res; - throw new Error(`Command "${this}" exited with code ${res.code}`); - }, err => { - throw new Error(`Command "${this}" failed: ${err?.message ?? err}`); - }); - } - return this.#completed; - } - - get stdin(): WritableStream { - if (this.#stdin === null) - this.#stdin = new TransformStream({}, strategy, strategy); - return this.#stdin.writable; - } - - get stdout(): ReadableStream { - if (this.#stdout === null) - this.#stdout = new TransformStream({}, strategy, strategy); - return this.#stdout.readable; - } - - get stderr(): ReadableStream { - if (this.#stderr === null) - this.#stderr = new TransformStream({}, strategy, strategy); - return this.#stderr.readable; - } - - then( - onfulfilled?: ((value: ExecutionResult) => TResult1 | PromiseLike) | null, - onrejected?: ((reason: any) => TResult2 | PromiseLike) | null - ): PromiseLike { - return this.completed.then(onfulfilled, onrejected); - } - - pipe(target: OpaqueCommand): PipedStdoutCommand; - pipe(target: TransformStream): TransformStreamCommand; - pipe(target: OpaqueCommand | TransformStream): PipedStdoutCommand | TransformStreamCommand { - if (target instanceof OpaqueCommand) - return new PipedStdoutCommand(this.#context, this, target); - return new TransformStreamCommand(this.#context, this, target); - } -} - -export class TransformStreamCommand extends OpaqueCommand { - #input: OpaqueCommand; - #output: ReadableStream | null = null; - #stdin: WritableStream; - #stdout: ReadableStream; - - public constructor( - context: BazxContext, - input: OpaqueCommand, - stream: TransformStream - ) { - super(context); - this.#input = input; - this.#stdin = stream.writable; - this.#stdout = stream.readable; - } - - protected async [RUN](exec: BazxContext['exec'], { stdin, stdout, stderr }: CommandStreamConfig): Promise { - const pipe = new TransformStream({}, strategy); - const inPipe = pipe.readable.pipeTo(this.#stdin); - const outPipe = stdout && this.#stdout.pipeTo(stdout); - const result = this.#input[RUN](exec, { - stdin, - stdout: pipe.writable, - stderr - }); - await Promise.all([inPipe, outPipe]); - return result; - } - - get stdin() { - return this.#input.stdin; - } - - get stdout() { - if (this.#output === null) { - const [first, second] = this.#stdout.tee(); - this.#stdout = first; - this.#output = second; - } - return this.#output; - } - - get stderr() { - return this.#input.stderr; - } - - toString() { - return `${this.#input} \x1B[35m|\x1B[0m \x1B[33m<#stream>\x1B[0m`; - } -} - -export class PipedStdoutCommand extends OpaqueCommand { - #left: OpaqueCommand; - #right: OpaqueCommand; - - public constructor( - context: BazxContext, - left: OpaqueCommand, - right: OpaqueCommand - ) { - super(context); - this.#left = left; - this.#right = right; - } - - protected async [RUN](exec: BazxContext['exec'], { stdin, stdout, stderr }: CommandStreamConfig): Promise { - const pipe = new TransformStream({}, strategy); - const writer = stderr?.getWriter(); - let left = 2; - const [, result] = await Promise.all([ - this.#left[RUN](exec, { - stdin, - stdout: pipe.writable, - stderr: !writer ? null : writableStreamFromWriter(writer, () => (--left || writer.close())) - }), - this.#right[RUN](exec, { - stdin: pipe.readable, - stdout, - stderr: !writer ? null : writableStreamFromWriter(writer, () => (--left || writer.close())) - }) - ]); - return result; - } - - get left() { - return this.#left; - } - - get right() { - return this.#right; - } - - // get stdin(): WritableStream { - // return this.#left.stdin; - // } - // get stdout(): ReadableStream { - // return this.#right.stdout; - // } - // get stderr(): ReadableStream { - // return this.#right.stderr; // TODO merge - // } - - toString() { - return `${this.#left} \x1B[35m|\x1B[0m ${this.#right}`; - } -} - -const strategy = new ByteLengthQueuingStrategy({ - highWaterMark: 65535 -}); - -export interface CommandStreamConfig { - stdin: ReadableStream | null, - stdout: WritableStream | null, - stderr: WritableStream | null -} - -export interface CommandInfo { - cmd: [string, ...string[]]; -} - -export class NativeCommand extends OpaqueCommand { - #info: CommandInfo; - - public constructor( - context: BazxContext, - info: CommandInfo - ) { - super(context); - this.#info = info; - } - - protected [RUN](exec: BazxContext['exec'], streams: CommandStreamConfig): PromiseLike { - return exec(this.#info.cmd, streams); - } - - get command() { - return this.#info.cmd[0]; - } - - get args() { - return this.#info.cmd.slice(1); - } - - toString() { - return `\x1B[32m${this.command}\x1B[0m ${this.args.map(x => `\x1B[4m${x}\x1B[0m`).join(' ')}`.trim(); - } -} - -function writableStreamFromWriter(writer: WritableStreamDefaultWriter, done: (err?: any) => void) { - return new WritableStream({ - async write(chunk) { - await writer!.write(chunk); - }, - abort(err) { - done(err); - }, - close() { - done(); - } - }); -} diff --git a/src/consumer.ts b/src/consumer.ts new file mode 100644 index 0000000..1b97168 --- /dev/null +++ b/src/consumer.ts @@ -0,0 +1,389 @@ +import { BazxExecConfig, BazxExec } from './provider.ts'; + +function getStrategyFromContext(context: BazxOptions) { + if (context.highWaterMark) + return new ByteLengthQueuingStrategy({ + highWaterMark: context.highWaterMark + }); +} + +const ENCODER = new TextEncoder(); + +const createStdoutLogger = (context: BazxOptions) => context.log ? new WritableStream({ + async write(chunk) { + await context.log!(chunk); + } +}) : undefined; + +const createStderrLogger = (context: BazxOptions) => { + if (!context.log) + return undefined; + if (context.noColors) + return new WritableStream({ + async write(chunk) { + await context.log!(chunk); + } + }); + const redBuffer = ENCODER.encode('\x1B[31m'); + const resetBuffer = ENCODER.encode('\x1B[0m'); + return new WritableStream({ + async write(chunk) { + const buffer = new Uint8Array(redBuffer.byteLength + chunk.byteLength + resetBuffer.byteLength); + buffer.set(redBuffer, 0); + buffer.set(chunk, redBuffer.byteLength); + buffer.set(resetBuffer, redBuffer.byteLength + chunk.byteLength); + await context.log!(buffer); + } + }); +}; + +function mergedWritableStream(writer: WritableStreamDefaultWriter, done: (err?: any) => void) { + return new WritableStream({ + async write(chunk) { + await writer!.write(chunk); + }, + async abort(err) { + await done(err); + }, + async close() { + await done(); + } + }); +} + +async function exec2StdoutResponse( + cmd: string, + cwd: string | undefined, + env: Record, + context: BazxOptions, + invoke: (config?: BazxExecConfig) => ReturnType +) { + const strategy = getStrategyFromContext(context); + const output = new TransformStream( + context.log ? { + async transform(chunk, controller) { + await context.log!(chunk); // Cheap logging hack + controller.enqueue(chunk); + } + } : {}, + strategy, + strategy + ); + const [{ code }, result] = await Promise.all([ + invoke({ + cwd, + env, + stdout: output.writable, + stderr: createStderrLogger(context) + }), + new Response(output.readable).blob() + ]); + return new CommandResponse( + cmd, + env, + code, + result + ); +} + +class Command implements Body, PromiseLike { + #context: BazxOptions; + + #fetch: (config?: BazxExecConfig) => ReturnType; + #cmd: (colors: boolean) => string; + #cwd: string | undefined = undefined; + #env: Record = Object.create(null); + + public constructor( + context: BazxOptions, + cmd: (colors: boolean) => string, + fetch: (config?: BazxExecConfig) => ReturnType + ) { + this.#context = context; + this.#fetch = fetch; + this.#cmd = cmd; + } + + then( + onfulfilled?: (value: Response) => TResult1 | PromiseLike, + onrejected?: (reason: any) => TResult2 | PromiseLike + ): PromiseLike { + return this.stdout.then(onfulfilled, onrejected); + } + + get body(): ReadableStream { + const buffer = new TransformStream(); + this.stdout + .then(rejectOnNonZeroExitCode) + .then(response => response.body!.pipeTo(buffer.writable)) + .catch(err => buffer.writable.abort(err).catch(() => { /* Prevent unhandled promise rejection */ })); + return buffer.readable; + } + + get bodyUsed(): boolean { + return false; // Each call to `body` creates a new stream, so the body is never "used" + } + + arrayBuffer(): Promise { + return this.stdout + .then(rejectOnNonZeroExitCode) + .then(response => response.arrayBuffer()); + } + + blob(): Promise { + return this.stdout + .then(rejectOnNonZeroExitCode) + .then(response => response.blob()); + } + + formData(): Promise { + return this.stdout + .then(rejectOnNonZeroExitCode) + .then(response => response.formData()); + } + + json(): Promise { + return this.stdout + .then(rejectOnNonZeroExitCode) + .then(response => response.json()); + } + + text(): Promise { + return this.stdout + .then(rejectOnNonZeroExitCode) + .then(response => response.text()); + } + + get url() { + return this.command; + } + + get command() { + return this.#cmd(false); + } + + cwd(path: string | null): Command { + this.#cwd = path ?? undefined; + return this; + } + + env(key: string, value: string | null): Command { + if (value === null) + delete this.#env[key]; + else + this.#env[key] = value; + return this; + } + + get ok() { + return this.status + .then((code) => code === 0); + } + + get status(): Promise { + return Promise.resolve(this.#context.log?.(ENCODER.encode(`$ ${this}\r\n`))) + .then(() => this.#fetch({ + cwd: this.#cwd, + env: this.#env, + stdout: createStdoutLogger(this.#context), + stderr: createStderrLogger(this.#context) + })) + .then(({ code }) => code); + } + + get stdout(): Promise { + return Promise.resolve(this.#context.log?.(ENCODER.encode(`$ ${this}\r\n`))) + .then(() => exec2StdoutResponse( + this.#cmd(false), + this.#cwd, + this.#env, + this.#context, + this.#fetch + )); + } + + toString() { + return this.#cmd(!this.#context.noColors); + } + + pipe(command: Command): Command; + pipe(transformStream: TransformStream): Command; + pipe(commandOrTransformStream: Command | TransformStream): Command; + pipe(commandOrTransformStream: Command | TransformStream) { + if (commandOrTransformStream instanceof Command) { + const other = commandOrTransformStream; + return new Command( + this.#context, + colors => { + const left = this.#cmd(colors); + const right = other.#cmd(colors); + return !(left && right) + ? left || right + : colors + ? `${left} \x1B[35m|\x1B[0m ${right}` + : `${left} | ${right}`; + }, + async ({ cwd, env = {}, stdin, stdout, stderr, signal } = {}) => { + const strategy = getStrategyFromContext(this.#context); + const pipe = new TransformStream({}, strategy, strategy); + const writer = stderr?.getWriter(); + let left = 2; + const [, result] = await Promise.all([ + this.#fetch({ + cwd: this.#cwd ?? cwd, + env: Object.assign({}, env, this.#env), + stdin, + stdout: pipe.writable, + stderr: !writer ? undefined : mergedWritableStream(writer, () => (--left || writer.close())), + signal + }), + other.#fetch({ + cwd: other.#cwd ?? cwd, + env: Object.assign({}, env, other.#env), + stdin: pipe.readable, + stdout, + stderr: !writer ? undefined : mergedWritableStream(writer, () => (--left || writer.close())), + signal + }) + ]); + return result; + }); + } + const stream = commandOrTransformStream; + return new Command( + this.#context, + colors => { + const left = this.#cmd(colors); + const right = colors ? `\x1B[33m${stream}\x1B[0m` : String(stream); + return !left + ? right + : colors + ? `${left} \x1B[35m|\x1B[0m ${right}` + : `${left} | ${right}`; + }, + async ({ cwd, env = {}, stdin, stdout, stderr, signal } = {}) => { + const [result] = await Promise.all([ + this.#fetch({ + cwd: this.#cwd ?? cwd, + env: Object.assign({}, env, this.#env), + stdin, + stdout: stream.writable, + stderr, + signal + }), + stream.readable.pipeTo(stdout || new WritableStream({})) + ]); + return result; + }); + } +} + +class CommandResponse extends Response { + #url: string; + #status: number; + + public constructor( + cmd: string, + env: Record, + status: number, + stdout: Blob + ) { + super(stdout, { + headers: env + }); + this.#url = cmd; + this.#status = status; + } + + // @ts-ignore + get url() { + return this.#url; + } + + // @ts-ignore + get status() { + return this.#status; + } + + // @ts-ignore + get ok() { + return this.#status === 0; + } +} + +function rejectOnNonZeroExitCode(response: Response) { + if (!response.ok) + return Promise.reject( + Object.assign( + new Error(`Command ${response.url} exited with code ${response.status}`), + { response } + ) + ); + return Promise.resolve(response); +} + +function parse(xs: TemplateStringsArray, ...args: any[]) { + if (!Array.isArray(xs) || !Array.isArray(xs.raw)) + throw new Error('$ can only be used as a template string tag'); + const cmd: string[] = []; + let left = ''; + let i = 0; + for (let part of xs) { + for (let index; (index = part.indexOf(' ')) !== -1; part = part.slice(index + 1)) { + left += part.slice(0, index); + if (left) + cmd.push(left); + left = ''; + } + left += part; + left += i === args.length ? '' : args[i++]; + } + if (left) + cmd.push(left); + // const cmd = (xs[0]! + args.map((x, i) => x + xs[i + 1]!).join('')).split(' ').filter(x => x); // FIXME + if (cmd.length < 1) + throw new Error('Missing command name'); + return cmd as [string, ...string[]]; +} + +export interface BazxOptions { + highWaterMark?: number; + noColors?: boolean; + log?(chunk: Uint8Array): unknown | Promise; +} + +export interface $ { + (xs: TemplateStringsArray, ...args: any[]): Command; + + with(middleware: (exec: BazxExec) => BazxExec): $; +} + +export function createBaxz(exec: BazxExec, options: BazxOptions = {}): $ { + function $(xs: TemplateStringsArray, ...args: any[]) { + if (args.length === 0 && xs[0]?.length === 0) + return new Command(options, () => '', async ({ stdin, stdout, stderr } = {}) => { + await Promise.all([ + stdin?.cancel(), + stdout?.getWriter().close(), + stderr?.getWriter().close() + ]); + return ({ code: 0, signal: undefined }) + }); + const parsed = parse(xs, ...args); + return new Command( + options, + colors => colors + ? `\x1B[32m${parsed[0]}\x1B[0m ${parsed.slice(1).map(x => `\x1B[4m${x}\x1B[0m`).join(' ')}`.trim() + : parsed.join(' '), + config => exec(parsed, config) + ); + } + + function withMiddleware(middleware: (exec: BazxExec) => BazxExec): $ { + return createBaxz(middleware(exec), options); + } + + return Object.assign($, { + with: withMiddleware + }); +} diff --git a/src/middleware.ts b/src/middleware.ts new file mode 100644 index 0000000..6604f6d --- /dev/null +++ b/src/middleware.ts @@ -0,0 +1,50 @@ +import type { BazxExec } from './provider.ts'; + +export type BazxMiddleware = (exec: BazxExec) => BazxExec; + +export function filtered(predicate: (cmd: [string, ...string[]]) => boolean): BazxMiddleware { + return exec => function filteredExec(cmd, options) { + if (!predicate(cmd)) + throw new Error(`${cmd[0]}: not allowed to execute`); + return exec(cmd, options); + } +} + +export const swapStdoutStderr: BazxMiddleware = exec => (cmd, options = {}) => exec(cmd, { + ...options, + stdout: options?.stderr, // stdout becomes stderr + stderr: options?.stdout // stderr becomes stdout +}); + +export const shWrapper: BazxMiddleware = exec => (cmd, options) => + exec( + [ + 'sh', + '-c', + cmd + .map(a => a.includes(' ') ? `"${a}"` : a) + .join(' ') + ], + options + ); + +export const interceptCurl: BazxMiddleware = exec => async (cmd, options = {}) => { + if (cmd[0] !== 'curl') + return exec(cmd, options) + await options.stderr?.abort(); // middleware MUST ensure that all streams in `options` are closed (deadlock otherwise) + const response = await fetch( + new URL(cmd[cmd.length - 1 || 1] ?? '', new URL(options.cwd || '.', 'file:///')).href, + { + method: options.stdin ? 'POST' : 'GET', + headers: options.env, + body: options.stdin + } + ); + if (response.body) + await response.body.pipeTo(options.stdout ?? new WritableStream()); + else if (options.stdout) + await options.stdout?.abort(); + return { + code: response.status - 200 + } +} diff --git a/src/mod.ts b/src/mod.ts index b2d22a6..126e560 100644 --- a/src/mod.ts +++ b/src/mod.ts @@ -1,100 +1,3 @@ -import { - ExecutionResult, - OpaqueCommand, - CommandStreamConfig, - NativeCommand, - BazxContext -} from './bazx.ts'; - -export * from './utils.ts'; - -export interface Bazx { - (xs: TemplateStringsArray, ...args: any[]): NativeCommand; -} - -export interface BazxConfig { - logger?(cmd: OpaqueCommand): void; - stdout?: WritableStream; - stderr?: WritableStream; - - parse?(xs: TemplateStringsArray, ...args: any[]): [string, ...string[]]; - - exec(cmd: [string, ...string[]], streams: CommandStreamConfig): PromiseLike -} - -export interface BazxOptions { - logcmd: boolean; - errexit: boolean; -} - -export function create(config: BazxConfig, defaultOptions: Partial = {}): Bazx { - const options: BazxOptions = Object.assign({ - logcmd: true, - errexit: true - }, defaultOptions); - const context: BazxContext = { - get throw() { - return options.errexit; - }, - log, - exec - }; - return $; - - function mergeWritable(left: WritableStream, right: WritableStream) { - const middle = new TransformStream(); - const [first, second] = middle.readable.tee(); - first.pipeTo(left); - second.pipeTo(right); - return middle.writable; - } - - function log(cmd: OpaqueCommand, streams: CommandStreamConfig) { - if (options.logcmd) { - if (config.logger) - config.logger(cmd); - else if (config.stdout) { - const writer = config.stdout.getWriter(); - writer.write(new TextEncoder().encode(`$ ${cmd}\n`)); - writer.close(); - } - if (config.stdout) - streams.stdout = !streams.stdout ? config.stdout : mergeWritable(config.stdout, streams.stdout); - if (config.stderr) - streams.stderr = !streams.stderr ? config.stderr : mergeWritable(config.stderr, streams.stderr); - } - } - - function exec(cmd: [string, ...string[]], streams: CommandStreamConfig) { - return config.exec(cmd, streams); - } - - function $(xs: TemplateStringsArray, ...args: any[]) { - const cmd = config.parse?.(xs, ...args) ?? parse(xs, ...args); - return new NativeCommand(context, { cmd }); - } -} - -export function parse(xs: TemplateStringsArray, ...args: any[]) { - if (!Array.isArray(xs) || !Array.isArray(xs.raw)) - throw new Error('$ can only be used as a template string tag'); - const cmd: string[] = []; - let left = ''; - let i = 0; - for (let part of xs) { - for (let index; (index = part.indexOf(' ')) !== -1; part = part.slice(index + 1)) { - left += part.slice(0, index); - if (left) - cmd.push(left); - left = ''; - } - left += part; - left += i === args.length ? '' : args[i++]; - } - if (left) - cmd.push(left); - // const cmd = (xs[0]! + args.map((x, i) => x + xs[i + 1]!).join('')).split(' ').filter(x => x); // FIXME - if (cmd.length < 1) - throw new Error('Missing command name'); - return cmd as [string, ...string[]]; -} +export * from './provider.ts' +export * from './consumer.ts' +export * from './middleware.ts' diff --git a/src/provider.ts b/src/provider.ts new file mode 100644 index 0000000..0c1760c --- /dev/null +++ b/src/provider.ts @@ -0,0 +1,10 @@ +export interface BazxExecConfig { + cwd?: string; + env?: Record; + stdin?: ReadableStream; + stdout?: WritableStream; + stderr?: WritableStream; + signal?: AbortSignal; +} + +export type BazxExec = (cmd: [string, ...string[]], options?: BazxExecConfig) => Promise<{ code: number }>; diff --git a/src/utils.ts b/src/utils.ts deleted file mode 100644 index 8c0de05..0000000 --- a/src/utils.ts +++ /dev/null @@ -1,62 +0,0 @@ -import type { OpaqueCommand } from './bazx.ts'; - -async function collectAsString(readable: ReadableStream, preventTrim = false) { - let str = ''; - let read; - const decoder = new TextDecoder(); - const reader = readable.getReader(); - while (!(read = await reader.read()).done) - str += decoder.decode(read.value, { /*stream: true as any*/ }); // Stream mode not supported yet in Deno - str += decoder.decode(undefined, { stream: false }); - return preventTrim ? str : str.trim(); -} - -/** - * Read stdout from the specified command line as string - * @param command The command to read stdout from - * @returns The stdout stream concatenated as a string, plus process exit code - */ -export async function stdout(command: OpaqueCommand) { - const [stdout, result] = await Promise.all([ - collectAsString(command.stdout), - command - ]); - return { - ...result, - stdout - }; -} - -/** - * Read stderr from the specified command line as string - * @param command The command to read stderr from - * @returns The stderr stream concatenated as a string, plus process exit code - */ -export async function stderr(command: OpaqueCommand) { - const [stderr, result] = await Promise.all([ - collectAsString(command.stderr), - command - ]); - return { - ...result, - stderr - }; -} - -/** - * Read both stdout and stderr separately from the specified command line as string - * @param command The command to read stdout and stderr from - * @returns The stderr and stdout stream concatenated as a string, plus process exit code - */ -export async function collect(command: OpaqueCommand) { - const [stdout, stderr, result] = await Promise.all([ - collectAsString(command.stdout), - collectAsString(command.stderr), - command, - ]); - return { - ...result, - stdout, - stderr - }; -} diff --git a/test/basics.ts b/test/basics.ts deleted file mode 100644 index 1b475f5..0000000 --- a/test/basics.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { $, stdout } from '../mod.ts'; - -await $`ls -1`.pipe($`wc -l`) - -let branch = (await stdout($`git branch`)).stdout -await $`printf ${branch}`.pipe($`wc -l`) // The new line trimmed from stdout. - -let foo = `hi; echo 'oops'` -await $`echo "${foo}"`.pipe($`wc -l`) // Vars properly quoted. diff --git a/test/debug.js b/test/debug.js deleted file mode 100644 index f688db9..0000000 --- a/test/debug.js +++ /dev/null @@ -1,42 +0,0 @@ -import { $, stdout, collect } from '../mod.ts'; - -await $`echo two arguments, ${'one argument'}`; - -// FROM https://github.com/google/zx/issues/35#issue-880926668 -const dir = import.meta.url.slice(import.meta.url.indexOf('://') + 3, import.meta.url.lastIndexOf('/')); -console.log( - await collect( - $`find ${dir} -type f -print0` - .pipe($`xargs -0 grep foo`) - .pipe($`wc -l`) - ) -); - -console.log( - await collect( - $`cat ${'this will cause an error'}` - .pipe($`env`) - .pipe(new TransformStream({ - transform(chunk) { - console.warn('DROPPED', chunk.byteLength, 'BYTES'); - }, - flush(controller) { - controller.enqueue(new TextEncoder().encode('TRANSFORMED BY TRANSFORM STREAM\nTHIS LINE WILL BE REMOVED BY grep')); - } - })) - .pipe($`grep STREAM`) - ) -); - -console.log( - await stdout( - $`wget --limit-rate 3 -O - http://detectportal.firefox.com/success.txt` - .pipe(new TransformStream({ - transform(chunk, controller) { - controller.enqueue(new TextEncoder().encode(new TextDecoder().decode(chunk).toUpperCase())); - } - })) - ) -); - -await $`not_found`; diff --git a/test/env.js b/test/env.js deleted file mode 100644 index 93a9f2f..0000000 --- a/test/env.js +++ /dev/null @@ -1,25 +0,0 @@ -/// - -import { $, stdout } from '../mod.ts'; - -const VAR = 'CUSTOM_VARIABLE'; - -switch ((await Deno.permissions.query({ name: 'env' })).state) { - case 'prompt': - if ((await Deno.permissions.request({ name: 'env' })).state !== 'granted') - break; - case 'granted': - Deno.env.set(VAR, 'custom_value'); -} - -try { - console.log( - 'FOUND:', - (await stdout( - $`env` - .pipe($`grep -- ${VAR}`) - )).stdout - ); -} catch (err) { - console.log('NOT FOUND (missing permission?)'); -} diff --git a/test/showcase.js b/test/showcase.js deleted file mode 100644 index 7cceb46..0000000 --- a/test/showcase.js +++ /dev/null @@ -1,35 +0,0 @@ -import { $, collect } from '../mod.ts'; - -/** - * Returns a TransformStream that parse the input - * as JSON and ouputs a transformed value as string. - * @param {(obj: any) => any} transformer - * @returns {TransformStream} - */ -const jq = transformer => { - let json = ''; - const decoder = new TextDecoder(); - return new TransformStream({ - transform(chunk) { - json += decoder.decode(chunk); - }, - flush(controller) { - if (json) - controller.enqueue(new TextEncoder().encode(String(transformer(JSON.parse(json))))); - } - }); -}; - -const file = 'file not'; - -console.log( - // Simple API - await collect( // Optional output collection - $`cat ${file} found` // Parameters injection prevention - .pipe($`wget --progress=dot --limit-rate 50 -O - https://cataas.com/cat?json=true`) - .pipe(jq(({ created_at }) => created_at.split('T').join('\n'))) // Easy JavaScript thrid-party streams integration - .pipe($`grep -- -`) - ) -); - -await $`not_found`; // Clean errors reporting diff --git a/tsconfig.json b/tsconfig.json index 53765cc..49dd1bd 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,10 +6,8 @@ "noUncheckedIndexedAccess": true }, "include": [ - "src/**/*.ts", + "./*.ts", "src/*.ts", - "./deno.d.ts", - "./mod.ts", - "./deno.ts" + "examples/*.ts", ] } \ No newline at end of file