From ea54f17e911ba0b645e41eb53f2d3cca41c9c48e Mon Sep 17 00:00:00 2001 From: Taylor Downs Date: Sat, 3 Feb 2024 09:22:00 +0000 Subject: [PATCH 1/3] Create add-to-project.yml --- .github/workflows/add-to-project.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/workflows/add-to-project.yml diff --git a/.github/workflows/add-to-project.yml b/.github/workflows/add-to-project.yml new file mode 100644 index 000000000..05814510c --- /dev/null +++ b/.github/workflows/add-to-project.yml @@ -0,0 +1,21 @@ +name: Add new issues and PRs to project + +on: + issues: + types: + - opened + pull_request: + types: + - opened + +jobs: + add-to-project: + name: Add issue or PR to project + runs-on: ubuntu-latest + steps: + - uses: actions/add-to-project@RELEASE_VERSION + with: + # You can target a project in a different organization + # to the issue + project-url: https://github.com/orgs/OpenFn/projects/3 + github-token: ${{ secrets.PROJECT_MANAGEMENT_TOKEN }} From 6e2f64bf6845ffd379527850a461243a2ea89a84 Mon Sep 17 00:00:00 2001 From: Taylor Downs Date: Sat, 3 Feb 2024 09:24:11 +0000 Subject: [PATCH 2/3] Update add-to-project.yml --- .github/workflows/add-to-project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/add-to-project.yml b/.github/workflows/add-to-project.yml index 05814510c..35b5ba41c 100644 --- a/.github/workflows/add-to-project.yml +++ b/.github/workflows/add-to-project.yml @@ -13,7 +13,7 @@ jobs: name: Add issue or PR to project runs-on: ubuntu-latest steps: - - uses: actions/add-to-project@RELEASE_VERSION + - uses: actions/add-to-project@v0.5.0 with: # You can target a project in a different organization # to the issue From 0f00be04de5187e2d95f1f9f92d4557109e59104 Mon Sep 17 00:00:00 2001 From: josephjclark Date: Thu, 8 Feb 2024 12:26:12 +0000 Subject: [PATCH 3/3] Next release (#584) * engine: spike mapping console logs to an adaptor logger * runtime: messy tweak to module loading * engine,runtime: revert linker change and fix tests * engine: track test file * logger: dont stringify json output AND serialize errors This cause problems with the worker because errors get flattened to {}, and also we have to double parse. Now the logger will just emit whatever it logged to whatever the log emmiter is, so JSON stays as JSON. Which is good, but it no longer guarantees it'll be serializable * logger: tidy * engine: don't parse json logs coming out of the logger * engine, worker: better handling of objects coming from the logger The logger always sends raw json, but the log message is stringified by the engine, and rebuilt by the worker before sending to lightning this last bit needs work but its better * engine: fix tests * logger: tests and types * cli: update test * engine: types * worker: update tests * logger: set a special json emitter so that json logs get nicely printed in the CLI * logger: fix types * logger: log all json to .log * tests: fixes * logger: fix tests * logger: serialise print() properly * logger: types * engine: fix logs to gcp They were neglecting to parse the strings sent out by the new json logger * test: update log handling * engine: fix passing test It was secretly failing under the hood * runtime: add tests on job logger and errors * logger: improve detection of error objects * engine: tests on error logging * engine: restore adaptor logger * changesets * Tidy ups * engine: refactor log messages (and be a bit more lenient about structure) * worker: simplify logging * tiny tidyups * remove old docs * version: worker@0.8.1 --- docs/future/README.md | 11 --- docs/future/diagrams/README.md | 17 ----- .../diagrams/core-cli-requirements.drawio | 1 - .../future/diagrams/core-cli-requirements.svg | 3 - .../diagrams/core-compilation-steps.drawio | 1 - .../diagrams/core-compilation-steps.svg | 3 - .../diagrams/core-execution-steps.drawio | 1 - docs/future/diagrams/core-execution-steps.svg | 3 - docs/future/diagrams/kit-components.drawio | 1 - docs/future/diagrams/kit-components.svg | 3 - docs/future/editor.md | 25 ------ docs/future/history.md | 73 ------------------ docs/future/kit-components.md | 21 ----- integration-tests/cli/test/errors.test.ts | 6 +- integration-tests/cli/test/metadata.test.ts | 1 - integration-tests/worker/CHANGELOG.md | 12 +++ integration-tests/worker/package.json | 2 +- .../worker/test/integration.test.ts | 2 +- packages/cli/CHANGELOG.md | 10 +++ packages/cli/package.json | 2 +- packages/cli/test/util/print-versions.test.ts | 2 +- packages/compiler/CHANGELOG.md | 7 ++ packages/compiler/package.json | 2 +- packages/deploy/CHANGELOG.md | 7 ++ packages/deploy/package.json | 2 +- packages/engine-multi/CHANGELOG.md | 10 +++ packages/engine-multi/README.md | 10 +++ packages/engine-multi/package.json | 5 +- packages/engine-multi/src/api/call-worker.ts | 10 ++- packages/engine-multi/src/api/execute.ts | 4 +- packages/engine-multi/src/api/lifecycle.ts | 15 +++- .../engine-multi/src/api/validate-worker.ts | 1 - packages/engine-multi/src/events.ts | 7 +- packages/engine-multi/src/worker/events.ts | 8 +- .../engine-multi/src/worker/thread/helpers.ts | 27 +++++-- .../src/worker/thread/mock-run.ts | 2 +- .../engine-multi/src/worker/thread/run.ts | 18 ++++- .../@openfn/helper_1.0.0/index.js | 13 ++++ .../{ => @openfn}/helper_1.0.0/package.json | 4 +- .../node_modules/helper_1.0.0/index.cjs | 8 -- .../engine-multi/test/__repo__/package.json | 2 +- .../engine-multi/test/api/execute.test.ts | 2 +- .../engine-multi/test/api/lifecycle.test.ts | 6 +- packages/engine-multi/test/errors.test.ts | 4 +- .../engine-multi/test/integration.test.ts | 76 ++++++++++++++++++- .../engine-multi/test/worker/helper.test.ts | 42 ++++++++++ .../test/worker/mock-worker.test.ts | 6 +- packages/lightning-mock/CHANGELOG.md | 10 +++ packages/lightning-mock/package.json | 2 +- packages/logger/CHANGELOG.md | 7 ++ packages/logger/package.json | 2 +- packages/logger/src/logger.ts | 9 ++- packages/logger/src/options.ts | 19 ++--- packages/logger/src/sanitize.ts | 28 ++++++- packages/logger/src/util/default-emitter.ts | 10 +++ packages/logger/src/util/json-emitter.ts | 16 ++++ packages/logger/test/logger.test.ts | 73 +++++++++++------- packages/logger/test/mock.test.ts | 7 +- packages/logger/test/sanitize.test.ts | 9 +++ .../logger/test/util/json-emitter.test.ts | 65 ++++++++++++++++ packages/runtime/CHANGELOG.md | 7 ++ packages/runtime/package.json | 2 +- packages/runtime/src/modules/linker.ts | 1 - packages/runtime/test/runtime.test.ts | 45 +++++++++++ packages/ws-worker/CHANGELOG.md | 11 +++ packages/ws-worker/package.json | 2 +- packages/ws-worker/src/api/execute.ts | 34 +++++---- packages/ws-worker/src/mock/runtime-engine.ts | 2 +- packages/ws-worker/src/types.d.ts | 6 ++ packages/ws-worker/src/util/index.ts | 2 + packages/ws-worker/test/api/execute.test.ts | 12 +-- .../test/mock/runtime-engine.test.ts | 33 ++++++++ pnpm-lock.yaml | 3 + 73 files changed, 619 insertions(+), 296 deletions(-) delete mode 100644 docs/future/README.md delete mode 100644 docs/future/diagrams/README.md delete mode 100644 docs/future/diagrams/core-cli-requirements.drawio delete mode 100644 docs/future/diagrams/core-cli-requirements.svg delete mode 100644 docs/future/diagrams/core-compilation-steps.drawio delete mode 100644 docs/future/diagrams/core-compilation-steps.svg delete mode 100644 docs/future/diagrams/core-execution-steps.drawio delete mode 100644 docs/future/diagrams/core-execution-steps.svg delete mode 100644 docs/future/diagrams/kit-components.drawio delete mode 100644 docs/future/diagrams/kit-components.svg delete mode 100644 docs/future/editor.md delete mode 100644 docs/future/history.md delete mode 100644 docs/future/kit-components.md create mode 100644 packages/engine-multi/test/__repo__/node_modules/@openfn/helper_1.0.0/index.js rename packages/engine-multi/test/__repo__/node_modules/{ => @openfn}/helper_1.0.0/package.json (56%) delete mode 100644 packages/engine-multi/test/__repo__/node_modules/helper_1.0.0/index.cjs create mode 100644 packages/engine-multi/test/worker/helper.test.ts create mode 100644 packages/logger/src/util/default-emitter.ts create mode 100644 packages/logger/src/util/json-emitter.ts create mode 100644 packages/logger/test/util/json-emitter.test.ts diff --git a/docs/future/README.md b/docs/future/README.md deleted file mode 100644 index 09a65648f..000000000 --- a/docs/future/README.md +++ /dev/null @@ -1,11 +0,0 @@ -Future -====== - -A collection of docs describing and outlining the history of OpenFn internals -and ideas for the future. - -## Table of Contents - -- [History of Core](history.md) -- [Kit Components](kit-components.md) - diff --git a/docs/future/diagrams/README.md b/docs/future/diagrams/README.md deleted file mode 100644 index 2c2125cf4..000000000 --- a/docs/future/diagrams/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# Diagrams - -Diagrams are created using [draw.io](https://app.diagrams.net/). - -Editing diagrams can be done in the browser via the Draw.io web app, however -in order to programmically export the files to SVG for use in markdown -it is suggested to install the desktop version. - -See [here](https://github.com/jgraph/drawio-desktop/releases) to get the -appropriate version for your OS. - -## Exporting to SVG - -1. Install draw.io -2. Export the diagram to SVG: - `drawio --export --format svg` - diff --git a/docs/future/diagrams/core-cli-requirements.drawio b/docs/future/diagrams/core-cli-requirements.drawio deleted file mode 100644 index b065be4d7..000000000 --- a/docs/future/diagrams/core-cli-requirements.drawio +++ /dev/null @@ -1 +0,0 @@ -7VnbctowEP0aHsP4BoTHBEI7nbaTSSbT5KkjbMVWIluuLAfTr+8KS75gg8kEhtDkCe9qZUt7zlmvTM+ehNkXjuLgB/Mw7VmGl/Xsac+yTNOw4Ed6lspjWIPc43PiKV/puCV/sQ5U3pR4OKkFCsaoIHHd6bIowq6o+RDnbFEPe2S0/tQY+bjhuHURbXp/EU8EuffcGpX+r5j4gX6yORznIyHSwWonSYA8tqi47KuePeGMifwqzCaYyuzpvOTzZhtGi4VxHIldJvjf7qz7+Ma4mV5cTSN2d3n3h5+ZQ7U4sdQ7xh4kQJmMi4D5LEL0qvRecpZGHpa3NcEqY74zFivnExZiqdBEqWDgCkRI1SismC/vwTD6QAdlP0hbG9OsZi2V9cgiMUMhodIxYSknmMPqf+LFKtS7kIjD0Jwy9zl3zQjVj00EZ88FjADAZb55ueONSVWuBB7m4i2ZVEQXiPtYbMt4iT2oBrMQwwZhIscUCfJSXwhS7PWLuBJguFAYvwJvtcoXRFP1pJ41pLDeyzlc+GKVJSSw9sJDioEikmvPU8KiBn+aWa4gDyKIZVyY+bJg9B8pW7gB4qLvMTcNIetywiIgAt/GaJXwBQTKmTmZFGO2EuEFc0FAwheU+BGMC0lLhbUcw9l2tJvg6Alay7qaDZW9KEuDaShfUCkLOm7veJqDI+v3f5GvfRLytbvli7OY4yQhUpm7afgjKdgaDd6bgkdHVvDoqBLGGRHVTgDMB71GuC5XIQ29iD3qfnwSuh936x567xS4AVCgcMPre13mSsmQNkQppgwEHa6JvYFXjDmBPQHoaxOvy4FW9dcpi7iruGlatdJgd3FsD0XAMetFwBq3FIFBSxFwDlYEjG6AXcZ3bMtwht1UoLmkwyyCM5nE3eUkFg0O1FFpw63CBjRPGE0FvijBq0PptFLm0HBaa12Zc96EszhPVuEsCv3+8Tw/alEvjIde/Yx1en2ZqQ/9By/Qq6mwU7SsBMSMRCKp3PlaOkruDZ069wbGYI08+R1LKhVLewO7zO5q8UiAOb8/z3Ibq8bQeHdnubbD+RogPhSJeOPu1UewVeWvfnp6xavRXsuK05IVqyUr54fKyrA7KSfN0i1U2PzGOzZVnU9Qmh+H1pRzbIwG3S8JfWbY5bDwYeDUh4Jj42c6R+0hjb5j29U+8szoG12N5MqqnAS7u8uWLwJd/eYeu8vBjs3l+I295QYigFn+/5N3h+XfaPbVPw== \ No newline at end of file diff --git a/docs/future/diagrams/core-cli-requirements.svg b/docs/future/diagrams/core-cli-requirements.svg deleted file mode 100644 index 628d414a5..000000000 --- a/docs/future/diagrams/core-cli-requirements.svg +++ /dev/null @@ -1,3 +0,0 @@ - - -
state
json
state...
expression
js
expression...
Module Name
Module Name
core
executable/node script
core...
final_state
json
final_state...
Module
ModuleText is not SVG - cannot display \ No newline at end of file diff --git a/docs/future/diagrams/core-compilation-steps.drawio b/docs/future/diagrams/core-compilation-steps.drawio deleted file mode 100644 index 04d2d0563..000000000 --- a/docs/future/diagrams/core-compilation-steps.drawio +++ /dev/null @@ -1 +0,0 @@ -7ZpLc5tADIB/jWfaQz08DNhH20n6mD4OOTQ5bkCGbReWLEuM++urhcWYkIfblEAzOQVptSAkfRIQT+x1XLwXJI2+8ADYxDKCYmKfTCzLNA0L/yjNTmsMy6k0oaCB1jWKc/oLakOtzWkAWctQcs4kTdtKnycJ+LKlI0Lwbdtsw1n7qikJoaM49wnrar/TQEaVdm55jf4D0DCqr2y6i2olJrWxvpMsIgHfHqjs04m9FpzL6igu1sBU9Oq4VPvO7lndOyYgkcds+Mi2gQfezPj2iQdOsrxerON35lw7J3f1HUOAAdAiFzLiIU8IO220K8HzJAB1WhOlxuYz56lW/gApdzqbJJccVZGMmV7d8ESekZgyVRVrngsKAp34ChieFSTBUiUOl64Y939WqjPK6t2ZFPznPhsYx1V1D8rxe2OjVRlezIcHAqLrVRIRgnzAbrbPINY+8Bik2OE+AYxIetP2g+gaDPd2TZrwQGfqD7KmnbwhLK+TVqQCsozypJPPbrgOMoFFmSq7uAgVwNMN41s/IkJOA+7nMYZPbdhGVMJ5SsrIbdFQ7aySa0wRaHt1A0JSpGbJaJigWqpKuFUo2mm0hOLhPHXjWm9wNEy6nVi2lrcNm2YNXHTApWv0lArTegWoFRD7SICcIQGyOwCRgKSSixdNjzM+ehZD0IPhEruLMvxOLV4qsRZOipa009KIqZsdSZ07JHWzLnU+F92J1U7wXfgcJJNcZZzlEpbC1wkvtY00uzP2D+ayh0llu0ey5vTGmv3K2j9hzTmStfmQrDkd1gRc51TAILRleEKahCh5z8De7Tk3Ava8QdgrqLw4OL5sMESpAU8JNXcNry1aG3j/P17dI3n1huTV7c7GTA7Cag/DbzY8gO6ww+/FwOT9D8PP68B0RpNAXancq16vLIPBTfmxcmK5TOqgqy+ERN+/e52rT3Ordvz3ajwKy2U85+n+60tWnw39rk5YWQ08dd1/w7Xt2FNnbGTPhhytzTi9PFh5bLS+mEfh+ZHdYDFkN5h3usE6AowUbtso5yNQvmCq1MA1aFIKKRZFNoJn5R4eh0fArDMos3/zOGxOXsoEXxzJrHlPVT0PtIsOtKucMjXDEzWFjc7YRY6J4peoV1zDx2UIyhrHGD55yL/JJJHwdpzDvac2sZcfaxOL3tqE0SmC7yp8ukk/MaeUbmBMCe3hvcvxhu70+39+v3b65+/0Zv2DiUdbvXF3WT1Pq6/d7AdzKMBHGN9Mp9NR9e9eXsf6BB7F5scq5drBb37s098= \ No newline at end of file diff --git a/docs/future/diagrams/core-compilation-steps.svg b/docs/future/diagrams/core-compilation-steps.svg deleted file mode 100644 index 249c1d463..000000000 --- a/docs/future/diagrams/core-compilation-steps.svg +++ /dev/null @@ -1,3 +0,0 @@ - - -
expression
expression
adaptor
adaptor
acorn
acorn
require
require
ast
ast
Find all top level CallExpressions
Find all top level...
Check if they exist in exports
Check if they exis...
Build new CallExpressions that are called with (state)
Build new CallExpr...
Wrap in iife
Wrap in iife
Wrap in execute(...)
Wrap in execute(...)Text is not SVG - cannot display \ No newline at end of file diff --git a/docs/future/diagrams/core-execution-steps.drawio b/docs/future/diagrams/core-execution-steps.drawio deleted file mode 100644 index d2940fa1b..000000000 --- a/docs/future/diagrams/core-execution-steps.drawio +++ /dev/null @@ -1 +0,0 @@ -7ZpZc5swEIB/jR+TAWFw+mg7STOdHjPJQ483GdagRiAihI376ysZiSM4rts4kdvpU9jVvbvf6nBG3jyt3nKcJx9YBHSEnKgaeZcjhFzXQfKP0my0xkF+rYk5ibSuVdyRH2Aqam1JIih6FQVjVJC8rwxZlkEoejrMOVv3qy0Z7Y+a4xgGirsQ06H2M4lEUmsv0KTV3wCJEzOyG7ypS1JsKuuVFAmO2Lqj8q5G3pwzJuqvtJoDVdYzdqnbXT9R2kyMQyYOafAtuz1b3ny6viuDKYsfPrwr08sz3UshNmbBEMn1a5FxkbCYZZhetdoZZ2UWgerVlVJb5z1juVZ+ByE22pm4FEyqEpFSXbpkmbjGKaEqKOas5AS4nMRHkNaZQRZNld9k0YKy8L5WXRNqWheCs/vGGdKMs3oNauJPmsasUw4Wwh576HAVmMcg9tQLGgfK0AeWguAb2Y4DxYKs+vPAOgTjpl7rJfmhHfUbTtOTXGFa6pFGKKByurOF/IjF1khYgNHKQZqCpiY3mu8FywZRMDRyx38yknNVL61iRf35krJ1mGAuziMWlqk0umqwToiAuxxv7b2WFVXLOiScc5kFvNkKuCAStSklcSbVQsXPo/DaGyvaDrIbqPa7fugq08DXeOoEhTwtr1vaXYNw0iE9cF7Iu95/JHv2GB+I5IVNJMcDJHGEc8H46ZJ1BHj8k4PHtwGPtBbffNla3zfiVyUa4bLqSRstnTB0wYHQuc7uAHkd6oIBdbeAIzVSFtULL2AQEH1372Kp41q8KBgtBUx5qN2/1bbSeKcnep69AboChePL7FhecCB0/ktBh6ycIhvq3B5zLYJ/Qt3Qlb/k8IjUXRxIHbJK3cWAOg4PJeF2UCtkhySLpTR5hePi4x3PPnxvrLBXEfGl890hT0oteEow3LW75BF5tbtLmkeKX2+TyCawZpodYqd5Dts90twHlZXV+wjWCw4eSvUwMesbvFE/dc+suzFXTUfGjTSj3I0XrLKSHV5gvx1bRx4Fe5kHumDr5+O+7UaK5i7xz5953UMff9yJVZqHzz+3ZaaMUOUcioKo5xyHZJbJa/fl4Dgcer537p8Yie5k4IvPnKiMKIOhKKlQFlCm32bJOh1GpLj/W3Oh5497HvAntj2A0Gvkwt2Jb2I18+3Iz0dMhujgo43VuwgaHm3mHGracJ0UISxFnRObdOiURQ3mM08/qxQ9efb5N1Ou/fsOGp/Q4eeYTw+Wf/jy/oqrjJlmh/eQpTmhED06AZ3qs3sLaQfZWw1O0Krew9JojkDy43eLg49Of/BSL8X2V+1tWeefA7yrnw== \ No newline at end of file diff --git a/docs/future/diagrams/core-execution-steps.svg b/docs/future/diagrams/core-execution-steps.svg deleted file mode 100644 index 77bf2d0df..000000000 --- a/docs/future/diagrams/core-execution-steps.svg +++ /dev/null @@ -1,3 +0,0 @@ - - -
state
json
state...
adaptor
adaptor
Read and parse
Read and parse
require
require
Append state to sandbox
Append state to sand...
Run expression in sandbox
Run expression in...
Write resulting state to disk
Write resulting stat...
Create an execution sandbox using vm2
Create an executio...
compiled expression
compiled expressionText is not SVG - cannot display \ No newline at end of file diff --git a/docs/future/diagrams/kit-components.drawio b/docs/future/diagrams/kit-components.drawio deleted file mode 100644 index 845a5079b..000000000 --- a/docs/future/diagrams/kit-components.drawio +++ /dev/null @@ -1 +0,0 @@ -7Vvdd+IoFP9rfKwnAROTx9bW6c7p7M45nd2Z2TdMMDJNQoZgrf3rlxhIzIc2Y6PonH0yXOAC9+MH94IDOIlePjCULD5RH4cDYPgvA3g7AMA0oCl+Mso6p4wdkBMCRnzZqCQ8klesekrqkvg4rTTklIacJFWiR+MYe7xCQ4zRVbXZnIbVURMU4Abh0UNhk/qV+HwhqabtlhX3mAQLObQDxnlFhFRjuZJ0gXy62iLBuwGcMEp5/hW9THCYCU/JJe833VFbTIzhmHfp8PjxJ/iLOl8+fjFfrX/+/fuVxPgK2DmbZxQu5YrlbPlaiYDRZezjjIs5gDerBeH4MUFeVrsSShe0BY9CWZ1yRp8KUYlF3iDmSbWKCYriLKXhkuPrgpx1k7PAjOOXneszC6kJc8M0wpytRRPVYSQFLS3NcmR5taU31WaxpTLTkkQkbSUoeJfiFB9Sor8iXfPo0u1BcCO3KjgIWwRntwgOuMcSnOk25IR94ZeySBlf0IDGKLwrqTdVSZZtHihNJPEH5nwtzQ4tOa1KF78Q/k18G8OxJYvft6puM3kZqrBWhVgs+Nt24fuGBbBUuey3KamOcxrzKYpImBHucfiMOfHQRsmI8esMu0RFTGOsaFOSSVGy8lWLWUi9p5wkG+yzlEyM++1ESJ0umYf3qUcCMWIB5m/af9PwGA4RJ8/VifTvfUbD+5hgTLAYWAzno4RTJr6G/pCnrfb2gGZiO6vYCApJEItvTwgNM0HIXE/oLbyWFRHx/dwccUpe0WzDL1NZQknMN2u0bgbW7T4L2OnUcpeTXMu9ZVuJexxqJwRcGUPDtp2cWWftSHafs4WVvOwqmlyNqwzofJ4Ko6krt5jTO0Cjoe47n2xU3C/kzoWXTWgoGGfcoOtOp67bDxQDqyo8022BYtACxfbRkBhqQeICVIdWBVYPxVS9eAkuAi/NUcOBxNHbY2RWwcszwklGuZALjWXPI8Am3OuuxhAawK3CHegFRC1Q5QrBqVAUNIxgIoI6QfmAY8xQvmUKPwnk8JcBrJZ5dsDqNAStA2m7nUIVZm4fQ80qrMqDaomqxrFRFXZEVXOsE1Wh5g20GpWcW1CiQi4VY+WzLMKv9pCrz525cyij14hasMIOuVRYltdCcpn2z2WWULqZiJUTnGH1n3hVksVXkP0iYUzr16w+ZyNmlXPKqxV5xuoUogifGX4eDjLIVLw9GiUk3PAsR1PcScnoyJvGZDKdHmfTsEbaNw2zIT6Np/Gi7gSncR+li2INPQLAqKP/Wzrdv3kwLxxUudUn6i+FtkuHm/2CL4sTNDDiJDqFv/bgly6s+aU6S2nzS+ttBd3TlAupAUOYOU7fo6hlnDwFQ4G2F6qukQo0talLT3a57yPZASjadorr8zTlXAKamloCrXNIaXWK6Xq0h67msCPLchpzaMbhExpFG6S9Lu4I/hA6oWmCvU26SYA48khIOMHNiwP9gFs/t0JLN+DCs3K5g8+tF3AzN+7ocnrjWTXN/gLaL+tkk6tO+M6Q9uz91B7rji9VDnJLL18pe5qHwp6BcUtQwFDUtySPmPY1a/dpI6BdwqP/kfDtdKCcpkoIDk6VDbS7oifQiZ6g93Qgw8jjV7mbnwQ9j/cKrObxlvaLHrA7OXCgtnD4dPUj/d00Zdu6NQX1ZFcPwLxLwfM+sbnzydbRis29n2y3rlVO4e9HvE8Z17FZu8eDZvBfxvwPFPkkDs492q9LVX96VT2J0fy04RwSru332+AM77ffi5o7HoM6NeMc1YwuX4HsVdpdg1H9EgE6oyqjfIUNRn09i1LDtz8m3uQEfRyh2G+19N/nITF48yGxCR05yDttR72cKd7A1UznkDdwolj+4yZvXv5vCd79Bw== \ No newline at end of file diff --git a/docs/future/diagrams/kit-components.svg b/docs/future/diagrams/kit-components.svg deleted file mode 100644 index 110930044..000000000 --- a/docs/future/diagrams/kit-components.svg +++ /dev/null @@ -1,3 +0,0 @@ - - -
retrieve adaptor .d.ts
retrieve adaptor .d.ts
Editor
Editor
describe adaptor
describe adaptor
Code Generator Widget
Code Generator Widget
analyzer
Prev. "compiler"
analyzer...
Module
on npm
Module...
Hosted Files
on unpkg.com
Hosted Files...
Common Adaptor Introspection Facilities
Common Adaptor Intro...
Typescript
Typescript
Workflow Diagram
Workflow Diagram
react-flow
react-flow
elk-js
elk-js
compiler
compiler
Adaptor Loading Facilities
Adaptor Loading Faci...
retrieve on demand
retrieve on demandText is not SVG - cannot display \ No newline at end of file diff --git a/docs/future/editor.md b/docs/future/editor.md deleted file mode 100644 index 216bd8eab..000000000 --- a/docs/future/editor.md +++ /dev/null @@ -1,25 +0,0 @@ -## Editor - -### Monaco - -**Adding libraries to Moanco** - -```ts -monaco.languages.typescript.typescriptDefaults.addExtraLib('const arr = [];') -``` - -**Hiding Editor Lines** - -https://github.com/microsoft/monaco-editor/issues/45#issuecomment-1159168677 - -```ts -interface IMyStandaloneCodeEditor extends monaco.editor.IStandaloneCodeEditor { - setHiddenAreas(range: monaco.IRange[]): void; -} - -... - -const casted = editor as IMyStandaloneCodeEditor; -const range = new monaco.Range(1, 0, 1, 0); -casted.setHiddenAreas([range]); -``` \ No newline at end of file diff --git a/docs/future/history.md b/docs/future/history.md deleted file mode 100644 index 99f1af56b..000000000 --- a/docs/future/history.md +++ /dev/null @@ -1,73 +0,0 @@ -# History of Core - -## How core works - -OpenFn [`core`](https://github.com/OpenFn/core) is an npm module, -that both transpiles and executes a job based on the following inputs: - -- A state file -- An adaptor path or module name -- An output path for "final state" - -![](diagrams/core-cli-requirements.svg) - -## Compilation - -The first thing that core does is try to compile an expression. -Since we allow users to write function calls -(without having to write require or import statements), -we have to transpile the code to be able to reference the adaptor functions. - -![](diagrams/core-compilation-steps.svg) - -The process outlined above would result in the transformation shown below: - -**Input** - -```js -fn((state) => { - return { ...state, counter: 1 }; -}); - -fn(({ counter, ...rest }) => { - return { ...rest, counter: counter + 1 }; -}); -``` - -**Output** - -```js -fn((state) => { - console.log(state); - return state; -})(function () { - return execute( - fn((state) => { - return { ...state, counter: 1 }; - }), - fn(({ counter, ...rest }) => { - return { ...rest, counter: counter + 1 }; - }), - fn((state) => { - console.log(state); - return state; - }) - )(state); -})(); -``` - -The execute function is an async reducer (using vanilla promises, -as async/await and generators were in Stage 0 at the time of implementation). -This pattern allows users to write code that ‘feels’ synchronous but is -executed asynchronously. - -## Execution - -The execution environment ties all the work together. - -![](diagrams/core-execution-steps.svg) - -It’s important to note that code is executed in a sandboxed environment, -where the vast majority of NodeJS root functionality is not available. -We also check in the compilation step that function calls that are not in -the ‘scope’ of our sandbox and throw errors in these cases. diff --git a/docs/future/kit-components.md b/docs/future/kit-components.md deleted file mode 100644 index 96bd0745f..000000000 --- a/docs/future/kit-components.md +++ /dev/null @@ -1,21 +0,0 @@ -Kit Components -============== - -![Draft Status](https://img.shields.io/badge/status-draft-red) - - -![](diagrams/kit-components.svg) - -## Analyzer - -See [here](../../packages/compiler/) - -## Compiler - -Produce transpiled/transformed expressions - -To be used from the worker to compile a job (or set of jobs) in the required -structure. - -Would contain all the transforms, and the helpers borne from creating the -transforms. diff --git a/integration-tests/cli/test/errors.test.ts b/integration-tests/cli/test/errors.test.ts index e9e5a80b5..1c3e66a43 100644 --- a/integration-tests/cli/test/errors.test.ts +++ b/integration-tests/cli/test/errors.test.ts @@ -101,7 +101,7 @@ test.serial('circular workflow', async (t) => { assertLog(t, stdlogs, /Error validating execution plan/i); assertLog(t, stdlogs, /Workflow failed/i); - const error = stdlogs.find((l) => l.message[0].severity); + const error = stdlogs.find((l) => l.message[0].name === 'ValidationError'); t.regex(error.message[0].message, /circular dependency: b <-> a/i); }); @@ -116,7 +116,7 @@ test.serial('multiple inputs', async (t) => { assertLog(t, stdlogs, /Error validating execution plan/i); assertLog(t, stdlogs, /Workflow failed/i); - const error = stdlogs.find((l) => l.message[0].severity); + const error = stdlogs.find((l) => l.message[0].name === 'ValidationError'); t.regex(error.message[0].message, /multiple dependencies detected for: c/i); }); @@ -132,6 +132,6 @@ test.serial('invalid start', async (t) => { assertLog(t, stdlogs, /Workflow failed/i); // Find the error obejct which is logged out - const error = stdlogs.find((l) => l.message[0].severity); + const error = stdlogs.find((l) => l.message[0].name === 'ValidationError'); t.regex(error.message[0].message, /could not find start job: nope/i); }); diff --git a/integration-tests/cli/test/metadata.test.ts b/integration-tests/cli/test/metadata.test.ts index 158e17483..7be1413dc 100644 --- a/integration-tests/cli/test/metadata.test.ts +++ b/integration-tests/cli/test/metadata.test.ts @@ -21,7 +21,6 @@ test.serial( `openfn metadata -S "${state}" -a test=${modulePath} --log-json --log info`, async (t) => { const { stdout } = await run(t.title); - t.regex(stdout, /Generating metadata/); t.regex(stdout, /Metadata function found. Generating metadata/); t.notRegex(stdout, /Returning metadata from cache/); diff --git a/integration-tests/worker/CHANGELOG.md b/integration-tests/worker/CHANGELOG.md index 0c46f0e99..1d9079065 100644 --- a/integration-tests/worker/CHANGELOG.md +++ b/integration-tests/worker/CHANGELOG.md @@ -1,5 +1,17 @@ # @openfn/integration-tests-worker +## 1.0.34 + +### Patch Changes + +- Updated dependencies [649ca43] +- Updated dependencies [823b471] +- Updated dependencies [823b471] + - @openfn/logger@0.0.20 + - @openfn/engine-multi@0.4.1 + - @openfn/ws-worker@0.8.1 + - @openfn/lightning-mock@1.2.1 + ## 1.0.33 ### Patch Changes diff --git a/integration-tests/worker/package.json b/integration-tests/worker/package.json index 605b1f523..a5544372c 100644 --- a/integration-tests/worker/package.json +++ b/integration-tests/worker/package.json @@ -1,7 +1,7 @@ { "name": "@openfn/integration-tests-worker", "private": true, - "version": "1.0.33", + "version": "1.0.34", "description": "Lightning WOrker integration tests", "author": "Open Function Group ", "license": "ISC", diff --git a/integration-tests/worker/test/integration.test.ts b/integration-tests/worker/test/integration.test.ts index eeb9c02e0..399968c81 100644 --- a/integration-tests/worker/test/integration.test.ts +++ b/integration-tests/worker/test/integration.test.ts @@ -147,7 +147,7 @@ test("Don't send job logs to stdout", (t) => { }; lightning.once('run:complete', () => { - const jsonLogs = engineLogger._history.map((l) => JSON.parse(l)); + const jsonLogs = engineLogger._history; // The engine logger shouldn't print out any job logs const jobLog = jsonLogs.find((l) => l.name === 'JOB'); diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 561ee4d38..1b3cb9dc5 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,15 @@ # @openfn/cli +## 0.4.16 + +### Patch Changes + +- Updated dependencies [649ca43] + - @openfn/logger@0.0.20 + - @openfn/compiler@0.0.39 + - @openfn/deploy@0.4.1 + - @openfn/runtime@0.2.6 + ## 0.4.15 ### Patch Changes diff --git a/packages/cli/package.json b/packages/cli/package.json index 67b286895..235678dd5 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/cli", - "version": "0.4.15", + "version": "0.4.16", "description": "CLI devtools for the openfn toolchain.", "engines": { "node": ">=18", diff --git a/packages/cli/test/util/print-versions.test.ts b/packages/cli/test/util/print-versions.test.ts index a07ae2a83..9594eb317 100644 --- a/packages/cli/test/util/print-versions.test.ts +++ b/packages/cli/test/util/print-versions.test.ts @@ -109,7 +109,7 @@ test('json output', async (t) => { const logger = createMockLogger('', { level: 'info', json: true }); await printVersions(logger, { adaptors: ['http'], logJson: true }); - const last = JSON.parse(logger._last) as JSONLog; + const last = logger._last as JSONLog; t.is(last.level, 'always'); const [{ versions }] = last.message; diff --git a/packages/compiler/CHANGELOG.md b/packages/compiler/CHANGELOG.md index aac1758c3..48300926a 100644 --- a/packages/compiler/CHANGELOG.md +++ b/packages/compiler/CHANGELOG.md @@ -1,5 +1,12 @@ # @openfn/compiler +## 0.0.39 + +### Patch Changes + +- Updated dependencies [649ca43] + - @openfn/logger@0.0.20 + ## 0.0.38 ### Patch Changes diff --git a/packages/compiler/package.json b/packages/compiler/package.json index 42b8dd818..1b44313c6 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/compiler", - "version": "0.0.38", + "version": "0.0.39", "description": "Compiler and language tooling for openfn jobs.", "author": "Open Function Group ", "license": "ISC", diff --git a/packages/deploy/CHANGELOG.md b/packages/deploy/CHANGELOG.md index 0bdf140aa..46ce5ec5d 100644 --- a/packages/deploy/CHANGELOG.md +++ b/packages/deploy/CHANGELOG.md @@ -1,5 +1,12 @@ # @openfn/deploy +## 0.4.1 + +### Patch Changes + +- Updated dependencies [649ca43] + - @openfn/logger@0.0.20 + ## 0.4.0 ### Minor Changes diff --git a/packages/deploy/package.json b/packages/deploy/package.json index 3d96784f0..3eb36e86a 100644 --- a/packages/deploy/package.json +++ b/packages/deploy/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/deploy", - "version": "0.4.0", + "version": "0.4.1", "description": "Deploy projects to Lightning instances", "type": "module", "exports": { diff --git a/packages/engine-multi/CHANGELOG.md b/packages/engine-multi/CHANGELOG.md index 0f0c245e7..e58b97b27 100644 --- a/packages/engine-multi/CHANGELOG.md +++ b/packages/engine-multi/CHANGELOG.md @@ -1,5 +1,15 @@ # engine-multi +## 0.4.1 + +### Patch Changes + +- 823b471: Update handling of logs so that JSON messages are stringified +- Updated dependencies [649ca43] + - @openfn/logger@0.0.20 + - @openfn/compiler@0.0.39 + - @openfn/runtime@0.2.6 + ## 0.4.0 ### Minor Changes diff --git a/packages/engine-multi/README.md b/packages/engine-multi/README.md index 3942e207a..bc7b1b9a9 100644 --- a/packages/engine-multi/README.md +++ b/packages/engine-multi/README.md @@ -110,3 +110,13 @@ engine.execute(plan, { resolvers }); ``` Initial state and credentials are at the moment pre-loaded, with a "fully resolved" state object passed into the runtime. The Runtime has the ability to lazy load but implementing lazy loading across the worker_thread interface has proven tricky. + +## Note on Debugging + +Debugging in the engine can be really tricky. + +First there's the problem that a lot of code runs inside a worker thread in a child process, which is hard to get a breakpoint into (at the time of writing I haven't managed to do it). + +But also, any console.log statements inside the inner thread will get consumed by the adaptor logger and won't go to stdout. + +As a workaround to this, use console.debug inside the thread to print to stdout. This is not bound to the adaptor logger. diff --git a/packages/engine-multi/package.json b/packages/engine-multi/package.json index e5852aa3d..4985a9f89 100644 --- a/packages/engine-multi/package.json +++ b/packages/engine-multi/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/engine-multi", - "version": "0.4.0", + "version": "0.4.1", "description": "Multi-process runtime engine", "main": "dist/index.js", "type": "module", @@ -17,7 +17,8 @@ "@openfn/compiler": "workspace:*", "@openfn/language-common": "2.0.0-rc3", "@openfn/logger": "workspace:*", - "@openfn/runtime": "workspace:*" + "@openfn/runtime": "workspace:*", + "fast-safe-stringify": "^2.1.1" }, "devDependencies": { "@types/node": "^18.15.13", diff --git a/packages/engine-multi/src/api/call-worker.ts b/packages/engine-multi/src/api/call-worker.ts index 4ecc1d441..9306b7101 100644 --- a/packages/engine-multi/src/api/call-worker.ts +++ b/packages/engine-multi/src/api/call-worker.ts @@ -34,14 +34,20 @@ export default function initWorkers( logger ); - const callWorker: CallWorker = (task, args = [], events = [], options = {}) => - workers.exec(task, args, { + const callWorker: CallWorker = ( + task, + args = [], + events = [], + options = {} + ) => { + return workers.exec(task, args, { ...options, on: ({ type, ...args }: WorkerEvent) => { // just call the callback events[type]?.(args); }, }); + }; const closeWorkers = async (instant?: boolean) => workers.destroy(instant); diff --git a/packages/engine-multi/src/api/execute.ts b/packages/engine-multi/src/api/execute.ts index e860f9011..c35085581 100644 --- a/packages/engine-multi/src/api/execute.ts +++ b/packages/engine-multi/src/api/execute.ts @@ -67,7 +67,7 @@ const execute = async (context: ExecutionContext) => { type: workerEvents.LOG, workflowId: state.plan.id!, threadId: '-', // no thread at this point - message: { + log: { level: 'debug', message: [`Memory limit: ${workerOptions.memoryLimitMb}mb`], name: 'RTE', @@ -80,7 +80,7 @@ const execute = async (context: ExecutionContext) => { type: workerEvents.LOG, workflowId: state.plan.id!, threadId: '-', // no thread at this point - message: { + log: { level: 'debug', message: [`Timeout: ${workerOptions.timeout / 1000}s`], name: 'RTE', diff --git a/packages/engine-multi/src/api/lifecycle.ts b/packages/engine-multi/src/api/lifecycle.ts index 33fac1737..68dcae76a 100644 --- a/packages/engine-multi/src/api/lifecycle.ts +++ b/packages/engine-multi/src/api/lifecycle.ts @@ -120,13 +120,22 @@ export const log = ( ) => { const { threadId } = event; - if (event.message.name !== 'JOB') { - context.logger.proxy(event.message); + if (event.log.name !== 'JOB') { + // Forward the log event to the engine's logger + // Note that we may have to parse the serialized log string + const proxy = { + ...event.log, + message: + typeof event.log.message == 'string' + ? JSON.parse(event.log.message) + : event.log.message, + }; + context.logger.proxy(proxy); } context.emit(externalEvents.WORKFLOW_LOG, { threadId, - ...event.message, + ...event.log, }); }; diff --git a/packages/engine-multi/src/api/validate-worker.ts b/packages/engine-multi/src/api/validate-worker.ts index fb29b9ffc..4be3bab91 100644 --- a/packages/engine-multi/src/api/validate-worker.ts +++ b/packages/engine-multi/src/api/validate-worker.ts @@ -11,7 +11,6 @@ export default async (api: EngineAPI, timeout = 5000) => { // TODO argument drive this await api.callWorker('handshake', [], {}, { timeout }); } catch (e) { - console.error(e); throw new Error('Invalid worker path'); } }; diff --git a/packages/engine-multi/src/events.ts b/packages/engine-multi/src/events.ts index a7af03b1c..4dc1d63e6 100644 --- a/packages/engine-multi/src/events.ts +++ b/packages/engine-multi/src/events.ts @@ -1,8 +1,5 @@ -// TODO remove ths file in favour of types - -// TODO mayberename event constants -import { JSONLog } from '@openfn/logger'; import { Versions } from './types'; +import { SerializedLogEvent } from './worker/events'; // If the worker thread exists a process safely, it'll return this error code // any other error code is unexpected @@ -89,7 +86,7 @@ export interface JobErrorPayload extends ExternalEvent { next: string[]; // downstream jobs } -export interface WorkerLogPayload extends ExternalEvent, JSONLog {} +export interface WorkerLogPayload extends ExternalEvent, SerializedLogEvent {} export interface EdgeResolvedPayload extends ExternalEvent { edgeId: string; // interesting, we don't really have this yet. Is index more appropriate? key? yeah, it's target node basically diff --git a/packages/engine-multi/src/worker/events.ts b/packages/engine-multi/src/worker/events.ts index 9e871e338..698df06eb 100644 --- a/packages/engine-multi/src/worker/events.ts +++ b/packages/engine-multi/src/worker/events.ts @@ -67,8 +67,14 @@ export interface JobErrorEvent extends InternalEvent { next: string[]; } +export type SerializedLogEvent = Omit & { + // the message is either an array of strings/object to log, + // or a JSON array that was previously serialized + message: string | any[]; +}; + export interface LogEvent extends InternalEvent { - message: JSONLog; + log: SerializedLogEvent; } export interface ErrorEvent { diff --git a/packages/engine-multi/src/worker/thread/helpers.ts b/packages/engine-multi/src/worker/thread/helpers.ts index 8d8767075..cb8a2d417 100644 --- a/packages/engine-multi/src/worker/thread/helpers.ts +++ b/packages/engine-multi/src/worker/thread/helpers.ts @@ -2,7 +2,7 @@ // This is designed to minimize the amount of code we have to mock import process from 'node:process'; - +import stringify from 'fast-safe-stringify'; import createLogger, { SanitizePolicies } from '@openfn/logger'; import * as workerEvents from '../events'; @@ -11,17 +11,22 @@ import { ExecutionError, ExitError } from '../../errors'; import { publish } from './runtime'; import serializeError from '../../util/serialize-error'; +import { JSONLog } from '@openfn/logger'; export const createLoggers = ( workflowId: string, - sanitize?: SanitizePolicies + sanitize: SanitizePolicies = 'none', + publish?: any ) => { - const log = (message: string) => { - // Apparently the json log stringifies the message - // We don't really want it to do that + const log = (message: JSONLog) => { publish(workerEvents.LOG, { workflowId, - message: JSON.parse(message), + log: { + ...message, + // stringify the message now so that we know it's safe + // this also makes it more performant to feed up to the worker + message: stringify(message.message), + }, } as workerEvents.LogEvent); }; @@ -41,6 +46,7 @@ export const createLoggers = ( json: true, sanitize, }); + const jobLogger = createLogger('JOB', { logger: emitter, level: 'debug', @@ -48,7 +54,14 @@ export const createLoggers = ( sanitize, }); - return { logger, jobLogger }; + const adaptorLogger = createLogger('ADA', { + logger: emitter, + level: 'debug', + json: true, + sanitize, + }); + + return { logger, jobLogger, adaptorLogger }; }; // Execute wrapper function diff --git a/packages/engine-multi/src/worker/thread/mock-run.ts b/packages/engine-multi/src/worker/thread/mock-run.ts index 54d6de72b..c6b29b0d8 100644 --- a/packages/engine-multi/src/worker/thread/mock-run.ts +++ b/packages/engine-multi/src/worker/thread/mock-run.ts @@ -32,7 +32,7 @@ type MockExecutionPlan = { // optionally delay function mockRun(plan: MockExecutionPlan) { const [job] = plan.jobs; - const { jobLogger } = createLoggers(plan.id!); + const { jobLogger } = createLoggers(plan.id!, 'none', publish); const workflowId = plan.id; return new Promise((resolve) => { const jobId = job.id || ''; diff --git a/packages/engine-multi/src/worker/thread/run.ts b/packages/engine-multi/src/worker/thread/run.ts index 196853859..b6af70c87 100644 --- a/packages/engine-multi/src/worker/thread/run.ts +++ b/packages/engine-multi/src/worker/thread/run.ts @@ -29,9 +29,23 @@ register({ run: (plan: ExecutionPlan, runOptions: RunOptions) => { const { adaptorPaths, whitelist, sanitize, statePropsToRemove } = runOptions; - const { logger, jobLogger } = createLoggers(plan.id!, sanitize); - // TODO I would like to pull these options out of here + const { logger, jobLogger, adaptorLogger } = createLoggers( + plan.id!, + sanitize, + publish + ); + + // Save the debug function so that we can use it + const debug = console.debug; + + // override console: any console.log statements will now get treated as adaptor logs + console = adaptorLogger; + // Leave console.debug for local debugging + // This goes to stdout but not the adpator logger + console.debug = debug; + + // TODO I would like to pull these options out of here const options = { // disable the run/step timeout timeout: 0, diff --git a/packages/engine-multi/test/__repo__/node_modules/@openfn/helper_1.0.0/index.js b/packages/engine-multi/test/__repo__/node_modules/@openfn/helper_1.0.0/index.js new file mode 100644 index 000000000..54e37b93c --- /dev/null +++ b/packages/engine-multi/test/__repo__/node_modules/@openfn/helper_1.0.0/index.js @@ -0,0 +1,13 @@ +export function exit() { + return function (state) { + process.exit(42) + return state; + } +}; + +export function log(message) { + return function (state) { + console.log(message) + return state; + } +}; diff --git a/packages/engine-multi/test/__repo__/node_modules/helper_1.0.0/package.json b/packages/engine-multi/test/__repo__/node_modules/@openfn/helper_1.0.0/package.json similarity index 56% rename from packages/engine-multi/test/__repo__/node_modules/helper_1.0.0/package.json rename to packages/engine-multi/test/__repo__/node_modules/@openfn/helper_1.0.0/package.json index 429f14f13..a3d7607fb 100644 --- a/packages/engine-multi/test/__repo__/node_modules/helper_1.0.0/package.json +++ b/packages/engine-multi/test/__repo__/node_modules/@openfn/helper_1.0.0/package.json @@ -1,7 +1,7 @@ { - "name": "helper", + "name": "@openfn/helper", "version": "1.0.0", "type": "module", - "main": "index.cjs", + "main": "index.js", "private": true } diff --git a/packages/engine-multi/test/__repo__/node_modules/helper_1.0.0/index.cjs b/packages/engine-multi/test/__repo__/node_modules/helper_1.0.0/index.cjs deleted file mode 100644 index 42a79abc5..000000000 --- a/packages/engine-multi/test/__repo__/node_modules/helper_1.0.0/index.cjs +++ /dev/null @@ -1,8 +0,0 @@ -module.exports = { - exit: function() { - return function (state) { - process.exit(42) - return state; - } - } -}; diff --git a/packages/engine-multi/test/__repo__/package.json b/packages/engine-multi/test/__repo__/package.json index 679a959d6..5d53c92d6 100644 --- a/packages/engine-multi/test/__repo__/package.json +++ b/packages/engine-multi/test/__repo__/package.json @@ -3,6 +3,6 @@ "private": true, "version": "1.0.0", "dependencies": { - "helper_1.0.0": "@npm:helper@1.0.0" + "@openfn/helper_1.0.0": "@npm:@openfn/helper@1.0.0" } } diff --git a/packages/engine-multi/test/api/execute.test.ts b/packages/engine-multi/test/api/execute.test.ts index 2eca8194a..deda81d22 100644 --- a/packages/engine-multi/test/api/execute.test.ts +++ b/packages/engine-multi/test/api/execute.test.ts @@ -186,8 +186,8 @@ test.serial('should emit a log event', async (t) => { await execute(context); t.is(workflowLog.workflowId, 'y'); - t.is(workflowLog.message[0], 'hi'); t.is(workflowLog.level, 'info'); + t.deepEqual(workflowLog.message, JSON.stringify(['hi'])); }); test.serial('log events are timestamped in hr time', async (t) => { diff --git a/packages/engine-multi/test/api/lifecycle.test.ts b/packages/engine-multi/test/api/lifecycle.test.ts index 624f72388..b6c0566a2 100644 --- a/packages/engine-multi/test/api/lifecycle.test.ts +++ b/packages/engine-multi/test/api/lifecycle.test.ts @@ -170,10 +170,10 @@ test(`log: emits ${e.WORKFLOW_LOG}`, (t) => { const event = { workflowId, threadId: 'a', - message: { + log: { level: 'info', name: 'job', - message: ['oh hai'], + message: JSON.stringify(['oh hai']), time: Date.now() - 100, }, }; @@ -182,7 +182,7 @@ test(`log: emits ${e.WORKFLOW_LOG}`, (t) => { t.deepEqual(evt, { workflowId, threadId: 'a', - ...event.message, + ...event.log, }); done(); }); diff --git a/packages/engine-multi/test/errors.test.ts b/packages/engine-multi/test/errors.test.ts index 80860876b..e9202584b 100644 --- a/packages/engine-multi/test/errors.test.ts +++ b/packages/engine-multi/test/errors.test.ts @@ -3,7 +3,7 @@ import path from 'node:path'; import createEngine, { EngineOptions } from '../src/engine'; import { createMockLogger } from '@openfn/logger'; -import { WORKFLOW_COMPLETE, WORKFLOW_ERROR } from '../src/events'; +import { WORKFLOW_ERROR } from '../src/events'; let engine; @@ -160,7 +160,7 @@ test.serial('emit a crash error on process.exit()', (t) => { id: 'z', jobs: [ { - adaptor: 'helper@1.0.0', + adaptor: '@openfn/helper@1.0.0', expression: 'export default [exit()]', }, ], diff --git a/packages/engine-multi/test/integration.test.ts b/packages/engine-multi/test/integration.test.ts index 129770fe7..fed31f5b5 100644 --- a/packages/engine-multi/test/integration.test.ts +++ b/packages/engine-multi/test/integration.test.ts @@ -1,4 +1,5 @@ import test from 'ava'; +import path from 'node:path'; import createAPI from '../src/api'; import { createMockLogger } from '@openfn/logger'; @@ -115,6 +116,8 @@ test.serial('trigger workflow-complete', (t) => { const plan = createPlan(); api.execute(plan).on('workflow-complete', (evt) => { + t.falsy(evt.state.errors); + t.is(evt.workflowId, plan.id); t.truthy(evt.duration); t.truthy(evt.state); @@ -129,20 +132,85 @@ test.serial('trigger workflow-log for job logs', (t) => { return new Promise(async (done) => { api = await createAPI({ logger, - compile: { - skip: true, + }); + + const plan = createPlan([ + { + expression: `${withFn}fn((s) => { console.log('hola'); return s; })`, }, + ]); + + let didLog = false; + + api.execute(plan).on('workflow-log', (evt) => { + if (evt.name === 'JOB') { + didLog = true; + t.deepEqual(evt.message, JSON.stringify(['hola'])); + t.pass('workflow logged'); + } + }); + + api.execute(plan).on('workflow-complete', (evt) => { + t.true(didLog); + t.falsy(evt.state.errors); + done(); + }); + }); +}); + +test.serial('log errors', (t) => { + return new Promise(async (done) => { + api = await createAPI({ + logger, }); const plan = createPlan([ { - expression: `${withFn}console.log('hola')`, + expression: `${withFn}fn((s) => { console.log(new Error('hola')); return s; })`, }, ]); api.execute(plan).on('workflow-log', (evt) => { if (evt.name === 'JOB') { - t.deepEqual(evt.message, ['hola']); + t.log(evt); + t.deepEqual( + evt.message, + JSON.stringify([ + { + name: 'Error', + message: 'hola', + }, + ]) + ); + t.pass('workflow logged'); + } + }); + + api.execute(plan).on('workflow-complete', (evt) => { + done(); + }); + }); +}); + +test.serial('trigger workflow-log for adaptor logs', (t) => { + return new Promise(async (done) => { + api = await createAPI({ + logger, + repoDir: path.resolve('./test/__repo__'), + }); + + const plan = createPlan([ + { + // This will trigger console.log from inside the adaptor + // rather than from job code directly + expression: "log('hola')", + adaptor: '@openfn/helper@1.0.0', + }, + ]); + + api.execute(plan).on('workflow-log', (evt) => { + if (evt.name === 'ADA') { + t.deepEqual(evt.message, JSON.stringify(['hola'])); t.pass('workflow logged'); done(); } diff --git a/packages/engine-multi/test/worker/helper.test.ts b/packages/engine-multi/test/worker/helper.test.ts new file mode 100644 index 000000000..445fac3ed --- /dev/null +++ b/packages/engine-multi/test/worker/helper.test.ts @@ -0,0 +1,42 @@ +import test from 'ava'; + +import { createLoggers } from '../../src/worker/thread/helpers'; + +test('createLogger: runtime logger should emit an event on log', (t) => { + const message = 'testing1234'; + + const publish = (type: string, payload: any) => { + t.is(type, 'worker:log'); + t.is(payload.workflowId, 'x'); + t.is(payload.log.level, 'info'); + t.is(payload.log.name, 'R/T'); + + // The log message is always encoded into a string + const parsedMessage = JSON.parse(payload.log.message); + t.deepEqual(parsedMessage, [message]); + }; + + const { logger } = createLoggers('x', 'none', publish); + + logger.log(message); +}); + +test('createLogger: runtime logger should emit a nicely serialised error on log', (t) => { + const message = new Error('err'); + + const publish = (type: string, payload: any) => { + t.is(type, 'worker:log'); + + const parsedMessage = JSON.parse(payload.log.message); + t.deepEqual(parsedMessage, [ + { + name: 'Error', + message: 'err', + }, + ]); + }; + + const { logger } = createLoggers('x', 'none', publish); + + logger.log(message); +}); diff --git a/packages/engine-multi/test/worker/mock-worker.test.ts b/packages/engine-multi/test/worker/mock-worker.test.ts index 19486175e..679f663a1 100644 --- a/packages/engine-multi/test/worker/mock-worker.test.ts +++ b/packages/engine-multi/test/worker/mock-worker.test.ts @@ -145,10 +145,10 @@ test('Publish a job log event', async (t) => { let log; let id; await workers.exec('run', [plan], { - on: ({ workflowId, type, message }) => { + on: ({ workflowId, type, log: _log }) => { if (type === e.LOG) { didFire = true; - log = message; + log = _log; id = workflowId; } }, @@ -157,7 +157,7 @@ test('Publish a job log event', async (t) => { t.is(id, plan.id); t.is(log.level, 'info'); - t.deepEqual(log.message, ['test']); t.is(log.name, 'JOB'); t.truthy(log.time); + t.deepEqual(log.message, JSON.stringify(['test'])); }); diff --git a/packages/lightning-mock/CHANGELOG.md b/packages/lightning-mock/CHANGELOG.md index b66106aa0..1184018dd 100644 --- a/packages/lightning-mock/CHANGELOG.md +++ b/packages/lightning-mock/CHANGELOG.md @@ -1,5 +1,15 @@ # @openfn/lightning-mock +## 1.2.1 + +### Patch Changes + +- Updated dependencies [649ca43] +- Updated dependencies [823b471] + - @openfn/logger@0.0.20 + - @openfn/engine-multi@0.4.1 + - @openfn/runtime@0.2.6 + ## 1.2.0 ### Minor Changes diff --git a/packages/lightning-mock/package.json b/packages/lightning-mock/package.json index bae475e51..32b41dfd4 100644 --- a/packages/lightning-mock/package.json +++ b/packages/lightning-mock/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/lightning-mock", - "version": "1.2.0", + "version": "1.2.1", "private": true, "description": "A mock Lightning server", "main": "dist/index.js", diff --git a/packages/logger/CHANGELOG.md b/packages/logger/CHANGELOG.md index 07aff0583..39c72e12a 100644 --- a/packages/logger/CHANGELOG.md +++ b/packages/logger/CHANGELOG.md @@ -1,5 +1,12 @@ # @openfn/logger +## 0.0.20 + +### Patch Changes + +- 649ca43: In JSON mode, do not stringify emitted messages. + Better handling of error objects + ## 0.0.19 ### Patch Changes diff --git a/packages/logger/package.json b/packages/logger/package.json index 65c3f61bf..e0b75aa14 100644 --- a/packages/logger/package.json +++ b/packages/logger/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/logger", - "version": "0.0.19", + "version": "0.0.20", "description": "Cross-package logging utility", "module": "dist/index.js", "author": "Open Function Group ", diff --git a/packages/logger/src/logger.ts b/packages/logger/src/logger.ts index 73c487c3c..74e59e486 100644 --- a/packages/logger/src/logger.ts +++ b/packages/logger/src/logger.ts @@ -1,6 +1,5 @@ import c from 'chalk'; import iconfirm from '@inquirer/confirm'; -import stringify from 'fast-safe-stringify'; import * as symbols from './symbols'; import sanitize from './sanitize'; import getDurationString from './util/duration'; @@ -163,6 +162,7 @@ export default function (name?: string, options: LogOptions = {}): Logger { sanitize(o, { stringify: false, policy: options.sanitize, + serializeErrors: true, }) ); if (message.length === 1 && message[0] === null) { @@ -178,7 +178,10 @@ export default function (name?: string, options: LogOptions = {}): Logger { time: hrtimestamp().toString(), }; - emitter[level](stringify(output)); + // Emit the output directly, without any further + // serialisation. Note that this may cause us to log + // non-serialisable stuff + emitter[level](output); }; const logString = ( @@ -237,7 +240,7 @@ export default function (name?: string, options: LogOptions = {}): Logger { const print = (...args: any[]) => { if (opts.level !== NONE) { if (opts.json) { - emitter.info(JSON.stringify({ message: args })); + emitter.info({ message: args }); } else { emitter.info(...args); } diff --git a/packages/logger/src/options.ts b/packages/logger/src/options.ts index d51d05fd7..6cfa7f7a7 100644 --- a/packages/logger/src/options.ts +++ b/packages/logger/src/options.ts @@ -1,4 +1,6 @@ import { SanitizePolicies } from './sanitize'; +import defaultEmitter from './util/default-emitter'; +import jsonEmitter from './util/json-emitter'; export type LogLevel = 'debug' | 'info' | 'default' | 'none'; @@ -36,24 +38,12 @@ export type LogOptions = { sanitize?: SanitizePolicies; }; -// TODO not crazy about the handling of this -// but to support the success handler we need to alias console.log -const defaultEmitter = { - ...console, - // Direct error and warn logs to stdout, so that they appear in sequence - error: (...args: any[]) => console.log(...args), - warn: (...args: any[]) => console.log(...args), - success: (...args: any[]) => console.log(...args), - always: (...args: any[]) => console.log(...args), -}; - export const defaults: Required = { level: 'default', - // TODO support an array of emitters here - logger: defaultEmitter, // I guess? hideNamespace: false, hideIcons: false, + logger: defaultEmitter, // Not implemented wrap: false, @@ -70,6 +60,9 @@ const parseOptions = (opts: LogOptions = {}): Required => { // First default all values const options = { ...defaults, + // If logging to json, and no emitter is provided, + // use this emitter which will serialise the output to JSON + logger: opts.json ? jsonEmitter : defaultEmitter, ...opts, }; diff --git a/packages/logger/src/sanitize.ts b/packages/logger/src/sanitize.ts index 00c980da3..7597a8d95 100644 --- a/packages/logger/src/sanitize.ts +++ b/packages/logger/src/sanitize.ts @@ -15,6 +15,8 @@ type SanitizeOptions = { // This is potentially important so we do want to break // but! we should throw in the CLI< not here. policy?: SanitizePolicies; + + serializeErrors?: boolean; // false by default }; const scrubbers: Record any> = { @@ -24,13 +26,37 @@ const scrubbers: Record any> = { none: (item) => item, }; +// If an error is generated inside the sandbox, it does not seem to be instanceof error +// So let's walk the prototype chain to see if it LOOKs like an error +const isError = (obj: any) => { + if (obj instanceof Error) { + return true; + } + + let o = obj; + while (o && o.constructor) { + if (o.constructor.name === 'Error') { + return true; + } + o = o.prototype?.constructor; + } + return false; +}; + // Sanitize console output const sanitize = (item: any, options: SanitizeOptions = {}) => { // Stringify output to ensure we show deep nesting const maybeStringify = (o: any) => options.stringify === false ? o : stringify(o, undefined, 2); - if (item instanceof Error) { + if (isError(item)) { + if (options.serializeErrors) { + return { + name: item.name, + message: item.message || item.toString(), + // TODO stack? Tricky + }; + } return item; } diff --git a/packages/logger/src/util/default-emitter.ts b/packages/logger/src/util/default-emitter.ts new file mode 100644 index 000000000..e7579c6bc --- /dev/null +++ b/packages/logger/src/util/default-emitter.ts @@ -0,0 +1,10 @@ +import type { LogEmitter } from '../options'; + +export default { + ...console, + // Direct error and warn logs to stdout, so that they appear in sequence + error: (...args: any[]) => console.log(...args), + warn: (...args: any[]) => console.log(...args), + success: (...args: any[]) => console.log(...args), + always: (...args: any[]) => console.log(...args), +} as LogEmitter; diff --git a/packages/logger/src/util/json-emitter.ts b/packages/logger/src/util/json-emitter.ts new file mode 100644 index 000000000..b8f688ca5 --- /dev/null +++ b/packages/logger/src/util/json-emitter.ts @@ -0,0 +1,16 @@ +import stringify from 'fast-safe-stringify'; +import { LogEmitter } from '../options'; + +const jsonEmitter: Partial = { + ...console, +}; + +['log', 'info', 'success', 'always', 'debug', 'warn', 'error'].forEach((fn) => { + // @ts-ignore + jsonEmitter[fn] = (...args: any[]) => { + const stringified = args.map((value) => stringify(value)); + console.log(...stringified); + }; +}); + +export default jsonEmitter as LogEmitter; diff --git a/packages/logger/test/logger.test.ts b/packages/logger/test/logger.test.ts index f5a2a4166..53e9a7515 100644 --- a/packages/logger/test/logger.test.ts +++ b/packages/logger/test/logger.test.ts @@ -1,6 +1,6 @@ import test from 'ava'; import chalk from 'chalk'; -import { styleLevel, LogFns, StringLog } from '../src/logger'; +import { styleLevel, LogFns, StringLog, JSONLog } from '../src/logger'; import { defaults as defaultOptions, LogLevel } from '../src/options'; import hrtimestamp from '../src/util/timestamp'; import { SECRET } from '../src/sanitize'; @@ -194,10 +194,10 @@ test('sanitize: summarise object', (t) => { test(`${level} - as json`, (t) => { const options = { level, json: true }; - const logger = createLogger('x', options); + const logger = createLogger('x', options); logger[fn]('abc'); - const result = JSON.parse(logger._last); + const result = logger._last; t.assert(Object.keys(result).length === 4); t.assert(result.level === level); @@ -208,14 +208,14 @@ test('sanitize: summarise object', (t) => { }); test(`JSON timestamps are bigints representing sensible times`, (t) => { - const testStartTime = new Date().toISOString() + const testStartTime = new Date().toISOString(); const startTime = hrtimestamp(); const options = { level: 'info' as const, json: true }; - const logger = createLogger('x', options); + const logger = createLogger('x', options); logger.info("what's the time mr wolf"); - const { time } = JSON.parse(logger._last); + const { time } = logger._last; // The time we get here is NOT a bigint because it's been serialized t.true(typeof time === 'string'); t.is(time.length, 19); @@ -244,24 +244,21 @@ big end time: ${endDate.toISOString()}`); t.true(endTime - startTime < 1e6); }); -// TODO this test needs to pass without the timeout test('timestamps increase in time', async (t) => { const options = { level: 'info' as const, json: true }; - const logger = createLogger('x', options); + const logger = createLogger('x', options); - for(let i = 0; i < 10; i += 1) { - // await new Promise(done => setTimeout(done, 2)) + for (let i = 0; i < 10; i += 1) { logger.info("what's the time mr wolf"); } - let last = 0; - logger._history.forEach(l => { - const { time } = JSON.parse(l); - t.log(time) - t.true(time > last) + let last = '0'; + logger._history.forEach(({ time }) => { + t.log(typeof time, time); + t.true(time > last); last = time; - }) -}) + }); +}); test('print() should be barebones', (t) => { const options = { level: 'default' as const }; @@ -283,10 +280,11 @@ test('print() should not log if level is none', (t) => { test('print() should log as json', (t) => { const options = { json: true }; - const logger = createLogger('x', options); + const logger = createLogger('x', options); logger.print('abc'); - const [level, message] = logger._last; + // @ts-ignore + const { level, message } = logger._parse(logger._last); t.is(level, 'print'); t.deepEqual(message, { message: ['abc'] }); }); @@ -333,6 +331,20 @@ test('in json mode with level=none, logs errors only', (t) => { t.assert(logger._history.length === 1); }); +test('json mode should serialize errors nicely', (t) => { + const logger = createLogger(undefined, { + level: 'debug', + json: true, + }); + const e = new Error('wibble'); + + logger.info(e); + + const result = logger._last; + t.is(result.level, 'info'); + t.deepEqual(result.message[0], { name: 'Error', message: 'wibble' }); +}); + test('with level=default, logs success, error and warning but not info and debug', (t) => { const logger = createLogger('x', { level: 'default' }); @@ -429,14 +441,14 @@ test('sanitize state in second arg', (t) => { }); test('sanitize state in json logging', (t) => { - const logger = createLogger(undefined, { json: true }); + const logger = createLogger(undefined, { json: true }); logger.success({ configuration: { x: 'y', }, data: {}, }); - const { message } = JSON.parse(logger._last); + const { message } = logger._last; t.is(message[0].configuration.x, SECRET); }); @@ -487,16 +499,21 @@ test('log a circular object', async (t) => { ); }); -test('log a circular object as JSON', async (t) => { +// This fails now because I'm not stringifying the output +// I think this has to just be OK. +// Maybe the worker needs to stringify the result downstream, +// but I don't think the logger itself should? +test.skip('log a circular object as JSON', async (t) => { const z: any = {}; const a = { z, }; z.a = a; - const logger = createLogger(undefined, { json: true }); + const logger = createLogger(undefined, { json: true }); logger.success(a); - const { message } = JSON.parse(logger._last); + const { message } = logger._last; + t.log(message); t.deepEqual(message[0], { z: { a: '[Circular]', @@ -550,20 +567,20 @@ test('proxy string arguments to string', (t) => { }); test('proxy a json argument to json', (t) => { - const logger = createLogger('x', { json: true }); + const logger = createLogger('x', { json: true }); logger.proxy({ name: 'y', level: 'success', message: ['hello'] }); - const { name, level, message } = JSON.parse(logger._last as any); + const { name, level, message } = logger._last; t.is(name, 'y'); t.is(level, 'success'); t.deepEqual(message, ['hello']); }); test('proxy string arguments to json', (t) => { - const logger = createLogger('x', { json: true }); + const logger = createLogger('x', { json: true }); logger.proxy('y', 'success', ['hello']); - const { name, level, message } = JSON.parse(logger._last as any); + const { name, level, message } = logger._last; t.is(name, 'y'); t.is(level, 'success'); t.deepEqual(message, ['hello']); diff --git a/packages/logger/test/mock.test.ts b/packages/logger/test/mock.test.ts index 8c5613674..4444ffbc6 100644 --- a/packages/logger/test/mock.test.ts +++ b/packages/logger/test/mock.test.ts @@ -1,6 +1,7 @@ import test from 'ava'; import chalk from 'chalk'; import mockLogger from '../src/mock'; +import { JSONLog } from '../src'; // disable chalk colours in unit tests chalk.level = 0; @@ -179,14 +180,14 @@ test('print should include the message', async (t) => { }); test('log JSON', async (t) => { - const logger = mockLogger('a', { json: true }); + const logger = mockLogger('a', { json: true }); logger.success('z'); - const { level, message, name, time } = JSON.parse(logger._last); + const { level, message, name, time } = logger._last; t.is(name, 'a'); t.is(level, 'success'); t.is(message[0], 'z'); - t.true(!isNaN(time)); + t.true(typeof time === 'string'); }); test('find a log', (t) => { diff --git a/packages/logger/test/sanitize.test.ts b/packages/logger/test/sanitize.test.ts index 0c53a340f..834571657 100644 --- a/packages/logger/test/sanitize.test.ts +++ b/packages/logger/test/sanitize.test.ts @@ -40,6 +40,15 @@ test("Don't stringify a custom error", (t) => { t.assert(result instanceof Error); }); +test('do stringify an error if asked to', (t) => { + const e = new Error('test'); + const result = sanitize(e, { serializeErrors: true }); + t.deepEqual(result, { + name: 'Error', + message: 'test', + }); +}); + test('stringify an object', (t) => { const result = sanitize({}); t.is(result, '{}'); diff --git a/packages/logger/test/util/json-emitter.test.ts b/packages/logger/test/util/json-emitter.test.ts new file mode 100644 index 000000000..346971f27 --- /dev/null +++ b/packages/logger/test/util/json-emitter.test.ts @@ -0,0 +1,65 @@ +import test from 'ava'; +import jsonEmitter from '../../src/util/json-emitter'; +import { LogFns } from '../../src/logger'; + +const levels: LogFns[] = [ + 'log', + 'info', + 'success', + 'always', + 'debug', + 'warn', + 'error', +]; + +const history: any[] = []; + +test.before(() => { + // All json functions emit to log - so we just have to override that one function here + console.log = (...args: any[]) => { + history.push(args); + }; +}); + +levels.forEach((level) => { + test(`should log a string to ${level}`, (t) => { + jsonEmitter[level]('hello'); + + const last = history.pop(); + t.is(last.length, 1); + t.is(last[0], '"hello"'); + }); + + test(`should log a number to ${level}`, (t) => { + jsonEmitter[level](1); + + const last = history.pop(); + t.is(last.length, 1); + t.is(last[0], '1'); + }); + + test(`should log a boolean to ${level}`, (t) => { + jsonEmitter[level](false); + + const last = history.pop(); + t.is(last.length, 1); + t.is(last[0], 'false'); + }); + + test(`should log an error to ${level}`, (t) => { + jsonEmitter[level](new Error('err')); + + const last = history.pop(); + t.is(last.length, 1); + t.is(last[0], '{}'); + }); + + test(`should log an object to ${level}`, (t) => { + const o = { a: 1, b: 2, c: 3 }; + jsonEmitter[level](o); + + const last = history.pop(); + t.is(last.length, 1); + t.is(last[0], JSON.stringify(o)); + }); +}); diff --git a/packages/runtime/CHANGELOG.md b/packages/runtime/CHANGELOG.md index 9d1a0158a..c993ebd55 100644 --- a/packages/runtime/CHANGELOG.md +++ b/packages/runtime/CHANGELOG.md @@ -1,5 +1,12 @@ # @openfn/runtime +## 0.2.6 + +### Patch Changes + +- Updated dependencies [649ca43] + - @openfn/logger@0.0.20 + ## 0.2.5 ### Patch Changes diff --git a/packages/runtime/package.json b/packages/runtime/package.json index f4c1be126..d176dc9a5 100644 --- a/packages/runtime/package.json +++ b/packages/runtime/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/runtime", - "version": "0.2.5", + "version": "0.2.6", "description": "Job processing runtime.", "type": "module", "exports": { diff --git a/packages/runtime/src/modules/linker.ts b/packages/runtime/src/modules/linker.ts index 09c689a6f..42d848888 100644 --- a/packages/runtime/src/modules/linker.ts +++ b/packages/runtime/src/modules/linker.ts @@ -69,7 +69,6 @@ const linker: Linker = async (specifier, context, options = {}) => { target = target.default; } } - const exportNames = Object.keys(target); // Wrap up the real module into a Synthetic Module const m = new vm.SyntheticModule( diff --git a/packages/runtime/test/runtime.test.ts b/packages/runtime/test/runtime.test.ts index 48c735f16..d4bb1888d 100644 --- a/packages/runtime/test/runtime.test.ts +++ b/packages/runtime/test/runtime.test.ts @@ -386,6 +386,51 @@ test('log errors, write to state, and continue', async (t) => { t.truthy(logger._find('error', /failed job a/i)); }); +test('log job code to the job logger', async (t) => { + const plan: ExecutionPlan = { + jobs: [ + { + id: 'a', + expression: 'export default [(s) => { console.log("hi"); return s;}]', + }, + ], + }; + + const jobLogger = createMockLogger('JOB', { level: 'debug', json: true }); + await run(plan, {}, { jobLogger }); + + t.is(jobLogger._history.length, 1); + const [out] = jobLogger._history; + + t.is(out.level, 'info'); + t.is(out.message[0], 'hi'); +}); + +test('log and serialize an error to the job logger', async (t) => { + const plan: ExecutionPlan = { + jobs: [ + { + id: 'a', + expression: + 'export default [(s) => { console.log(new Error("hi")); return s;}]', + }, + ], + }; + + const jobLogger = createMockLogger('JOB', { level: 'debug', json: true }); + await run(plan, {}, { jobLogger }); + + t.is(jobLogger._history.length, 1); + const [out] = jobLogger._history; + t.log(out); + + t.is(out.level, 'info'); + t.is(out.message[0].name, 'Error'); + t.is(out.message[0].message, 'hi'); + // should not be an error instance + t.falsy(out.message[0].stack); +}); + test('error reports can be overwritten', async (t) => { const plan: ExecutionPlan = { jobs: [ diff --git a/packages/ws-worker/CHANGELOG.md b/packages/ws-worker/CHANGELOG.md index 642bb9f92..9fda2dddf 100644 --- a/packages/ws-worker/CHANGELOG.md +++ b/packages/ws-worker/CHANGELOG.md @@ -1,5 +1,16 @@ # ws-worker +## 0.8.1 + +### Patch Changes + +- 823b471: Update handling of logs to accept stringified messages +- Updated dependencies [649ca43] +- Updated dependencies [823b471] + - @openfn/logger@0.0.20 + - @openfn/engine-multi@0.4.1 + - @openfn/runtime@0.2.6 + ## 0.8.0 ### Minor Changes diff --git a/packages/ws-worker/package.json b/packages/ws-worker/package.json index 670a14606..0872384f1 100644 --- a/packages/ws-worker/package.json +++ b/packages/ws-worker/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/ws-worker", - "version": "0.8.0", + "version": "0.8.1", "description": "A Websocket Worker to connect Lightning to a Runtime Engine", "main": "dist/index.js", "type": "module", diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index 6bfd7046d..9ce817bf4 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -9,23 +9,24 @@ import { STEP_COMPLETE, STEP_START, } from '../events'; -import { RunOptions, Channel, RunState } from '../types'; -import { getWithReply, createRunState } from '../util'; +import { + getWithReply, + createRunState, + throttle as createThrottle, +} from '../util'; +import handleStepComplete from '../events/step-complete'; +import handleStepStart from '../events/step-start'; +import handleRunComplete from '../events/run-complete'; +import handleRunError from '../events/run-error'; -import type { JSONLog, Logger } from '@openfn/logger'; +import type { RunOptions, Channel, RunState, JSONLog } from '../types'; +import type { Logger } from '@openfn/logger'; import type { RuntimeEngine, Resolvers, WorkflowStartPayload, } from '@openfn/engine-multi'; -import { ExecutionPlan } from '@openfn/runtime'; - -import handleStepComplete from '../events/step-complete'; -import handleStepStart from '../events/step-start'; -import handleRunComplete from '../events/run-complete'; -import handleRunError from '../events/run-error'; - -import createThrottler from '../util/throttle'; +import type { ExecutionPlan } from '@openfn/runtime'; const enc = new TextDecoder('utf-8'); @@ -66,7 +67,7 @@ export function execute( const context: Context = { channel, state, logger, engine, onFinish }; - const throttle = createThrottler(); + const throttle = createThrottle(); type EventHandler = (context: any, event: any) => void; @@ -205,7 +206,14 @@ export function onJobLog({ channel, state }: Context, event: JSONLog) { // lightning-friendly log object const log: RunLogPayload = { run_id: state.plan.id!, - message: event.message, + // The message body, the actual thing that is logged, + // may be always encoded into a string + // Parse it here before sending on to lightning + // TODO this needs optimising! + message: + typeof event.message === 'string' + ? JSON.parse(event.message) + : event.message, source: event.name, level: event.level, timestamp: timeInMicroseconds.toString(), diff --git a/packages/ws-worker/src/mock/runtime-engine.ts b/packages/ws-worker/src/mock/runtime-engine.ts index 068575fb2..b8f2741e5 100644 --- a/packages/ws-worker/src/mock/runtime-engine.ts +++ b/packages/ws-worker/src/mock/runtime-engine.ts @@ -109,7 +109,7 @@ async function createMock() { threadId: threadId, level: 'info', json: true, - message: args, + message: JSON.stringify(args), time: Date.now(), }); }, diff --git a/packages/ws-worker/src/types.d.ts b/packages/ws-worker/src/types.d.ts index 614fdb87b..8cc0709dd 100644 --- a/packages/ws-worker/src/types.d.ts +++ b/packages/ws-worker/src/types.d.ts @@ -113,3 +113,9 @@ export interface Channel extends PhxChannel { push:

(event: string, payload?: P) => ReceiveHook; // join: () => ReceiveHook; } + +// override the JSON log typing because the log message +// might be JSON encoded in a string +export type JSONLog = Omit & { + message: string | any[]; +}; diff --git a/packages/ws-worker/src/util/index.ts b/packages/ws-worker/src/util/index.ts index e7a85a406..6c9b2b0e3 100644 --- a/packages/ws-worker/src/util/index.ts +++ b/packages/ws-worker/src/util/index.ts @@ -3,8 +3,10 @@ import tryWithBackoff from './try-with-backoff'; import getWithReply from './get-with-reply'; import stringify from './stringify'; import createRunState from './create-run-state'; +import throttle from './throttle'; export { + throttle, convertRun, tryWithBackoff, getWithReply, diff --git a/packages/ws-worker/test/api/execute.test.ts b/packages/ws-worker/test/api/execute.test.ts index 0128ec5e6..3d23375ba 100644 --- a/packages/ws-worker/test/api/execute.test.ts +++ b/packages/ws-worker/test/api/execute.test.ts @@ -1,5 +1,5 @@ import test from 'ava'; -import { JSONLog, createMockLogger } from '@openfn/logger'; +import { createMockLogger } from '@openfn/logger'; import { STEP_START, @@ -24,7 +24,7 @@ import { mockChannel } from '../../src/mock/sockets'; import { stringify, createRunState } from '../../src/util'; import type { ExecutionPlan } from '@openfn/runtime'; -import type { Run, RunState } from '../../src/types'; +import type { Run, RunState, JSONLog } from '../../src/types'; const enc = new TextEncoder(); @@ -71,7 +71,7 @@ test('jobLog should should send a log event outside a run', async (t) => { name: 'R/T', level: 'info', time: getBigIntTimestamp(), - message: ['ping'], + message: JSON.stringify(['ping']), }; // The logger should print in nanoseconds (19 digits) @@ -79,7 +79,7 @@ test('jobLog should should send a log event outside a run', async (t) => { const result = { run_id: plan.id, - message: log.message, + message: JSON.parse(log.message), // Conveniently this won't have rounding errors because the last // 3 digits are always 000, because of how we generate the stamp above timestamp: log.time.substring(0, 16), @@ -109,7 +109,7 @@ test('jobLog should should send a log event inside a run', async (t) => { name: 'R/T', level: 'info', time: getBigIntTimestamp(), - message: ['ping'], + message: JSON.stringify(['ping']), }; // The logger should print in nanoseconds (19 digits) @@ -124,7 +124,7 @@ test('jobLog should should send a log event inside a run', async (t) => { const channel = mockChannel({ [RUN_LOG]: (evt) => { t.truthy(evt.step_id); - t.deepEqual(evt.message, log.message); + t.deepEqual(evt.message, JSON.parse(log.message)); t.is(evt.level, log.level); t.is(evt.source, log.name); t.is(evt.timestamp, log.time.substring(0, 16)); diff --git a/packages/ws-worker/test/mock/runtime-engine.test.ts b/packages/ws-worker/test/mock/runtime-engine.test.ts index 65ed5d19d..bfb9eba63 100644 --- a/packages/ws-worker/test/mock/runtime-engine.test.ts +++ b/packages/ws-worker/test/mock/runtime-engine.test.ts @@ -194,6 +194,39 @@ test('only listen to events for the correct workflow', async (t) => { t.pass(); }); +test('log events should stringify a string message', async (t) => { + const wf = clone(sampleWorkflow); + wf.jobs[0].expression = + 'fn((s) => {console.log("haul away joe"); return s; })'; + + engine.listen(wf.id, { + 'workflow-log': ({ message }) => { + t.is(typeof message, 'string'); + const result = JSON.parse(message); + t.deepEqual(result, ['haul away joe']); + }, + }); + + engine.execute(wf); + await waitForEvent(engine, 'workflow-complete'); +}); + +test('log events should stringify an object message', async (t) => { + const wf = clone(sampleWorkflow); + wf.jobs[0].expression = 'fn((s) => {console.log({ x: 22 }); return s; })'; + + engine.listen(wf.id, { + 'workflow-log': ({ message }) => { + t.is(typeof message, 'string'); + const result = JSON.parse(message); + t.deepEqual(result, [{ x: 22 }]); + }, + }); + + engine.execute(wf); + await waitForEvent(engine, 'workflow-complete'); +}); + test('do nothing for a job if no expression and adaptor (trigger node)', async (t) => { const workflow = { id: 'w1', diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 03bcddc98..17e5bf7f3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -383,6 +383,9 @@ importers: '@openfn/runtime': specifier: workspace:* version: link:../runtime + fast-safe-stringify: + specifier: ^2.1.1 + version: 2.1.1 devDependencies: '@types/node': specifier: ^18.15.13