Merge pull request #642 from OpenFn/compiler-release

Compiler release

integration-tests/worker/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @openfn/integration-tests-worker
 
+## 1.0.37
+
+### Patch Changes
+
+- @openfn/engine-multi@1.1.2
+- @openfn/lightning-mock@2.0.2
+- @openfn/ws-worker@1.1.2
+
 ## 1.0.36
 
 ### Patch Changes

integration-tests/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@openfn/integration-tests-worker",
   "private": true,
-  "version": "1.0.36",
+  "version": "1.0.37",
   "description": "Lightning WOrker integration tests",
   "author": "Open Function Group <[email protected]>",
   "license": "ISC",

packages/cli/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @openfn/cli
 
+## 1.1.3
+
+### Patch Changes
+
+- Updated dependencies [6dcce3d]
+- Updated dependencies [1d37ca1]
+  - @openfn/compiler@0.1.0
+
 ## 1.1.2
 
 ### Patch Changes

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "1.1.2",
+  "version": "1.1.3",
   "description": "CLI devtools for the openfn toolchain.",
   "engines": {
     "node": ">=18",

packages/compiler/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @openfn/compiler
 
+## 0.1.0
+
+### Minor Changes
+
+- 6dcce3d: Support latest ecmascript version
+- 1d37ca1: Basic support for lazy state ($) operator
+
 ## 0.0.41
 
 ### Patch Changes

packages/compiler/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/compiler",
-  "version": "0.0.41",
+  "version": "0.1.0",
   "description": "Compiler and language tooling for openfn jobs.",
   "author": "Open Function Group <[email protected]>",
   "license": "ISC",

packages/compiler/src/parse.ts

@@ -20,10 +20,9 @@ export default function parse(source: string) {
     range: true,
     parser: {
       parse: (source: string) =>
-        // TODO this can't parse nullish coalescence, so maybe we just need a more modern ecma version!
         acorn.parse(source, {
-          sourceType: 'module', // Note: this is different to v1 (but back compatible I think)
-          ecmaVersion: 10,
+          sourceType: 'module',
+          ecmaVersion: 'latest',
           allowHashBang: true,
           locations: true,
         }),

packages/compiler/src/transform.ts

@@ -5,6 +5,7 @@ import createLogger, { Logger } from '@openfn/logger';
 
 import addImports, { AddImportsOptions } from './transforms/add-imports';
 import ensureExports from './transforms/ensure-exports';
+import lazyState from './transforms/lazy-state';
 import topLevelOps, {
   TopLevelOpsOptions,
 } from './transforms/top-level-operations';
@@ -13,7 +14,8 @@ export type TransformerName =
   | 'add-imports'
   | 'ensure-exports'
   | 'top-level-operations'
-  | 'test';
+  | 'test'
+  | 'lazy-state';
 
 type TransformFunction = (
   path: NodePath<any, any>,
@@ -36,6 +38,7 @@ export type TransformOptions = {
   ['ensure-exports']?: boolean;
   ['top-level-operations']?: TopLevelOpsOptions | boolean;
   ['test']?: any;
+  ['lazy-state']?: any;
 };
 
 const defaultLogger = createLogger();
@@ -46,7 +49,7 @@ export default function transform(
   options: TransformOptions = {}
 ) {
   if (!transformers) {
-    transformers = [ensureExports, topLevelOps, addImports] as Transformer[];
+    transformers = [lazyState, ensureExports, topLevelOps, addImports] as Transformer[];
   }
   const logger = options.logger || defaultLogger;
   const transformerIndex = indexTransformers(transformers, options);

packages/compiler/src/transforms/add-imports.ts

@@ -15,6 +15,8 @@ import type { Transformer } from '../transform';
 import type { Logger } from '@openfn/logger';
 
 const globals = [
+  '\\$', // TMP hack to fix a problem with lazy-state (needs double escaping to work)
+
   'AggregateError',
   'Array',
   'ArrayBuffer',

packages/compiler/src/transforms/lazy-state.ts

@@ -0,0 +1,46 @@
+/*
+ * Convert $.a.b.c references into (state) => state.a.b.c
+ * Should this only run at top level?
+ * Ideally it would run on all arguments to operations - but we probably don't really know what an operation is
+ * So for now, first pass, it's only top level.
+ * (alternatively I guess it just dumbly converts everything and if it breaks, it breaks)
+ * 
+ * TODO (maybe):
+ *  - only convert $-expressions which are arguments to operations (needs type defs)
+ *  - warn if converting a non-top-level $-expression
+ *  - if not top level, convert to state.a.b.c (ie don't wrap the function)
+ */
+import { builders as b, namedTypes } from 'ast-types';
+import type { NodePath } from 'ast-types/lib/node-path';
+import type { Transformer } from '../transform';
+
+function visitor(path: NodePath<namedTypes.MemberExpression>) {
+  let first = path.node.object;
+  while(first.hasOwnProperty('object')) {
+    first = (first as namedTypes.MemberExpression).object;
+  }
+
+  let firstIdentifer = first as namedTypes.Identifier;
+
+  if (first && firstIdentifer.name === "$") {
+    // rename $ to state
+    firstIdentifer.name = "state";
+
+    // Now nest the whole thing in an arrow
+    const params = b.identifier('state')
+    const arrow = b.arrowFunctionExpression(
+      [params],
+      path.node
+    )
+    path.replace(arrow)
+  }
+
+  // Stop parsing this member expression
+  return;
+}
+
+export default {
+  id: 'lazy-state',
+  types: ['MemberExpression'],
+  visitor,
+} as Transformer;

packages/compiler/test/asts/simple-operation.json

@@ -1 +1 @@
-{"program":{"type":"Program","start":0,"end":5,"loc":{"start":{"line":1,"column":0,"token":0},"end":{"line":1,"column":5,"token":4},"lines":{"infos":[{"line":"fn();","indent":0,"locked":false,"sliceStart":0,"sliceEnd":5}],"mappings":[],"cachedSourceMap":null,"length":1,"name":null},"indent":0,"tokens":[{"type":"Identifier","value":"fn","loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}},{"type":"Punctuator","value":"(","loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":3}}},{"type":"Punctuator","value":")","loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":4}}},{"type":"Punctuator","value":";","loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":5}}}]},"body":[{"type":"ExpressionStatement","start":0,"end":5,"loc":{"start":{"line":1,"column":0,"token":0},"end":{"line":1,"column":5,"token":4},"lines":{"infos":[{"line":"fn();","indent":0,"locked":false,"sliceStart":0,"sliceEnd":5}],"mappings":[],"cachedSourceMap":null,"length":1,"name":null},"tokens":[{"type":"Identifier","value":"fn","loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}},{"type":"Punctuator","value":"(","loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":3}}},{"type":"Punctuator","value":")","loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":4}}},{"type":"Punctuator","value":";","loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":5}}}],"indent":0},"expression":{"type":"CallExpression","start":0,"end":4,"loc":{"start":{"line":1,"column":0,"token":0},"end":{"line":1,"column":4,"token":3},"lines":{"infos":[{"line":"fn();","indent":0,"locked":false,"sliceStart":0,"sliceEnd":5}],"mappings":[],"cachedSourceMap":null,"length":1,"name":null},"tokens":[{"type":"Identifier","value":"fn","loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}},{"type":"Punctuator","value":"(","loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":3}}},{"type":"Punctuator","value":")","loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":4}}},{"type":"Punctuator","value":";","loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":5}}}],"indent":0},"callee":{"type":"Identifier","start":0,"end":2,"loc":{"start":{"line":1,"column":0,"token":0},"end":{"line":1,"column":2,"token":1},"lines":{"infos":[{"line":"fn();","indent":0,"locked":false,"sliceStart":0,"sliceEnd":5}],"mappings":[],"cachedSourceMap":null,"length":1,"name":null},"tokens":[{"type":"Identifier","value":"fn","loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}},{"type":"Punctuator","value":"(","loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":3}}},{"type":"Punctuator","value":")","loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":4}}},{"type":"Punctuator","value":";","loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":5}}}],"indent":0},"name":"fn"},"arguments":[]}}],"sourceType":"module","errors":[]},"name":null,"loc":{"start":{"line":1,"column":0,"token":0},"end":{"line":1,"column":5,"token":4},"lines":{"infos":[{"line":"fn();","indent":0,"locked":false,"sliceStart":0,"sliceEnd":5}],"mappings":[],"cachedSourceMap":null,"length":1,"name":null},"indent":0,"tokens":[{"type":"Identifier","value":"fn","loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}},{"type":"Punctuator","value":"(","loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":3}}},{"type":"Punctuator","value":")","loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":4}}},{"type":"Punctuator","value":";","loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":5}}}]},"type":"File","comments":null,"tokens":[{"type":"Identifier","value":"fn","loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}},{"type":"Punctuator","value":"(","loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":3}}},{"type":"Punctuator","value":")","loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":4}}},{"type":"Punctuator","value":";","loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":5}}}]}
+{"program":{"type":"Program","start":0,"end":5,"loc":{"start":{"line":1,"column":0,"token":0},"end":{"line":1,"column":5,"token":4},"lines":{"infos":[{"line":"fn();","indent":0,"locked":false,"sliceStart":0,"sliceEnd":5}],"mappings":[],"cachedSourceMap":null,"length":1,"name":null},"indent":0,"tokens":[{"type":"Identifier","value":"fn","loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}},{"type":"Punctuator","value":"(","loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":3}}},{"type":"Punctuator","value":")","loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":4}}},{"type":"Punctuator","value":";","loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":5}}}]},"body":[{"type":"ExpressionStatement","start":0,"end":5,"loc":{"start":{"line":1,"column":0,"token":0},"end":{"line":1,"column":5,"token":4},"lines":{"infos":[{"line":"fn();","indent":0,"locked":false,"sliceStart":0,"sliceEnd":5}],"mappings":[],"cachedSourceMap":null,"length":1,"name":null},"tokens":[{"type":"Identifier","value":"fn","loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}},{"type":"Punctuator","value":"(","loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":3}}},{"type":"Punctuator","value":")","loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":4}}},{"type":"Punctuator","value":";","loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":5}}}],"indent":0},"expression":{"type":"CallExpression","start":0,"end":4,"loc":{"start":{"line":1,"column":0,"token":0},"end":{"line":1,"column":4,"token":3},"lines":{"infos":[{"line":"fn();","indent":0,"locked":false,"sliceStart":0,"sliceEnd":5}],"mappings":[],"cachedSourceMap":null,"length":1,"name":null},"tokens":[{"type":"Identifier","value":"fn","loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}},{"type":"Punctuator","value":"(","loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":3}}},{"type":"Punctuator","value":")","loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":4}}},{"type":"Punctuator","value":";","loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":5}}}],"indent":0},"callee":{"type":"Identifier","start":0,"end":2,"loc":{"start":{"line":1,"column":0,"token":0},"end":{"line":1,"column":2,"token":1},"lines":{"infos":[{"line":"fn();","indent":0,"locked":false,"sliceStart":0,"sliceEnd":5}],"mappings":[],"cachedSourceMap":null,"length":1,"name":null},"tokens":[{"type":"Identifier","value":"fn","loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}},{"type":"Punctuator","value":"(","loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":3}}},{"type":"Punctuator","value":")","loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":4}}},{"type":"Punctuator","value":";","loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":5}}}],"indent":0},"name":"fn"},"arguments":[],"optional":false}}],"sourceType":"module","errors":[]},"name":null,"loc":{"start":{"line":1,"column":0,"token":0},"end":{"line":1,"column":5,"token":4},"lines":{"infos":[{"line":"fn();","indent":0,"locked":false,"sliceStart":0,"sliceEnd":5}],"mappings":[],"cachedSourceMap":null,"length":1,"name":null},"indent":0,"tokens":[{"type":"Identifier","value":"fn","loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}},{"type":"Punctuator","value":"(","loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":3}}},{"type":"Punctuator","value":")","loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":4}}},{"type":"Punctuator","value":";","loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":5}}}]},"type":"File","comments":null,"tokens":[{"type":"Identifier","value":"fn","loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}},{"type":"Punctuator","value":"(","loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":3}}},{"type":"Punctuator","value":")","loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":4}}},{"type":"Punctuator","value":";","loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":5}}}]}

packages/compiler/test/compile.test.ts

@@ -114,3 +114,43 @@ test('twitter example', async (t) => {
   const result = compile(source);
   t.deepEqual(result, expected);
 });
+
+test('compile with optional chaining', (t) => {
+  const source = 'fn(a.b?.c);';
+  const expected = 'export default [fn(a.b?.c)];';
+  const result = compile(source);
+  t.assert(result === expected);
+});
+
+test('compile with nullish coalescence', (t) => {
+  const source = 'fn(a ?? b);';
+  const expected = 'export default [fn(a ?? b)];';
+  const result = compile(source);
+  t.assert(result === expected);
+});
+
+test('compile a lazy state ($) expression', (t) => {
+  const source = 'get($.data.endpoint);';
+  const expected = 'export default [get(state => state.data.endpoint)];';
+  const result = compile(source);
+  t.assert(result === expected);
+});
+
+
+test('compile a lazy state ($) expression with dumb imports', (t) => {
+  const options = {
+    'add-imports': {
+      adaptor: {
+        name: '@openfn/language-common',
+        exportAll: true,
+      },
+    },
+  };
+  const source = 'get($.data.endpoint);';
+  const expected = `import { get } from "@openfn/language-common";
+export * from "@openfn/language-common";
+export default [get(state => state.data.endpoint)];`
+
+  const result = compile(source, options);
+  t.assert(result === expected);
+});

packages/compiler/test/parse.test.ts

@@ -11,34 +11,34 @@ test('parse a simple statement', (t) => {
 
   const ast = loadAst('simple-statement');
   const result = parse(source);
-  t.assert(ast === JSON.stringify(result));
+  t.is(ast, JSON.stringify(result));
 });
 
 test('parse an esm module', (t) => {
   const source = `import foo from 'bar'; export const x = 10;`;
   const ast = loadAst('esm');
   const result = parse(source);
-  t.assert(ast === JSON.stringify(result));
+  t.is(ast, JSON.stringify(result));
 });
 
 // This will still parse as a module, but it won't freak out when it see module.exports
 test('parse a CJS script', (t) => {
   const source = `module.exports = 10;`;
   const ast = loadAst('cjs');
   const result = parse(source);
-  t.assert(ast === JSON.stringify(result));
+  t.is(ast, JSON.stringify(result));
 });
 
 test('parse a single operation', (t) => {
   const source = `fn();`;
   const ast = loadAst('simple-operation');
   const result = parse(source);
-  t.assert(ast === JSON.stringify(result));
+  t.is(ast, JSON.stringify(result));
 });
 
 test('parse multiple operations', (t) => {
   const source = `fn();fn();fn();`;
   const ast = loadAst('multiple-operations');
   const result = parse(source);
-  t.assert(ast === JSON.stringify(result));
+  t.is(ast, JSON.stringify(result));
 });

packages/compiler/test/transforms/lazy-state.test.ts

@@ -0,0 +1,76 @@
+import test, { ExecutionContext } from 'ava';
+import { print } from 'recast';
+import { namedTypes, NodePath, builders as b } from 'ast-types';
+
+import parse from '../../src/parse';
+
+import transform from '../../src/transform';
+import visitors from '../../src/transforms/lazy-state';
+
+test('convert a simple dollar reference', (t) => {
+  const ast = parse('get($.data)');
+
+  const transformed = transform(ast, [visitors]);
+  const { code } = print(transformed)
+  t.log(code)
+
+  t.is(code, 'get(state => state.data)')
+})
+
+test('convert a chained dollar reference', (t) => {
+  const ast = parse('get($.a.b.c.d)');
+
+  const transformed = transform(ast, [visitors]);
+  const { code } = print(transformed)
+  t.log(code)
+
+  t.is(code, 'get(state => state.a.b.c.d)')
+})
+
+test('ignore a regular chain reference', (t) => {
+  const ast = parse('get(a.b.c.d)');
+
+  const transformed = transform(ast, [visitors]);
+  const { code } = print(transformed)
+  t.log(code)
+
+  t.is(code, 'get(a.b.c.d)')
+})
+
+test('ignore a string', (t) => {
+  const ast = parse('get("$.a.b")');
+
+  const transformed = transform(ast, [visitors]);
+  const { code } = print(transformed)
+  t.log(code)
+
+  t.is(code, 'get("$.a.b")')
+})
+
+// TODO do we want to support this?
+test('convert a nested dollar reference', (t) => {
+  const ast = parse(`fn(() => {
+  get($.data)
+})`);
+
+  const transformed = transform(ast, [visitors]);
+  const { code } = print(transformed)
+  t.log(code)
+
+  // syntax starts getting a but picky at this level,
+  // better to do ast tests
+  t.is(code, `fn(() => {
+  get(state => state.data)
+})`)
+})
+
+// TODO does our compiler not support optional chaining??
+test.skip('convert an optional chained simple dollar reference', (t) => {
+  const ast = parse('get($.a?.b.c.d)');
+
+  // const transformed = transform(ast, [visitors]);
+  // const { code } = print(transformed)
+  // t.log(code)
+
+  // t.is(code, 'get(state => state.a?.b.c.d)')
+})

packages/engine-multi/CHANGELOG.md

@@ -1,5 +1,13 @@
 # engine-multi
 
+## 1.1.2
+
+### Patch Changes
+
+- Updated dependencies [6dcce3d]
+- Updated dependencies [1d37ca1]
+  - @openfn/compiler@0.1.0
+
 ## 1.1.1
 
 ### Patch Changes
@@ -10,6 +18,7 @@
   - @openfn/compiler@0.0.41
   - @openfn/lexicon@1.0.0
   - @openfn/runtime@1.0.1
+
 ## 1.1.0
 
 ### Minor Changes

packages/engine-multi/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/engine-multi",
-  "version": "1.1.1",
+  "version": "1.1.2",
   "description": "Multi-process runtime engine",
   "main": "dist/index.js",
   "type": "module",

packages/lightning-mock/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @openfn/lightning-mock
 
+## 2.0.2
+
+### Patch Changes
+
+- @openfn/engine-multi@1.1.2
+
 ## 2.0.1
 
 ### Patch Changes

packages/lightning-mock/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/lightning-mock",
-  "version": "2.0.1",
+  "version": "2.0.2",
   "private": true,
   "description": "A mock Lightning server",
   "main": "dist/index.js",

packages/ws-worker/CHANGELOG.md

@@ -1,5 +1,11 @@
 # ws-worker
 
+## 1.1.2
+
+### Patch Changes
+
+- @openfn/engine-multi@1.1.2
+
 ## 1.1.1
 
 ### Patch Changes