-
Notifications
You must be signed in to change notification settings - Fork 11
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* worker: restructure env vars * Fixing: #604 * added new test for statePropsToRemove * requested changes * completed: requested changes * changeset * worker: simplify typings * worker: adjust tests * release: worker @1.1.10 --------- Co-authored-by: Satyam Mattoo <[email protected]>
- Loading branch information
1 parent
015b7f2
commit 4dafcb2
Showing
7 changed files
with
268 additions
and
118 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,7 @@ | ||
| { | ||
| "name": "@openfn/integration-tests-worker", | ||
| "private": true, | ||
| "version": "1.0.44", | ||
| "version": "1.0.45", | ||
| "description": "Lightning WOrker integration tests", | ||
| "author": "Open Function Group <[email protected]>", | ||
| "license": "ISC", | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,11 @@ | ||
| # ws-worker | ||
|
|
||
| ## 1.1.10 | ||
|
|
||
| ### Patch Changes | ||
|
|
||
| - bc45b3d: Restructure handling of env vars | ||
|
|
||
| ## 1.1.9 | ||
|
|
||
| ### Patch Changes | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,155 @@ | ||
| import yargs from 'yargs'; | ||
| import { LogLevel } from '@openfn/logger'; | ||
| import { hideBin } from 'yargs/helpers'; | ||
|
|
||
| type Args = { | ||
| _: string[]; | ||
| port?: number; | ||
| lightning?: string; | ||
| repoDir?: string; | ||
| secret?: string; | ||
| loop?: boolean; | ||
| log: LogLevel; | ||
| lightningPublicKey?: string; | ||
| mock: boolean; | ||
| backoff: string; | ||
| capacity?: number; | ||
| runMemory?: number; | ||
| statePropsToRemove?: string[]; | ||
| maxRunDurationSeconds: number; | ||
| }; | ||
|
|
||
| type ArgTypes = string | string[] | number | undefined; | ||
|
|
||
| function setArg( | ||
| argValue?: ArgTypes, | ||
| envValue?: string, | ||
| defaultValue?: ArgTypes | ||
| ): ArgTypes { | ||
| if ( | ||
| Array.isArray(defaultValue) && | ||
| !argValue && | ||
| typeof envValue === 'string' | ||
| ) { | ||
| return envValue.split(','); | ||
| } | ||
|
|
||
| if (typeof defaultValue === 'number' && envValue && !argValue) { | ||
| return parseInt(envValue); | ||
| } | ||
|
|
||
| return argValue ?? envValue ?? defaultValue; | ||
| } | ||
|
|
||
| export default function parseArgs(argv: string[]): Args { | ||
| const { | ||
| WORKER_BACKOFF, | ||
| WORKER_CAPACITY, | ||
| WORKER_LIGHTNING_PUBLIC_KEY, | ||
| WORKER_LIGHTNING_SERVICE_URL, | ||
| WORKER_LOG_LEVEL, | ||
| WORKER_MAX_RUN_DURATION_SECONDS, | ||
| WORKER_MAX_RUN_MEMORY_MB, | ||
| WORKER_PORT, | ||
| WORKER_REPO_DIR, | ||
| WORKER_SECRET, | ||
| WORKER_STATE_PROPS_TO_REMOVE, | ||
| } = process.env; | ||
|
|
||
| const parser = yargs(hideBin(argv)) | ||
| .command('server', 'Start a ws-worker server') | ||
| .option('port', { | ||
| alias: 'p', | ||
| description: 'Port to run the server on. Env: WORKER_PORT', | ||
| type: 'number', | ||
| }) | ||
| .option('lightning', { | ||
| alias: ['l', 'lightning-service-url'], | ||
| description: | ||
| 'Base url to Lightning websocket endpoint, eg, ws://localhost:4000/worker. Set to "mock" to use the default mock server. Env: WORKER_LIGHTNING_SERVICE_URL', | ||
| }) | ||
| .option('repo-dir', { | ||
| alias: 'd', | ||
| description: | ||
| 'Path to the runtime repo (where modules will be installed). Env: WORKER_REPO_DIR', | ||
| }) | ||
| .option('secret', { | ||
| alias: 's', | ||
| description: | ||
| 'Worker secret. (comes from WORKER_SECRET by default). Env: WORKER_SECRET', | ||
| }) | ||
| .option('lightning-public-key', { | ||
| description: | ||
| 'Base64-encoded public key. Used to verify run tokens. Env: WORKER_LIGHTNING_PUBLIC_KEY', | ||
| }) | ||
| .option('log', { | ||
| description: | ||
| 'Set the log level for stdout (default to info, set to debug for verbose output). Env: WORKER_LOG_LEVEL', | ||
| }) | ||
| .option('loop', { | ||
| description: 'Disable the claims loop', | ||
| type: 'boolean', | ||
| default: true, | ||
| }) | ||
| .option('mock', { | ||
| description: 'Use a mock runtime engine', | ||
| type: 'boolean', | ||
| default: false, | ||
| }) | ||
| .option('backoff', { | ||
| description: | ||
| 'Claim backoff rules: min/max (in seconds). Env: WORKER_BACKOFF', | ||
| }) | ||
| .option('capacity', { | ||
| description: 'max concurrent workers. Env: WORKER_CAPACITY', | ||
| type: 'number', | ||
| }) | ||
| .option('state-props-to-remove', { | ||
| description: | ||
| 'A list of properties to remove from the final state returned by a job. Env: WORKER_STATE_PROPS_TO_REMOVE', | ||
| type: 'array', | ||
| }) | ||
| .option('run-memory', { | ||
| description: | ||
| 'Maximum memory allocated to a single run, in mb. Env: WORKER_MAX_RUN_MEMORY_MB', | ||
| type: 'number', | ||
| }) | ||
| .option('max-run-duration-seconds', { | ||
| alias: 't', | ||
| description: | ||
| 'Default run timeout for the server, in seconds. Env: WORKER_MAX_RUN_DURATION_SECONDS', | ||
| type: 'number', | ||
| }); | ||
|
|
||
| const args = parser.parse() as Args; | ||
|
|
||
| return { | ||
| ...args, | ||
| port: setArg(args.port, WORKER_PORT, 2222), | ||
| lightning: setArg( | ||
| args.lightning, | ||
| WORKER_LIGHTNING_SERVICE_URL, | ||
| 'ws://localhost:4000/worker' | ||
| ), | ||
| repoDir: setArg(args.repoDir, WORKER_REPO_DIR), | ||
| secret: setArg(args.secret, WORKER_SECRET), | ||
| lightningPublicKey: setArg( | ||
| args.lightningPublicKey, | ||
| WORKER_LIGHTNING_PUBLIC_KEY | ||
| ), | ||
| log: setArg(args.log, WORKER_LOG_LEVEL as LogLevel, 'debug'), | ||
| backoff: setArg(args.backoff, WORKER_BACKOFF, '1/10'), | ||
| capacity: setArg(args.capacity, WORKER_CAPACITY, 5), | ||
| statePropsToRemove: setArg( | ||
| args.statePropsToRemove, | ||
| WORKER_STATE_PROPS_TO_REMOVE, | ||
| ['configuration', 'response'] | ||
| ), | ||
| runMemory: setArg(args.runMemory, WORKER_MAX_RUN_MEMORY_MB, 500), | ||
| maxRunDurationSeconds: setArg( | ||
| args.maxRunDurationSeconds, | ||
| WORKER_MAX_RUN_DURATION_SECONDS, | ||
| 300 | ||
| ), | ||
| } as Args; | ||
| } |
Oops, something went wrong.