From b065553274907e7ee42acf70ed9c70207b67bdb2 Mon Sep 17 00:00:00 2001 From: yianzhang14 <78504318+yianzhang14@users.noreply.github.com> Date: Mon, 9 Sep 2024 11:51:38 -0500 Subject: [PATCH] Native globus rewrite + some more refactoring (#125) * refactored statistic * refactoring + documentation * finished refactoring on maintainers * finished connector refactoring * finished eslinting all remaining ts files * documented server.ts + added markdown files * refactored cli.ts * fixed some typing issues * revamped packages + redid express structure + more linting * more type hinting + async removal on models * linting on main src/ files * changes to type assertions * finished maintainers * make types more meaningful * finished with util rework + minor changes elsewhere * finished linting + helper rework + better nullguarding + max line size * fixed helper typo & slurmconnector doc * add specialized manifest download functionality + lint action * documentation initial commit + helper moving + abstract functions + initial lazy supervisor * finished codebase overview doc * finished codebase documentation doc * added caching to folder uploaders + a bit more linting * disable cached upload on data * added cache refresing functionality * fixed swagger error * fixed cli * readd typedoc * more package updates * added mysql * disable uml * move all documentation out of build * fixed generate swagger bug * reenable documentation * fixed gitutil bug + updated packages * recursive mkdirs * more gitutil bugfixes * silenced console.logs * reenabled urlencoded * fixed folder uploader bug * more bugfixing + removing console logs * fixed remotefsexists bug * job upload now works * update build script to copy example jsons * fixed github actions * added ruff for the python files for now * add js globus sdk * added typing to cachedupload * switched to git sdk * switch to fs promises * automated branch resolution + out of date checking * moved configs to example * undo config.json move * fixed action typo * removed all refreshing code + added automatic logic to decide when to re-upload git repo * added yaml parser * add js globus sdk * sync package lock * rework folder typing to be more enumy * leftover changes from folder rework * added cache model * force synchronize * synchronize in the json * actually force reinitialize * turned strict on + updated packages + migrated to typeorm 0.3 + migrated to redis 5 + cleaned up redis classes * reworked tsconfig * remove typeorm-uml * actually disable uml + add cache to entities * actually disable uml instead of tsdoc * fixed redis * connect to redis client * reworked cache + deprecated specialized manifest + added more git caching * reworked refreshing in getexecutablemanifest * reworked db to not be a class per typeorm guidelines * leftover db port changes * remove ormconfig * initialize database in server.ts * fixed some minor errors * initialize datasource in init hello world * reenable register * move this.register * fix db registering * fix db registering * add print statements * debugging job2object * updated dockerfile + removed print statements * fixed typing on nullable model columns * better undefined/null handling + typeorm function fixes * added more is_testing clauses to console logs * fixed registercache * more fixes to hpcpath -> cachepath * disable removing comments * styling change + git manifest cache + new git routes * rework server structure * initialize cache * change rimraf * changed promises to awaits * change unix time scaling * fix linting in cli * undo examples folder * rename a file * fixed merge bugs * fixed linting * change the eslint config * revert back to commonjs * change global variables * disable typeorm redis caching * update packages * changed file structure + updated linting * update file paths * update file paths again * fix paths and eslint * more path fixes * added axios-based globus ts impl * fix globus imports * fix entities * more typeorm fixes * fix typeorm again * change a path * add logging * add submission id retrieval --- cli.ts | 4 +- configs/config.ts | 2 +- docker/docker-entrypoint.sh | 2 +- docs/codebase.md | 20 +- eslint.config.js | 44 ++-- package-lock.json | 86 ++----- package.json | 15 +- src/connectors/BaseConnector.ts | 11 +- src/connectors/ConnectionPool.ts | 3 +- src/connectors/SingularityConnector.ts | 6 +- src/connectors/SlurmConnector.ts | 8 +- src/{lib => helpers}/FolderUtil.ts | 3 +- src/{lib => helpers}/GitUtil.ts | 7 +- src/helpers/GlobusTransferUtil.ts | 215 ++++++++++++++++++ src/{lib => helpers}/GlobusUtil.ts | 12 +- src/{lib => helpers}/Helper.ts | 2 +- src/{lib => helpers}/JobUtil.ts | 4 +- src/{lib => helpers}/PythonUtil.ts | 2 + src/{lib => helpers}/XSEDEUtil.ts | 3 +- src/{lib => helpers}/python/globus_init.py | 0 src/{lib => helpers}/python/globus_monitor.py | 0 .../python/globus_query_status.py | 0 .../python/globus_refresh_transfer_token.py | 0 .../python/globus_user_mapping.py | 0 src/{lib => helpers}/python/pyproject.toml | 0 src/maintainers/BaseMaintainer.ts | 7 +- .../CommunityContributionMaintainer.ts | 15 +- src/models/Event.ts | 1 + src/models/Job.ts | 4 +- src/models/Log.ts | 1 + FolderRoutes.ts => src/server/FolderRoutes.ts | 30 ++- GitRoutes.ts => src/server/GitRoutes.ts | 11 +- InfoRoutes.ts => src/server/InfoRoutes.ts | 13 +- JobRoutes.ts => src/server/JobRoutes.ts | 16 +- ServerUtil.ts => src/server/ServerUtil.ts | 44 +--- UserRoutes.ts => src/server/UserRoutes.ts | 20 +- server.ts => src/server/server.ts | 37 ++- src/{ => utils}/DB.ts | 14 +- src/{ => utils}/Emitter.ts | 13 +- src/{ => utils}/FolderUploader.ts | 34 +-- src/{ => utils}/JupyterHub.ts | 6 +- src/{ => utils}/Redis.ts | 10 +- src/{ => utils}/SSHCredentialGuard.ts | 6 +- src/{ => utils}/Statistic.ts | 3 +- src/{ => utils}/Supervisor.ts | 15 +- src/{ => utils}/errors.ts | 0 src/{ => utils}/types.ts | 5 +- test/TestHelper.ts | 2 +- test/service/Emitter.test.ts | 4 +- tools/generate-swagger.ts | 9 +- tools/globus-refresh-transfer-token.ts | 8 +- tsconfig.json | 11 +- 52 files changed, 499 insertions(+), 289 deletions(-) rename src/{lib => helpers}/FolderUtil.ts (98%) rename src/{lib => helpers}/GitUtil.ts (99%) create mode 100644 src/helpers/GlobusTransferUtil.ts rename src/{lib => helpers}/GlobusUtil.ts (92%) rename src/{lib => helpers}/Helper.ts (99%) rename src/{lib => helpers}/JobUtil.ts (99%) rename src/{lib => helpers}/PythonUtil.ts (99%) rename src/{lib => helpers}/XSEDEUtil.ts (97%) rename src/{lib => helpers}/python/globus_init.py (100%) rename src/{lib => helpers}/python/globus_monitor.py (100%) rename src/{lib => helpers}/python/globus_query_status.py (100%) rename src/{lib => helpers}/python/globus_refresh_transfer_token.py (100%) rename src/{lib => helpers}/python/globus_user_mapping.py (100%) rename src/{lib => helpers}/python/pyproject.toml (100%) rename FolderRoutes.ts => src/server/FolderRoutes.ts (95%) rename GitRoutes.ts => src/server/GitRoutes.ts (95%) rename InfoRoutes.ts => src/server/InfoRoutes.ts (96%) rename JobRoutes.ts => src/server/JobRoutes.ts (98%) rename ServerUtil.ts => src/server/ServerUtil.ts (81%) rename UserRoutes.ts => src/server/UserRoutes.ts (93%) rename server.ts => src/server/server.ts (73%) rename src/{ => utils}/DB.ts (69%) rename src/{ => utils}/Emitter.ts (93%) rename src/{ => utils}/FolderUploader.ts (95%) rename src/{ => utils}/JupyterHub.ts (95%) rename src/{ => utils}/Redis.ts (96%) rename src/{ => utils}/SSHCredentialGuard.ts (93%) rename src/{ => utils}/Statistic.ts (98%) rename src/{ => utils}/Supervisor.ts (97%) rename src/{ => utils}/errors.ts (100%) rename src/{ => utils}/types.ts (99%) diff --git a/cli.ts b/cli.ts index b9810ebe..98613f47 100644 --- a/cli.ts +++ b/cli.ts @@ -1,6 +1,8 @@ import { Command } from "commander"; -import dataSource from "./src/DB"; + import { Git } from "./src/models/Git"; +import dataSource from "./src/utils/DB"; + const pkg: {version: string} = require("../package.json"); // eslint-disable-line const cmd = new Command(); diff --git a/configs/config.ts b/configs/config.ts index ccb70518..b0d83810 100644 --- a/configs/config.ts +++ b/configs/config.ts @@ -5,7 +5,7 @@ import { containerConfig, jupyterGlobusMapConfig, kernelConfig, -} from "../src/types"; +} from "../src/utils/types"; // eslint-disable-next-line import/order import rawConfig from "../config.json"; // base config import rawContainerConfig from "./container.json"; // docker container config diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 2e859ca9..55079b84 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -29,4 +29,4 @@ EOT # run server redis-server /job_supervisor/docker/redis.conf --daemonize yes -node /job_supervisor/production/server.js \ No newline at end of file +npm run start \ No newline at end of file diff --git a/docs/codebase.md b/docs/codebase.md index 42282080..6ce6d314 100644 --- a/docs/codebase.md +++ b/docs/codebase.md @@ -49,7 +49,7 @@ Dependencies: - `src/DB.ts`: provides a reference to mutate the underlying database *(not used)* - `src/errors.ts`: used for a custom `ConnectorError` on a failed SSH connection - `src/Helper.ts`: general helper functions (mainly type guards) -- `src/lib/FileUtil.ts`: used for some helpful zipping/local file functions for the host server *(possibly wrong import, FileUtil does not exist)* +- `src/shared/FileUtil.ts`: used for some helpful zipping/local file functions for the host server *(possibly wrong import, FileUtil does not exist)* - `src/BaseMaintainer.ts`: used to allow the relevant logs to propagate to the maintainer of the connector's job - `src/types.ts`: various custom types - `src/connectors/connectionPool.ts`: used to get the SSH configs to open SSH connections @@ -74,7 +74,7 @@ Improvements ### Utility classes -These classes are found in the `src/lib/` directory, and these generally serve as a static helper functions abstracting away more basic operations for other parts of the codebase. The functionalities for each one are described below: +These classes are found in the `src/shared/` directory, and these generally serve as a static helper functions abstracting away more basic operations for other parts of the codebase. The functionalities for each one are described below: - `FolderUtil`: This class generally deals with files on the local server, providing functions that can zip and unzip folders/files along with functions that can detect if a file is zipped and delete folders/files. @@ -88,7 +88,7 @@ These classes are found in the `src/lib/` directory, and these generally serve a - `src/FolderUploader.ts`: to handle zipping/unzipping on local file uploads - `src/BaseConnector.ts`: for unzipping upon downloading something - - `src/lib/GitUtil.ts`: unzipping files + - `src/shared/GitUtil.ts`: unzipping files - `GitUtil`: This class handles interactions with [github](https://github.com) repositories on the hosting server. It supports things like cloning/refreshing/pulling repositores from the GitHub remote link, and, more importantly, it supports the retrieval and processing of executable manifests stored in job GitHub repositores that describe how a job should be run. - Update: new functions that just `wget` the manifests instead of fully repulling have been added as of commit `e2a4f0`. @@ -110,7 +110,7 @@ These classes are found in the `src/lib/` directory, and these generally serve a - `configs/config.ts`: needed to get redis credentials and globus ids - `src/DB.ts`: used to interface with the `globus_transfer_refresh_token` database to get the refresh token associated with an hpc's globus identity - `src/types.ts`: import custom functions needed for redis & configs/folders - - `src/lib/Helper.ts`: for null guard type assertions + - `src/shared/Helper.ts`: for null guard type assertions - `src/PythonUtil.ts`: used to actually execute the python scripts that directly interface with globus via the Python SDK - Dependents: - `server.ts`: needed for a route that gives information regarding globus and to support downloading folders from the server @@ -134,7 +134,7 @@ These classes are found in the `src/lib/` directory, and these generally serve a - `child_process`: used to run the python programs externally via forking - `configs/config.ts`: used for a testing flag controlling print statements - Dependents: - - `src/lib/GlobusUtil.ts`: uses these functions to interface with globus via python scripts + - `src/shared/GlobusUtil.ts`: uses these functions to interface with globus via python scripts - `tools/globus-refresh-transfer-token.ts`: also uses these functions to interface with globus via python scripts - `XSEDEUtil`: This class provides functionality for accessing [XSEDE](https://www.xsede.org/) commands--in particular, it is able to log jobs to XSEDE, given that the HPC in question has the credentials to do so in the config. - Dependencies: @@ -170,13 +170,13 @@ Dependencies: - `configs/config.ts`: used to access testing flags, hpc configs, and maintainer configs - `src/connectors/`: allows maintainers to actually communicate with HPCs - `src/DB.ts` & `src/models/*`: used to interface with `Job` repo for updates -- `src/lib/Helper.ts`: general utility (null guarding) +- `src/shared/Helper.ts`: general utility (null guarding) - `src/Supervisor.ts`: unused (supposed to be parent pointer) - `src/types.ts`: various custom types for configs/events/slurm - `src/FolderUploader.ts`: used to upload things to the HPC (in `CommunityContributionMaintainer`) - `src/GitUtil.ts`: used to get executable manifests for a given job (associated with a git repo; in `CommunityContributionMaintainer`) -- `src/lib/JobUtil.ts`: used to have a result folder content manager (unclear whether this is separate or the same as the one in `server.ts`) -- `src/lib/XSEDEUtil`: for job logging to XSEDE +- `src/shared/JobUtil.ts`: used to have a result folder content manager (unclear whether this is separate or the same as the one in `server.ts`) +- `src/shared/XSEDEUtil`: for job logging to XSEDE Dependents: @@ -352,7 +352,7 @@ Dependencies: - `redis`: to implement the key-value store to keep track of credentials - `util`: for `redis` function storing with `promisify` - `configs/config.ts`: to get redis credentails -- `src/lib/Helper.ts`: generating random ids for credentials (given the time) +- `src/shared/Helper.ts`: generating random ids for credentials (given the time) - `src/types.ts`: miscellaneous assorted custom types used for typing Dependents: @@ -388,7 +388,7 @@ Dependencies: - `src/connectors/ConnectionPool.ts`: used for keeping track of the number of jobs on each HPC & for ad-hoc additions of SSH connections for jobs with private credentials - `src/DB.ts` & `src/models/`: used to connect with the database to log errors with job initialization (and set the respective finishedAt time) and to register job events/logs upon job completion - `src/Emitter.ts`: used to register events/logs generated during the job maintain cycle -- `src/lib/Helper.ts`: null/assertion checking +- `src/shared/Helper.ts`: null/assertion checking - `src/maintainers/BaseMaintainer`: used to allow supervisors to create maintainers for queued jobs - `src/Queue.ts`: used to implement the job queue of a supervisor - `src/types.ts`: miscellaneous custom typings used diff --git a/eslint.config.js b/eslint.config.js index 61545082..0afa844d 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -1,32 +1,30 @@ -/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-var-requires */ +/* eslint-disable */ -const globals = require("globals"); const eslint = require("@eslint/js"); const tseslint = require("typescript-eslint"); const importPlugin = require("eslint-plugin-import"); -const { FlatCompat } = require("@eslint/eslintrc"); const stylistic = require("@stylistic/eslint-plugin"); +const globals = require("globals"); -const compat = new FlatCompat({ - baseDirectory: __dirname, -}); -module.exports = [ - ...compat.config({ - env: { - node: true - }, - parserOptions: { - ecmaVersion: "latest", - project: ["tsconfig(.*)?.json"] - } - }), - { - languageOptions: { globals: globals.browser }, - }, +module.exports = tseslint.config( eslint.configs.recommended, + ...tseslint.configs.recommendedTypeChecked, ...tseslint.configs.stylisticTypeChecked, + { + languageOptions: { + parserOptions: { + project: true, + tsconfigRootDir: __dirname + }, + globals: { + ...globals.node + } + }, + + }, + { files: ["**/*.ts", "**/*.tsx"], plugins: { @@ -66,7 +64,7 @@ module.exports = [ order: "asc", }, groups: ["external", "builtin", "parent", ["sibling", "index"]], - "newlines-between": "never", + "newlines-between": "always", pathGroups: [ { group: "external", @@ -103,4 +101,8 @@ module.exports = [ ] }, }, -]; \ No newline at end of file + + { + ignores: ["production/*"] + } +); \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 25f863d4..430f677c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,6 @@ "dependencies": { "axios": "^0.21.4", "better-sqlite3": "^7.5.1", - "body-parser": "^1.20.1", "commander": "^6.2.1", "express": "^4.18.2", "express-fileupload": "^1.2.0", @@ -25,13 +24,11 @@ "rimraf": "^5.0.5", "typeorm": "0.3.20", "util": "^0.12.3", - "validator": "^13.7.0", - "yaml": "^2.4.1" + "validator": "^13.7.0" }, "devDependencies": { "@inscreen/typeorm-uml": "^1.6.5-is1", "@stylistic/eslint-plugin": "^1.6.1", - "@types/axios": "^0.14.0", "@types/babel__core": "^7.20.5", "@types/express": "^4.17.21", "@types/express-fileupload": "^1.4.4", @@ -39,8 +36,6 @@ "@types/morgan": "^1.9.9", "@types/node": "^16.11.10", "@types/node-ssh": "^7.0.1", - "@types/redis": "^4.0.11", - "@types/rimraf": "^4.0.5", "@types/ssh2": "^0.5.46", "@types/swagger-jsdoc": "^6.0.4", "@types/swagger-ui-express": "^4.1.6", @@ -2539,16 +2534,6 @@ "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", "devOptional": true }, - "node_modules/@types/axios": { - "version": "0.14.0", - "resolved": "https://registry.npmjs.org/@types/axios/-/axios-0.14.0.tgz", - "integrity": "sha512-KqQnQbdYE54D7oa/UmYVMZKq7CO4l8DEENzOKc4aBRwxCXSlJXGz83flFx5L7AWrOQnmuN3kVsRdt+GZPPjiVQ==", - "deprecated": "This is a stub types definition for axios (https://github.com/mzabriskie/axios). axios provides its own type definitions, so you don't need @types/axios installed!", - "dev": true, - "dependencies": { - "axios": "*" - } - }, "node_modules/@types/babel__core": { "version": "7.20.5", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", @@ -2772,26 +2757,6 @@ "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", "dev": true }, - "node_modules/@types/redis": { - "version": "4.0.11", - "resolved": "https://registry.npmjs.org/@types/redis/-/redis-4.0.11.tgz", - "integrity": "sha512-bI+gth8La8Wg/QCR1+V1fhrL9+LZUSWfcqpOj2Kc80ZQ4ffbdL173vQd5wovmoV9i071FU9oP2g6etLuEwb6Rg==", - "deprecated": "This is a stub types definition. redis provides its own type definitions, so you do not need this installed.", - "dev": true, - "dependencies": { - "redis": "*" - } - }, - "node_modules/@types/rimraf": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/@types/rimraf/-/rimraf-4.0.5.tgz", - "integrity": "sha512-DTCZoIQotB2SUJnYgrEx43cQIUYOlNZz0AZPbKU4PSLYTUdML5Gox0++z4F9kQocxStrCmRNhi4x5x/UlwtKUA==", - "deprecated": "This is a stub types definition. rimraf provides its own type definitions, so you do not need this installed.", - "dev": true, - "dependencies": { - "rimraf": "*" - } - }, "node_modules/@types/semver": { "version": "7.5.7", "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.7.tgz", @@ -3761,12 +3726,12 @@ } }, "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -3934,6 +3899,15 @@ "node": ">=4" } }, + "node_modules/chalk/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/char-regex": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz", @@ -4688,15 +4662,6 @@ "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" }, - "node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true, - "engines": { - "node": ">=0.8.0" - } - }, "node_modules/escodegen": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", @@ -5299,9 +5264,9 @@ "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" }, "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, "dependencies": { "to-regex-range": "^5.0.1" @@ -11807,9 +11772,9 @@ } }, "node_modules/ws": { - "version": "7.5.9", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", - "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", + "version": "7.5.10", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.10.tgz", + "integrity": "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==", "dev": true, "engines": { "node": ">=8.3.0" @@ -11855,17 +11820,6 @@ "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", "dev": true }, - "node_modules/yaml": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.4.1.tgz", - "integrity": "sha512-pIXzoImaqmfOrL7teGUBt/T7ZDnyeGBWyXQBvOVhLkWLN37GXv8NMLK406UY6dS51JfcQHsmcW5cJ441bHg6Lg==", - "bin": { - "yaml": "bin.mjs" - }, - "engines": { - "node": ">= 14" - } - }, "node_modules/yargs": { "version": "16.2.0", "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", diff --git a/package.json b/package.json index 87c47817..fd8bf9f3 100644 --- a/package.json +++ b/package.json @@ -3,21 +3,19 @@ "version": "1.0.0", "description": "", "main": "cli.js", - "directories": { - "test": "test" - }, "scripts": { "test": "jest -c ./test/jest.config.js", "tsdoc": "typedoc --entryPointStrategy expand ./src --out ./production/tsdoc", "uml": "# typeorm-uml -d production/uml.png", "swagger": "node ./production/tools/generate-swagger.js", "doc-gen": "npm run tsdoc && npm run uml && npm run swagger", - "cpy": "cp -r ./configs ./production && cp ./*.json ./production && cp -r ./src/lib/python ./production/src/lib/python", + "cpy": "cp -r ./configs ./production && cp ./*.json ./production && cp -r ./src/helpers/python ./production/src/helpers/python", "prebuild": "rm -rf ./production", "build": "tsc", "postbuild": "npm run cpy && npm run doc-gen", "lint": "eslint .", - "typeorm": "typeorm-ts-node-commonjs" + "typeorm": "typeorm-ts-node-commonjs", + "start": "node ./production/src/server/server.js" }, "repository": { "type": "git", @@ -32,7 +30,6 @@ "dependencies": { "axios": "^0.21.4", "better-sqlite3": "^7.5.1", - "body-parser": "^1.20.1", "commander": "^6.2.1", "express": "^4.18.2", "express-fileupload": "^1.2.0", @@ -46,13 +43,11 @@ "rimraf": "^5.0.5", "typeorm": "0.3.20", "util": "^0.12.3", - "validator": "^13.7.0", - "yaml": "^2.4.1" + "validator": "^13.7.0" }, "devDependencies": { "@inscreen/typeorm-uml": "^1.6.5-is1", "@stylistic/eslint-plugin": "^1.6.1", - "@types/axios": "^0.14.0", "@types/babel__core": "^7.20.5", "@types/express": "^4.17.21", "@types/express-fileupload": "^1.4.4", @@ -60,8 +55,6 @@ "@types/morgan": "^1.9.9", "@types/node": "^16.11.10", "@types/node-ssh": "^7.0.1", - "@types/redis": "^4.0.11", - "@types/rimraf": "^4.0.5", "@types/ssh2": "^0.5.46", "@types/swagger-jsdoc": "^6.0.4", "@types/swagger-ui-express": "^4.1.6", diff --git a/src/connectors/BaseConnector.ts b/src/connectors/BaseConnector.ts index 7051c7b5..6101dad3 100644 --- a/src/connectors/BaseConnector.ts +++ b/src/connectors/BaseConnector.ts @@ -1,11 +1,14 @@ + import { existsSync, unlink, writeFileSync } from "fs"; import * as path from "path"; + import { config, hpcConfigMap } from "../../configs/config"; -import { ConnectorError } from "../errors"; -import FileUtil from "../lib/FolderUtil"; // shouldn't this be registerUtil? -import * as Helper from "../lib/Helper"; +import FileUtil from "../helpers/FolderUtil"; // shouldn't this be registerUtil? +import * as Helper from "../helpers/Helper"; import BaseMaintainer from "../maintainers/BaseMaintainer"; -import { options, hpcConfig, SSH, callableFunction } from "../types"; +import { ConnectorError } from "../utils/errors"; +import { options, hpcConfig, SSH, callableFunction } from "../utils/types"; + import connectionPool from "./ConnectionPool"; /** diff --git a/src/connectors/ConnectionPool.ts b/src/connectors/ConnectionPool.ts index cef8cfa1..2805470e 100644 --- a/src/connectors/ConnectionPool.ts +++ b/src/connectors/ConnectionPool.ts @@ -1,6 +1,7 @@ import NodeSSH = require("node-ssh"); + import { config, hpcConfigMap } from "../../configs/config"; -import { SSH, SSHConfig } from "../types"; +import { SSH, SSHConfig } from "../utils/types"; // dictionary recording ssh connections for community accounts (which have public ssh ability) const connectionPool: Record = {}; diff --git a/src/connectors/SingularityConnector.ts b/src/connectors/SingularityConnector.ts index 72856de5..8b7738d1 100644 --- a/src/connectors/SingularityConnector.ts +++ b/src/connectors/SingularityConnector.ts @@ -1,7 +1,9 @@ import * as path from "path"; + import { containerConfigMap, hpcConfigMap, kernelConfigMap } from "../../configs/config"; -import * as Helper from "../lib/Helper"; -import { slurm, executableManifest } from "../types"; +import * as Helper from "../helpers/Helper"; +import { slurm, executableManifest } from "../utils/types"; + import SlurmConnector from "./SlurmConnector"; // import { kernelConfig } from "../types"; diff --git a/src/connectors/SlurmConnector.ts b/src/connectors/SlurmConnector.ts index 96aef844..89470295 100644 --- a/src/connectors/SlurmConnector.ts +++ b/src/connectors/SlurmConnector.ts @@ -1,8 +1,10 @@ import * as path from "path"; + import { config } from "../../configs/config"; -import { ConnectorError } from "../errors"; -import * as Helper from "../lib/Helper"; -import { slurm } from "../types"; +import * as Helper from "../helpers/Helper"; +import { ConnectorError } from "../utils/errors"; +import { slurm } from "../utils/types"; + import BaseConnector from "./BaseConnector"; // import { FolderUploaderHelper } from "../FolderUploader"; diff --git a/src/lib/FolderUtil.ts b/src/helpers/FolderUtil.ts similarity index 98% rename from src/lib/FolderUtil.ts rename to src/helpers/FolderUtil.ts index 6a0a2fe6..8524abfe 100644 --- a/src/lib/FolderUtil.ts +++ b/src/helpers/FolderUtil.ts @@ -1,7 +1,8 @@ import { spawn } from "child_process"; import * as fs from "fs"; import * as path from "path"; -import { FileNotExistError } from "../errors"; + +import { FileNotExistError } from "../utils/errors"; /** * Utility class for dealing with (zipped) files. diff --git a/src/lib/GitUtil.ts b/src/helpers/GitUtil.ts similarity index 99% rename from src/lib/GitUtil.ts rename to src/helpers/GitUtil.ts index 9c627ddd..a1ee9dbd 100644 --- a/src/lib/GitUtil.ts +++ b/src/helpers/GitUtil.ts @@ -1,12 +1,14 @@ import { clone, pull, fetch, checkout, log } from "isomorphic-git"; import http from "isomorphic-git/http/node"; import { rimraf } from "rimraf"; + import * as fs from "fs"; import * as path from "path"; import { promisify } from "util"; + import { config } from "../../configs/config"; -import dataSource from "../DB"; import { Git } from "../models/Git"; +import dataSource from "../utils/DB"; import { executableManifest, integerRule, @@ -18,7 +20,8 @@ import { slurm_integer_time_unit_config, slurm_string_option_configs, stringOptionRule, -} from "../types"; +} from "../utils/types"; + import FolderUtil from "./FolderUtil"; const exec: Function = promisify(require("child_process").exec); // eslint-disable-line diff --git a/src/helpers/GlobusTransferUtil.ts b/src/helpers/GlobusTransferUtil.ts new file mode 100644 index 00000000..36e8d110 --- /dev/null +++ b/src/helpers/GlobusTransferUtil.ts @@ -0,0 +1,215 @@ +import axios, { AxiosResponse } from "axios"; + +import { config } from "../../configs/config"; +import { GlobusTransferRefreshToken } from "../models/GlobusTransferRefreshToken"; +import dataSource from "../utils/DB"; +import { GlobusFolder } from "../utils/types"; + + +const baseUrl = "https://transfer.api.globus.org/v0.10"; + +interface GlobusAuthResponse { + access_token: string; + expires_in: number; + token_type: string; + refresh_token: string; + scope: string; +} + +export class GlobusTransferUtil { + private accessToken!: string; + private time = -1; + private delay = -1; + + private async init() { + if (this.accessToken !== undefined && (new Date().getTime() - this.time) <= this.delay) { + return; + } + + const refreshTokenRepo = dataSource.getRepository(GlobusTransferRefreshToken); + const refreshToken = await refreshTokenRepo.findOneBy({ + identity: "apadmana@illinois.edu" // TODO: make this work for other identities + }); + + if (refreshToken === null) { + throw new Error("HPC config does not specify a valid refresh token identity"); + } + + const data = new URLSearchParams({ + grant_type: "refresh_token", + refresh_token: refreshToken.transferRefreshToken, + client_id: config.globus_client_id + }); + + const response: AxiosResponse = await axios.post("https://auth.globus.org/v2/oauth2/token", data, { + headers: { + "Content-Type": "application/x-www-form-urlencoded" + } + }); + + this.time = new Date().getTime(); + this.delay = response.data.expires_in * 1000; + + this.accessToken = response.data.access_token; + } + + private async getSubmissionId(): Promise { + await this.init(); + + try { + const response: AxiosResponse<{ value: string }> = await axios.get(`${baseUrl}/submission_id`, { + headers: { + "Authorization": `Bearer ${this.accessToken}` + } + }); + + if (response.status === 200) { + return response.data.value; + } + + } catch (err) { + console.error("error getting submission id for transfer submission: ", err); + } + + throw new Error("Something went wrong getting the submission id"); + } + + /** + * Initializes globus job + * + * @static + * @async + * @param {GlobusFolder} from - from transfer folder + * @param {GlobusFolder} to - to transfer folder + * @param {hpcConfig} hpcConfig - hpcConfiguration + * @param {string} [label=""] - task label + * @return {Promise} - taskId + * @throws {Error} - thrown if globus query status fails + */ + public async initTransfer( + from: GlobusFolder, + to: GlobusFolder, + label="" + ): Promise { + await this.init(); + + const data = { + DATA_TYPE: "transfer", + submission_id: await this.getSubmissionId(), + label: (label !== "" ? `${label}_${Math.floor(Math.random() * 1000)}` : undefined), + source_endpoint: from.endpoint, + destination_endpoint: to.endpoint, + DATA: [{ + DATA_TYPE: "transfer_item", + source_path: from.path, + destination_path: to.path, + recursive: true + }] + }; + + try { + const response: AxiosResponse<{ task_id: string }> = await axios.post(`${baseUrl}/transfer`, data, { + headers: { + "Content-Type": "application/json", + "Authorization": `Bearer ${this.accessToken}` + } + }); + + if (response.status !== 200 && response.status !== 201) { + console.error("Failed to submit transfer task: status code ", response.status); + } else { + return response.data.task_id; + } + } catch (err) { + console.error("Error submitting transfer task", err); + throw err; + } + + throw new Error("Something went wrong initializing globus transfer"); + } + + public async monitorTransfer(taskId: string): Promise { + await this.init(); + + let tryAgain = true; + + try { + while (true) { // eslint-disable-line no-constant-condition + const response: AxiosResponse<{ status: string }> = await axios.get(`${baseUrl}/task/${taskId}`, { + headers: { + "Authorization": `Bearer ${this.accessToken}` + } + }); + + if (response.status === 200) { + if (response.data.status === "SUCCEEDED" || response.data.status === "FAILED") { + return response.data.status; + } else { + await new Promise(r => setTimeout(r, 2000)); + } + } else if (tryAgain) { + await this.init(); + tryAgain = false; + } else { + console.error("Failed to get task, status code: ", response.status); + break; + } + } + + + } catch (err) { + console.error("Error getting transfer task status: ", err); + } + + throw new Error("Something went wrong monitoring transfer"); + } + + public async queryTransferStatus(taskId: string): Promise { + await this.init(); + + try { + const response: AxiosResponse<{ status: string }> = await axios.get(`${baseUrl}/task/${taskId}`, { + headers: { + "Authorization": `Bearer ${this.accessToken}` + } + }); + + if (response.status === 200) { + return response.data.status; + } else { + console.error("Failed to get task, status code: ", response.status); + } + } catch (err) { + console.error("Error getting transfer task status: ", err); + throw err; + } + + throw new Error("Something went wrong querying transfer status"); + } + + private escape(username: string, escapeChar="_", safe=new Set("abcdefghijklmnopqrstuvwxyz0123456789")) { + const escapedUsername: string[] = []; + + for (const char of username) { + if (safe.has(char)) { + escapedUsername.push(char); + } else { + for (const byte of Buffer.from(char, "utf8")) { + escapedUsername.push(escapeChar); + escapedUsername.push(byte.toString(16).toUpperCase()); + } + } + } + return escapedUsername.join(""); + } + + public mapUsername(initial_username: string, mapping_func: string | null) { + if (mapping_func === "iguide-mapping") { + return `iguide-claim-${this.escape(initial_username, "-").toLowerCase()}`; + } else { + return this.escape(initial_username); + } + } +} + +export const GlobusClient = new GlobusTransferUtil(); \ No newline at end of file diff --git a/src/lib/GlobusUtil.ts b/src/helpers/GlobusUtil.ts similarity index 92% rename from src/lib/GlobusUtil.ts rename to src/helpers/GlobusUtil.ts index 350d9003..45af3d25 100644 --- a/src/lib/GlobusUtil.ts +++ b/src/helpers/GlobusUtil.ts @@ -1,8 +1,9 @@ import { config } from "../../configs/config"; -import dataSource from "../DB"; import { GlobusTransferRefreshToken } from "../models/GlobusTransferRefreshToken"; -import { GlobusFolder, hpcConfig } from "../types"; +import dataSource from "../utils/DB"; +import { GlobusFolder, hpcConfig } from "../utils/types"; + import * as Helper from "./Helper"; import PythonUtil from "./PythonUtil"; @@ -33,7 +34,6 @@ export default class GlobusUtil { GlobusTransferRefreshToken ); - Helper.nullGuard(hpcConfig.globus); const g = await globusTransferRefreshTokenRepo.findOneBy({ identity: hpcConfig.globus.identity }); @@ -76,7 +76,7 @@ export default class GlobusUtil { taskId: string, hpcConfig: hpcConfig ): Promise { - return await this._queryStatus(taskId, hpcConfig, "globus_monitor.py"); + return this._queryStatus(taskId, hpcConfig, "globus_monitor.py"); } /** @@ -91,7 +91,7 @@ export default class GlobusUtil { taskId: string, hpcConfig: hpcConfig ): Promise { - return await this._queryStatus(taskId, hpcConfig, "globus_query_status.py"); + return this._queryStatus(taskId, hpcConfig, "globus_query_status.py"); } /** @@ -137,7 +137,7 @@ export default class GlobusUtil { GlobusTransferRefreshToken ); const g = await globusTransferRefreshTokenRepo.findOneBy({ - identity: hpcConfig.globus?.identity + identity: hpcConfig.globus.identity }); let out: Record; diff --git a/src/lib/Helper.ts b/src/helpers/Helper.ts similarity index 99% rename from src/lib/Helper.ts rename to src/helpers/Helper.ts index 962bcabe..2d75458e 100644 --- a/src/lib/Helper.ts +++ b/src/helpers/Helper.ts @@ -1,6 +1,6 @@ import { config, hpcConfigMap, jupyterGlobusMap } from "../../configs/config"; import { Job } from "../models/Job"; -import { callableFunction } from "../types"; +import { callableFunction } from "../utils/types"; // import * as fs from "fs"; diff --git a/src/lib/JobUtil.ts b/src/helpers/JobUtil.ts similarity index 99% rename from src/lib/JobUtil.ts rename to src/helpers/JobUtil.ts index 043279e5..838a5182 100644 --- a/src/lib/JobUtil.ts +++ b/src/helpers/JobUtil.ts @@ -1,14 +1,14 @@ import { hpcConfigMap } from "../../configs/config"; // import path = require("path"); -import dataSource from "../DB"; import { Job } from "../models/Job"; +import dataSource from "../utils/DB"; import { slurm_integer_storage_unit_config, slurm_integer_time_unit_config, slurmInputRules, slurm_integer_configs, slurm -} from "../types"; +} from "../utils/types"; /** * Class providing various useful (static) functions for handling jobs. diff --git a/src/lib/PythonUtil.ts b/src/helpers/PythonUtil.ts similarity index 99% rename from src/lib/PythonUtil.ts rename to src/helpers/PythonUtil.ts index 378e3b93..4cca4326 100644 --- a/src/lib/PythonUtil.ts +++ b/src/helpers/PythonUtil.ts @@ -1,6 +1,8 @@ import { spawn } from "child_process"; + import { config } from "../../configs/config"; + /** * Helper class for interfacing with python scripts in the ./python directory. Primarily deals with globus. */ diff --git a/src/lib/XSEDEUtil.ts b/src/helpers/XSEDEUtil.ts similarity index 97% rename from src/lib/XSEDEUtil.ts rename to src/helpers/XSEDEUtil.ts index 7c8ad3c6..350fe172 100644 --- a/src/lib/XSEDEUtil.ts +++ b/src/helpers/XSEDEUtil.ts @@ -1,7 +1,8 @@ import axios from "axios"; + import { config } from "../../configs/config"; import { Job } from "../models/Job"; -import { hpcConfig } from "../types"; +import { hpcConfig } from "../utils/types"; /** * Class for accessing XSEDE commands. May be deprecated (https://www.xsede.org/)? diff --git a/src/lib/python/globus_init.py b/src/helpers/python/globus_init.py similarity index 100% rename from src/lib/python/globus_init.py rename to src/helpers/python/globus_init.py diff --git a/src/lib/python/globus_monitor.py b/src/helpers/python/globus_monitor.py similarity index 100% rename from src/lib/python/globus_monitor.py rename to src/helpers/python/globus_monitor.py diff --git a/src/lib/python/globus_query_status.py b/src/helpers/python/globus_query_status.py similarity index 100% rename from src/lib/python/globus_query_status.py rename to src/helpers/python/globus_query_status.py diff --git a/src/lib/python/globus_refresh_transfer_token.py b/src/helpers/python/globus_refresh_transfer_token.py similarity index 100% rename from src/lib/python/globus_refresh_transfer_token.py rename to src/helpers/python/globus_refresh_transfer_token.py diff --git a/src/lib/python/globus_user_mapping.py b/src/helpers/python/globus_user_mapping.py similarity index 100% rename from src/lib/python/globus_user_mapping.py rename to src/helpers/python/globus_user_mapping.py diff --git a/src/lib/python/pyproject.toml b/src/helpers/python/pyproject.toml similarity index 100% rename from src/lib/python/pyproject.toml rename to src/helpers/python/pyproject.toml diff --git a/src/maintainers/BaseMaintainer.ts b/src/maintainers/BaseMaintainer.ts index 5f85e7b0..5078c75d 100644 --- a/src/maintainers/BaseMaintainer.ts +++ b/src/maintainers/BaseMaintainer.ts @@ -1,4 +1,5 @@ import validator from "validator"; + import { config, hpcConfigMap, @@ -7,9 +8,9 @@ import { import BaseConnector from "../connectors/BaseConnector"; import SingularityConnector from "../connectors/SingularityConnector"; import SlurmConnector from "../connectors/SlurmConnector"; -import dataSource from "../DB"; -import * as Helper from "../lib/Helper"; +import * as Helper from "../helpers/Helper"; import { Job } from "../models/Job"; +import dataSource from "../utils/DB"; // import Supervisor from "../Supervisor"; import { maintainerConfig, @@ -17,7 +18,7 @@ import { slurm, jobMaintainerUpdatable, hpcConfig, -} from "../types"; +} from "../utils/types"; /** * This is an abstract class for compute core job maintainers, which are responsible for submitting jobs and monitoring them. diff --git a/src/maintainers/CommunityContributionMaintainer.ts b/src/maintainers/CommunityContributionMaintainer.ts index 5cd8a3e6..86926958 100644 --- a/src/maintainers/CommunityContributionMaintainer.ts +++ b/src/maintainers/CommunityContributionMaintainer.ts @@ -1,13 +1,14 @@ import SingularityConnector from "../connectors/SingularityConnector"; -import dataSource from "../DB"; -import { BaseFolderUploader, FolderUploaderHelper } from "../FolderUploader"; -import GitUtil from "../lib/GitUtil"; -import * as Helper from "../lib/Helper"; -import XSEDEUtil from "../lib/XSEDEUtil"; +import GitUtil from "../helpers/GitUtil"; +import * as Helper from "../helpers/Helper"; +import XSEDEUtil from "../helpers/XSEDEUtil"; import { Folder } from "../models/Folder"; import { Git } from "../models/Git"; -import { ResultFolderContentManager } from "../Redis"; -import { executableManifest, GitFolder } from "../types"; +import dataSource from "../utils/DB"; +import { BaseFolderUploader, FolderUploaderHelper } from "../utils/FolderUploader"; +import { ResultFolderContentManager } from "../utils/Redis"; +import { executableManifest, GitFolder } from "../utils/types"; + import BaseMaintainer from "./BaseMaintainer"; /** diff --git a/src/models/Event.ts b/src/models/Event.ts index 525b973a..d6c71d26 100644 --- a/src/models/Event.ts +++ b/src/models/Event.ts @@ -7,6 +7,7 @@ import { BeforeInsert, BeforeUpdate, } from "typeorm"; + import { Job } from "./Job"; /** Class representing a job event. */ diff --git a/src/models/Job.ts b/src/models/Job.ts index 5a147c65..4b391b80 100644 --- a/src/models/Job.ts +++ b/src/models/Job.ts @@ -10,6 +10,7 @@ import { ManyToOne, // JoinColumn, } from "typeorm"; + import BaseMaintainer from "../maintainers/BaseMaintainer"; import { credential, @@ -18,7 +19,8 @@ import { LocalFolder, NeedUploadFolder, slurm, -} from "../types"; +} from "../utils/types"; + import { Event } from "./Event"; import { Folder } from "./Folder"; import { Log } from "./Log"; diff --git a/src/models/Log.ts b/src/models/Log.ts index fe0249bc..984aabfc 100644 --- a/src/models/Log.ts +++ b/src/models/Log.ts @@ -7,6 +7,7 @@ import { BeforeInsert, BeforeUpdate, } from "typeorm"; + import { Job } from "./Job"; /** Class representing a job log. */ diff --git a/FolderRoutes.ts b/src/server/FolderRoutes.ts similarity index 95% rename from FolderRoutes.ts rename to src/server/FolderRoutes.ts index 61546f2b..f1ee7c5a 100644 --- a/FolderRoutes.ts +++ b/src/server/FolderRoutes.ts @@ -1,18 +1,22 @@ import express = require("express"); + + import * as path from "path"; + import { hpcConfigMap, -} from "./configs/config"; -import { authMiddleWare, requestErrors, validator, schemas, prepareDataForDB, globusTaskList } from "./ServerUtil"; -import dataSource from "./src/DB"; -import GlobusUtil from "./src/lib/GlobusUtil"; -import * as Helper from "./src/lib/Helper"; -import { Folder } from "./src/models/Folder"; +} from "../../configs/config"; +import { GlobusClient } from "../helpers/GlobusTransferUtil"; +import * as Helper from "../helpers/Helper"; +import { Folder } from "../models/Folder"; +import dataSource from "../utils/DB"; import type { updateFolderBody, initGlobusDownloadBody, GlobusFolder -} from "./src/types"; +} from "../utils/types"; + +import { authMiddleWare, requestErrors, validator, schemas, prepareDataForDB, globusTaskList } from "./ServerUtil"; const folderRouter = express.Router(); @@ -247,8 +251,6 @@ folderRouter.post( return; } - Helper.nullGuard(hpcConfig.globus); - // init transfer const fromPath: string = (body.fromPath !== undefined ? path.join(folder.globusPath, body.fromPath) @@ -258,16 +260,13 @@ folderRouter.post( // console.log(from, to); try { - Helper.nullGuard(hpcConfig.globus); - // start the transfer - const globusTaskId = await GlobusUtil.initTransfer( + const globusTaskId = await GlobusClient.initTransfer( from, to, - hpcConfig, `job-id-${jobId}-download-folder-${folder.id}` ); - + // record the task as ongoing for the given folder await globusTaskList.put(folderId, globusTaskId); res.json({ globus_task_id: globusTaskId }); @@ -325,9 +324,8 @@ folderRouter.get( throw new Error("No task id found."); } - const status = await GlobusUtil.queryTransferStatus( + const status = await GlobusClient.queryTransferStatus( globusTaskId, - hpcConfigMap[folder.hpc] ); // remove the folder from the ongoing globus task list if the globus transfer finished diff --git a/GitRoutes.ts b/src/server/GitRoutes.ts similarity index 95% rename from GitRoutes.ts rename to src/server/GitRoutes.ts index f7ca84a6..9363a8cb 100644 --- a/GitRoutes.ts +++ b/src/server/GitRoutes.ts @@ -1,9 +1,10 @@ import express = require("express"); -import dataSource from "./src/DB"; -import GitUtil from "./src/lib/GitUtil"; -import * as Helper from "./src/lib/Helper"; -import { Git } from "./src/models/Git"; -import { executableManifest } from "./src/types"; + +import GitUtil from "../helpers/GitUtil"; +import * as Helper from "../helpers/Helper"; +import { Git } from "../models/Git"; +import dataSource from "../utils/DB"; +import { executableManifest } from "../utils/types"; const gitRouter = express.Router(); diff --git a/InfoRoutes.ts b/src/server/InfoRoutes.ts similarity index 96% rename from InfoRoutes.ts rename to src/server/InfoRoutes.ts index 65b92070..cba6b239 100644 --- a/InfoRoutes.ts +++ b/src/server/InfoRoutes.ts @@ -1,11 +1,14 @@ import express = require("express"); + import * as fs from "fs"; -import { hpcConfigMap, maintainerConfigMap, containerConfigMap, jupyterGlobusMap } from "./configs/config"; + +import { hpcConfigMap, maintainerConfigMap, containerConfigMap, jupyterGlobusMap } from "../../configs/config"; +import * as Helper from "../helpers/Helper"; +import { Job } from "../models/Job"; +import dataSource from "../utils/DB"; +import { hpcConfig, maintainerConfig, containerConfig, jupyterGlobusMapConfig, announcementsConfig } from "../utils/types"; + import { authMiddleWare, statistic } from "./ServerUtil"; -import dataSource from "./src/DB"; -import * as Helper from "./src/lib/Helper"; -import { Job } from "./src/models/Job"; -import { hpcConfig, maintainerConfig, containerConfig, jupyterGlobusMapConfig, announcementsConfig } from "./src/types"; const infoRouter = express.Router(); diff --git a/JobRoutes.ts b/src/server/JobRoutes.ts similarity index 98% rename from JobRoutes.ts rename to src/server/JobRoutes.ts index e59c7be8..09ef607e 100644 --- a/JobRoutes.ts +++ b/src/server/JobRoutes.ts @@ -1,17 +1,19 @@ import express = require("express"); + import { hpcConfigMap, maintainerConfigMap, -} from "./configs/config"; -import { authMiddleWare, requestErrors, validator, schemas, sshCredentialGuard, prepareDataForDB, supervisor, resultFolderContent } from "./ServerUtil"; -import dataSource from "./src/DB"; -import * as Helper from "./src/lib/Helper"; -import JobUtil from "./src/lib/JobUtil"; -import { Job } from "./src/models/Job"; +} from "../../configs/config"; +import * as Helper from "../helpers/Helper"; +import JobUtil from "../helpers/JobUtil"; +import { Job } from "../models/Job"; +import dataSource from "../utils/DB"; import type { createJobBody, updateJobBody, -} from "./src/types"; +} from "../utils/types"; + +import { authMiddleWare, requestErrors, validator, schemas, sshCredentialGuard, prepareDataForDB, supervisor, resultFolderContent } from "./ServerUtil"; const jobRouter = express.Router(); diff --git a/ServerUtil.ts b/src/server/ServerUtil.ts similarity index 81% rename from ServerUtil.ts rename to src/server/ServerUtil.ts index 45115d9a..c5492ecd 100644 --- a/ServerUtil.ts +++ b/src/server/ServerUtil.ts @@ -1,17 +1,17 @@ import { Request, NextFunction, Response } from "express"; import jsonschema = require("jsonschema"); -import dataSource from "./src/DB"; -import JupyterHub from "./src/JupyterHub"; -import { Folder } from "./src/models/Folder"; -import { Git } from "./src/models/Git"; -import { ResultFolderContentManager, GlobusTaskListManager } from "./src/Redis"; -import { SSHCredentialGuard } from "./src/SSHCredentialGuard"; -import Statistic from "./src/Statistic"; -import Supervisor from "./src/Supervisor"; + +import { Folder } from "../models/Folder"; +import dataSource from "../utils/DB"; +import JupyterHub from "../utils/JupyterHub"; +import { ResultFolderContentManager, GlobusTaskListManager } from "../utils/Redis"; +import { SSHCredentialGuard } from "../utils/SSHCredentialGuard"; +import Statistic from "../utils/Statistic"; +import Supervisor from "../utils/Supervisor"; import type { authReqBody, updateFolderBody, -} from "./src/types"; +} from "../utils/types"; // global object instantiation @@ -135,32 +135,6 @@ export async function prepareDataForDB( return out; } -// initializes a hello world repository in the DB -async function initHelloWorldGit() { - await dataSource.initialize(); - - const helloWorldGit = await dataSource - .getRepository(Git) - .findOneBy({ - id: "hello_world" - }); - - if (helloWorldGit === null) { - const git = { - id: "hello_world", - address: "https://github.com/cybergis/cybergis-compute-hello-world.git", - isApproved: true, - createdAt: new Date(), - updatedAt: new Date(), - }; - - await dataSource.getRepository(Git).save(git); - } -} - -// call initialization stuff -void initHelloWorldGit(); - export const authMiddleWare = async ( req: Request, res: Response, diff --git a/UserRoutes.ts b/src/server/UserRoutes.ts similarity index 93% rename from UserRoutes.ts rename to src/server/UserRoutes.ts index 441c7c3d..f4ddfcee 100644 --- a/UserRoutes.ts +++ b/src/server/UserRoutes.ts @@ -1,14 +1,18 @@ import express = require("express"); + + import * as path from "path"; + import { jupyterGlobusMap -} from "./configs/config"; +} from "../../configs/config"; +import { GlobusClient } from "../helpers/GlobusTransferUtil"; +import * as Helper from "../helpers/Helper"; +import JobUtil from "../helpers/JobUtil"; +import { Job } from "../models/Job"; +import dataSource from "../utils/DB"; + import { authMiddleWare } from "./ServerUtil"; -import dataSource from "./src/DB"; -import GlobusUtil from "./src/lib/GlobusUtil"; -import * as Helper from "./src/lib/Helper"; -import JobUtil from "./src/lib/JobUtil"; -import { Job } from "./src/models/Job"; const userRouter = express.Router(); @@ -54,7 +58,7 @@ userRouter.get("/", authMiddleWare, (req, res) => { * 404: * description: Returns an error if the user"s username is not in the allowlist */ -userRouter.get("/jupyter-globus", authMiddleWare, async (req, res) => { +userRouter.get("/jupyter-globus", authMiddleWare, (req, res) => { if (!Helper.isAllowlisted(res.locals.host as string)) { res.status(404).json({ error: "Cannot find jupyterhubHost in allowlist" }); return; @@ -72,7 +76,7 @@ userRouter.get("/jupyter-globus", authMiddleWare, async (req, res) => { try { // get a processed username (mapping changes depending on the host) - username = await GlobusUtil.mapUsername( + username = GlobusClient.mapUsername( username, jupyterGlobus.user_mapping ?? null ); diff --git a/server.ts b/src/server/server.ts similarity index 73% rename from server.ts rename to src/server/server.ts index 1bc0accd..551afb7a 100644 --- a/server.ts +++ b/src/server/server.ts @@ -1,30 +1,53 @@ - import express = require("express"); -// import { Console } from "console"; -const swaggerDocument: Record = require("../production/swagger.json"); // eslint-disable-line -// import bodyParser = require("body-parser"); import fileUpload = require("express-fileupload"); import morgan = require("morgan"); import swaggerUI = require("swagger-ui-express"); + import { config, -} from "./configs/config"; +} from "../../configs/config"; +import { Git } from "../models/Git"; +import dataSource from "../utils/DB"; + import folderRouter from "./FolderRoutes"; import gitRouter from "./GitRoutes"; import infoRouter from "./InfoRoutes"; import jobRouter from "./JobRoutes"; -import dataSource from "./src/DB"; import userRouter from "./UserRoutes"; +const swaggerDocument: Record = require("../../swagger.json"); // eslint-disable-line + // create the express app const app = express(); +// initializes a hello world repository in the DB +async function initHelloWorldGit() { + const helloWorldGit = await dataSource + .getRepository(Git) + .findOneBy({ + id: "hello_world" + }); + + if (helloWorldGit === null) { + const git = { + id: "hello_world", + address: "https://github.com/cybergis/cybergis-compute-hello-world.git", + isApproved: true, + createdAt: new Date(), + updatedAt: new Date(), + }; + + await dataSource.getRepository(Git).save(git); + } +} // establish database connection dataSource .initialize() .then(() => { console.log("Data Source has been initialized!"); + + initHelloWorldGit().catch(() => {false;}); }) .catch((err) => { console.error("Error during Data Source initialization:", err); @@ -55,7 +78,7 @@ app.use( ); // create documentation routes -app.use("/ts-docs", express.static("production/tsdoc")); +app.use("/ts-docs", express.static("../../tsdoc")); app.use("/api-docs", swaggerUI.serve, swaggerUI.setup(swaggerDocument)); /** diff --git a/src/DB.ts b/src/utils/DB.ts similarity index 69% rename from src/DB.ts rename to src/utils/DB.ts index e05dca4e..336e14c3 100644 --- a/src/DB.ts +++ b/src/utils/DB.ts @@ -2,10 +2,18 @@ import { DataSourceOptions, DataSource, } from "typeorm"; -import { config } from "../configs/config"; -import * as Helper from "./lib/Helper"; -const entities = [__dirname + "/models/**/*.js"]; +import { config } from "../../configs/config"; +import * as Helper from "../helpers/Helper"; +import { Cache } from "../models/Cache"; +import { Event } from "../models/Event"; +import { Folder } from "../models/Folder"; +import { Git } from "../models/Git"; +import { GlobusTransferRefreshToken } from "../models/GlobusTransferRefreshToken"; +import { Job } from "../models/Job"; +import { Log } from "../models/Log"; + +const entities = [Cache, Event, Folder, Git, GlobusTransferRefreshToken, Job, Log]; let dbConfig: DataSourceOptions = { name: "default", diff --git a/src/Emitter.ts b/src/utils/Emitter.ts similarity index 93% rename from src/Emitter.ts rename to src/utils/Emitter.ts index 29ad811a..c465aae4 100644 --- a/src/Emitter.ts +++ b/src/utils/Emitter.ts @@ -1,8 +1,9 @@ -import { config } from "../configs/config"; +import { config } from "../../configs/config"; +import { Event } from "../models/Event"; +import { Job } from "../models/Job"; +import { Log } from "../models/Log"; + import dataSource from "./DB"; -import { Event } from "./models/Event"; -import { Job } from "./models/Job"; -import { Log } from "./models/Log"; /** * This class abstracts away the "emission" of events/signals relating to job statuses via mutations to the database. @@ -78,7 +79,7 @@ class Emitter { * @returns {Promise{Event[]}} list of events */ async getEvents(jobId: string): Promise { - return await dataSource + return dataSource .createQueryBuilder(Event, "event") .where("event.jobId = :jobId", { jobId: jobId }) .orderBy("event.createdAt", "DESC") @@ -92,7 +93,7 @@ class Emitter { * @return {Promise} list of logs */ async getLogs(jobId: string): Promise { - return await dataSource + return dataSource .createQueryBuilder(Log, "log") .where("log.jobId = :jobId", { jobId: jobId }) .orderBy("log.createdAt", "DESC") diff --git a/src/FolderUploader.ts b/src/utils/FolderUploader.ts similarity index 95% rename from src/FolderUploader.ts rename to src/utils/FolderUploader.ts index f189d19f..5dd11903 100644 --- a/src/FolderUploader.ts +++ b/src/utils/FolderUploader.ts @@ -1,17 +1,21 @@ + + import * as fs from "fs"; import * as path from "path"; -import { hpcConfigMap } from "../configs/config"; -import BaseConnector from "./connectors/BaseConnector"; -import SingularityConnector from "./connectors/SingularityConnector"; -import SlurmConnector from "./connectors/SlurmConnector"; + +import { hpcConfigMap } from "../../configs/config"; +import BaseConnector from "../connectors/BaseConnector"; +import SingularityConnector from "../connectors/SingularityConnector"; +import SlurmConnector from "../connectors/SlurmConnector"; +import FolderUtil from "../helpers/FolderUtil"; +import GitUtil from "../helpers/GitUtil"; +import { GlobusClient } from "../helpers/GlobusTransferUtil"; +import * as Helper from "../helpers/Helper"; +import { Cache } from "../models/Cache"; +import { Folder } from "../models/Folder"; + import dataSource from "./DB"; import { NotImplementedError } from "./errors"; -import FolderUtil from "./lib/FolderUtil"; -import GitUtil from "./lib/GitUtil"; -import GlobusUtil from "./lib/GlobusUtil"; -import * as Helper from "./lib/Helper"; -import { Cache } from "./models/Cache"; -import { Folder } from "./models/Folder"; import { BaseFolder, GitFolder, @@ -317,8 +321,6 @@ class GlobusFolderUploader extends CachedFolderUploader { // eslint-disable-lin if (!this.hpcConfig) throw new Error(`cannot find hpcConfig with name ${hpcName}`); - if (!this.hpcConfig.globus) - throw new Error(`cannot find hpcConfig.globus with name ${hpcName}`); this.from = from; this.to = { @@ -337,17 +339,17 @@ class GlobusFolderUploader extends CachedFolderUploader { // eslint-disable-lin * @param {GlobusFolder} folder */ protected async uploadToFolder(folder: GlobusFolder) { - this.taskId = await GlobusUtil.initTransfer( + const taskId = await GlobusClient.initTransfer( this.from, folder, - this.hpcConfig, "job-id-" + this.jobId + "-upload-folder-" + this.id ); + this.taskId = taskId; + // get status of transfer - const status = await GlobusUtil.monitorTransfer( + const status = await GlobusClient.monitorTransfer( this.taskId, - this.hpcConfig ); if (status.includes("FAILED")) { diff --git a/src/JupyterHub.ts b/src/utils/JupyterHub.ts similarity index 95% rename from src/JupyterHub.ts rename to src/utils/JupyterHub.ts index f25f0ff0..c40b9c1a 100644 --- a/src/JupyterHub.ts +++ b/src/utils/JupyterHub.ts @@ -1,7 +1,9 @@ import axios from "axios"; + import * as path from "path"; -import { jupyterGlobusMap } from "../configs/config"; -import * as Helper from "./lib/Helper"; + +import { jupyterGlobusMap } from "../../configs/config"; +import * as Helper from "../helpers/Helper"; declare interface decodedToken { host: string; diff --git a/src/Redis.ts b/src/utils/Redis.ts similarity index 96% rename from src/Redis.ts rename to src/utils/Redis.ts index 1b5cdc9d..ab820103 100644 --- a/src/Redis.ts +++ b/src/utils/Redis.ts @@ -1,7 +1,9 @@ import { RedisClientType, createClient } from "redis"; -import { config } from "../configs/config"; + +import { config } from "../../configs/config"; +import { Job } from "../models/Job"; + import dataSource from "./DB"; -import { Job } from "./models/Job"; import { credential } from "./types"; class RedisStore { @@ -55,7 +57,7 @@ export class GlobusTaskListManager extends RedisStore { * @return {Promise} out - redis output */ public async get(label: string): Promise { - return await this.client.GET(`globus_task_${label}`); + return this.client.GET(`globus_task_${label}`); } /** @@ -190,7 +192,7 @@ export class JobQueue extends RedisStore { * @returns {number} length */ async length(): Promise { - return await this.client.LLEN(this.name); + return this.client.LLEN(this.name); } /** diff --git a/src/SSHCredentialGuard.ts b/src/utils/SSHCredentialGuard.ts similarity index 93% rename from src/SSHCredentialGuard.ts rename to src/utils/SSHCredentialGuard.ts index dca1f68f..885149a9 100644 --- a/src/SSHCredentialGuard.ts +++ b/src/utils/SSHCredentialGuard.ts @@ -1,6 +1,8 @@ import NodeSSH = require("node-ssh"); -import { hpcConfigMap } from "../configs/config"; -import * as Helper from "./lib/Helper"; + +import { hpcConfigMap } from "../../configs/config"; +import * as Helper from "../helpers/Helper"; + import { CredentialManager } from "./Redis"; class SSHCredentialGuard { diff --git a/src/Statistic.ts b/src/utils/Statistic.ts similarity index 98% rename from src/Statistic.ts rename to src/utils/Statistic.ts index 2808313a..48266147 100644 --- a/src/Statistic.ts +++ b/src/utils/Statistic.ts @@ -1,5 +1,6 @@ +import { Job } from "../models/Job"; + import dataSource from "./DB"; -import { Job } from "./models/Job"; /** * Wrapper class for requesting statistics from the database. diff --git a/src/Supervisor.ts b/src/utils/Supervisor.ts similarity index 97% rename from src/Supervisor.ts rename to src/utils/Supervisor.ts index df2fd065..2c269b24 100644 --- a/src/Supervisor.ts +++ b/src/utils/Supervisor.ts @@ -1,12 +1,15 @@ import NodeSSH = require("node-ssh"); + import * as events from "events"; -import { config, maintainerConfigMap, hpcConfigMap } from "../configs/config"; -import connectionPool from "./connectors/ConnectionPool"; + +import { config, maintainerConfigMap, hpcConfigMap } from "../../configs/config"; +import connectionPool from "../connectors/ConnectionPool"; +import * as Helper from "../helpers/Helper"; +import BaseMaintainer from "../maintainers/BaseMaintainer"; +import { Job } from "../models/Job"; + import dataSource from "./DB"; import Emitter from "./Emitter"; -import * as Helper from "./lib/Helper"; -import BaseMaintainer from "./maintainers/BaseMaintainer"; -import { Job } from "./models/Job"; import { JobQueue } from "./Redis"; import { SSH, callableFunction } from "./types"; @@ -71,7 +74,7 @@ class Supervisor { if (!job) continue; // eslint-disable-next-line - const maintainer: new(job: Job) => BaseMaintainer = require(`./maintainers/${ + const maintainer: new(job: Job) => BaseMaintainer = require(`../maintainers/${ maintainerConfigMap[job.maintainer].maintainer }`).default; // eslint-disable-line // ^ typescript compilation hack diff --git a/src/errors.ts b/src/utils/errors.ts similarity index 100% rename from src/errors.ts rename to src/utils/errors.ts diff --git a/src/types.ts b/src/utils/types.ts similarity index 99% rename from src/types.ts rename to src/utils/types.ts index a5f77cc3..a1d8a7ad 100644 --- a/src/types.ts +++ b/src/utils/types.ts @@ -2,7 +2,8 @@ import NodeSSH = require("node-ssh"); import { ConnectConfig } from "ssh2"; import { Prompt } from "ssh2-streams"; -import { Folder } from "./models/Folder"; + +import { Folder } from "../models/Folder"; type unit = "GB" | "MB" | "Minutes" | "Hours" | "Days" | "None"; @@ -198,7 +199,7 @@ export interface hpcConfig { init_sbatch_script: string[]; init_sbatch_options: string[]; description?: string; - globus?: { + globus: { identity: string; endpoint: string; root_path: string; diff --git a/test/TestHelper.ts b/test/TestHelper.ts index 3e457303..d07deecf 100644 --- a/test/TestHelper.ts +++ b/test/TestHelper.ts @@ -1,7 +1,7 @@ -import dataSource from "../src/DB"; import { Event } from "../src/models/Event"; import { Job } from "../src/models/Job"; import { Log } from "../src/models/Log"; +import dataSource from "../src/utils/DB"; export default class TestHelper { static async createJob( diff --git a/test/service/Emitter.test.ts b/test/service/Emitter.test.ts index 5e1543dd..a0baf5a8 100644 --- a/test/service/Emitter.test.ts +++ b/test/service/Emitter.test.ts @@ -1,7 +1,7 @@ import "jest"; import { config } from "../../configs/config"; -import { clearAll } from "../../src/DB"; -import Emitter from "../../src/Emitter"; +import { clearAll } from "../../src/utils/DB"; +import Emitter from "../../src/utils/Emitter"; import TestHelper from "../TestHelper"; beforeAll(() => { diff --git a/tools/generate-swagger.ts b/tools/generate-swagger.ts index 2886fda7..72607c6d 100644 --- a/tools/generate-swagger.ts +++ b/tools/generate-swagger.ts @@ -1,5 +1,6 @@ -import swaggerJsdoc = require("swagger-jsdoc"); -import fs = require("fs"); +import swaggerJSDoc from "swagger-jsdoc"; + +import * as fs from "fs"; const options = { definition: { @@ -9,10 +10,10 @@ const options = { version: "1.0.0", }, }, - apis: ["./server.ts"], // files containing annotations as above + apis: ["./src/server/*"], // files containing annotations as above }; -const output: object = swaggerJsdoc(options); +const output: object = swaggerJSDoc(options); fs.writeFile( "./production/swagger.json", JSON.stringify(output), diff --git a/tools/globus-refresh-transfer-token.ts b/tools/globus-refresh-transfer-token.ts index 14e45317..f6517854 100644 --- a/tools/globus-refresh-transfer-token.ts +++ b/tools/globus-refresh-transfer-token.ts @@ -1,14 +1,14 @@ import { config, hpcConfigMap } from "../configs/config"; -import dataSource from "../src/DB"; -import PythonUtil from "../src/lib/PythonUtil"; +import PythonUtil from "../src/helpers/PythonUtil"; import { GlobusTransferRefreshToken } from "../src/models/GlobusTransferRefreshToken"; +import dataSource from "../src/utils/DB"; const main = async () => { const identities: string[] = []; for (const i in hpcConfigMap) { if (hpcConfigMap[i].globus) { - if (!(hpcConfigMap[i].globus!.identity in identities)) { - identities.push(hpcConfigMap[i].globus!.identity); + if (!(hpcConfigMap[i].globus.identity in identities)) { + identities.push(hpcConfigMap[i].globus.identity); } } } diff --git a/tsconfig.json b/tsconfig.json index 0e6451d5..a0b59302 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { - "target": "ESNext", - "module": "commonjs", + "target": "ES2022", + "module": "CommonJS", "declaration": false, "strict": true, "noImplicitAny": true, @@ -13,16 +13,11 @@ "resolveJsonModule" : true, "strictNullChecks": true, "esModuleInterop": true, - "lib": [ - "es5", - "es6" - ], "moduleResolution": "node", - "sourceMap": true + "sourceMap": true, }, "include": [ "**/*", - "src/maintainers/*", ".eslintrc.js" ], "exclude": [