Merge pull request #41 from Serverless-Devs/zxy/fix-workflow

feat: verify when exec custom commands

packages/downloads/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@serverless-devs/downloads",
-  "version": "0.0.5",
+  "version": "0.0.6",
   "description": "download for serverless-devs",
   "main": "lib/index.js",
   "scripts": {

packages/downloads/src/index.ts

@@ -79,6 +79,22 @@ class Download {
         // ignore error in windows
       }
     }
+    // support github zip
+    const items = fs.readdirSync(dest as string, { withFileTypes: true });
+    const directories = items.filter(item => item.isDirectory());
+    // only one directory, move
+    if (directories.length === 1 && items.length === 1) {
+      try {
+        const directoryName = directories[0].name;
+        const directoryPath = path.join(dest as string, directoryName);
+        const tmpFilePath = path.join(dest as string, `../${filePath}-${Date.now()}`);
+        fs.moveSync(directoryPath, tmpFilePath, { overwrite: true });
+        fs.moveSync(tmpFilePath, dest as string, { overwrite: true });
+        fs.removeSync(tmpFilePath);
+      } catch(e) {
+        throw e;
+      }
+    }
   }
   private async doDownload(url: string): Promise<string> {
     const { headers, logger } = this.options;

packages/engine/__tests__/index.test.ts

@@ -3,6 +3,7 @@ import Engine from '../src';
 import path from 'path';
 import { AssertionError } from 'assert';
 import { get } from 'lodash';
+import { DevsError } from '@serverless-devs/utils';
 
 test('指定 template 不存在', async () => {
   const engine = new Engine({
@@ -330,7 +331,7 @@ test('validate projectName', async () => {
   });
   const context = await engine.start();
   console.log(context);
-  expect(get(context, 'error[0]')).toBeInstanceOf(AssertionError);
+  expect(get(context, 'error[0]')).toBeInstanceOf(DevsError);
   expect(get(context, 'error[0].message')).toBe(`The name of the project [deploy] overlaps with a command, please change it's name`);
   expect(get(context, 'error[0].code')).toBe('ERR_ASSERTION');
 });

packages/engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@serverless-devs/engine",
-  "version": "0.1.2-beta.4",
+  "version": "0.1.2-beta.9",
   "description": "a engine lib for serverless-devs",
   "main": "lib/index.js",
   "scripts": {

packages/engine/src/actions/index.ts

@@ -18,7 +18,7 @@ interface IRecord {
   magic: Record<string, any>; // 记录魔法变量
   componentProps: Record<string, any>; // 记录组件的inputs
   pluginOutput: Record<string, any>; // 记录plugin的outputs
-  lable: string; // 记录执行的label
+  label: string; // 记录执行的label
   step: IStepOptions; // 记录当前step
   allowFailure: boolean | IAllowFailure; // step allow_failure > action allow_failure
   command: string; // 记录当前执行的command
@@ -106,8 +106,8 @@ You can still use them now, but we suggest to modify them.`)
     const hooks = filter(this.actions, item => item.hookType === hookType);
     if (isEmpty(hooks)) return {};
     this.record.startTime = Date.now();
-    this.record.lable = this.option.hookLevel === IActionLevel.PROJECT ? `[${this.option.projectName}]` : IActionLevel.GLOBAL;
-    this.logger.debug(`Start executing the ${hookType}-action in ${this.record.lable}`);
+    this.record.label = this.option.hookLevel === IActionLevel.PROJECT ? `[${this.option.projectName}]` : IActionLevel.GLOBAL;
+    this.logger.debug(`Start executing the ${hookType}-action in ${this.record.label}`);
     // 确保 hooks 中的变量均为解析过后的真实值
     const newHooks = getInputs(hooks, this.record.magic);  
     // post-action应获取componentProps, 先清空pluginOutput
@@ -128,7 +128,7 @@ You can still use them now, but we suggest to modify them.`)
         await this.component(hook);
       }
     }
-    this.logger.debug(`The ${hookType}-action successfully to execute in ${this.record.lable}`);
+    this.logger.debug(`The ${hookType}-action successfully to execute in ${this.record.label}`);
 
     if (this.option.hookLevel === IActionLevel.GLOBAL) {
       this.logger.write(`${chalk.green('✔')} ${chalk.gray(`${IActionLevel.GLOBAL} ${hookType}-action completed (${getProcessTime(this.record.startTime)})`)}`);
@@ -207,7 +207,7 @@ You can still use them now, but we suggest to modify them.`)
           data: get(e, 'data'),
           stack: error.stack,
           exitCode: EXIT_CODE.RUN,
-          prefix: `${this.record.lable} ${hook.hookType}-action failed to [${this.record.command}]:`,
+          prefix: `${this.record.label} ${hook.hookType}-action failed to [${this.record.command}]:`,
           trackerType: ETrackerType.runtimeException,
         });
       }
@@ -221,7 +221,7 @@ You can still use them now, but we suggest to modify them.`)
     if (useAllowFailure) return;
     throw new DevsError(`The ${hook.path} directory does not exist.`, {
       exitCode: EXIT_CODE.DEVS,
-      prefix: `${this.record.lable} ${hook.hookType}-action failed to [${this.record.command}]:`,
+      prefix: `${this.record.label} ${hook.hookType}-action failed to [${this.record.command}]:`,
       trackerType: ETrackerType.parseException,
     });
   }
@@ -257,7 +257,7 @@ You can still use them now, but we suggest to modify them.`)
         data: get(e, 'data'),
         stack: error.stack,
         exitCode: EXIT_CODE.PLUGIN,
-        prefix: `${this.record.lable} ${hook.hookType}-action failed to [${this.record.command}]:`,
+        prefix: `${this.record.label} ${hook.hookType}-action failed to [${this.record.command}]:`,
         trackerType: ETrackerType.runtimeException,
       });
     }
@@ -291,11 +291,12 @@ You can still use them now, but we suggest to modify them.`)
       // 方法存在，执行报错，退出码101
       const newInputs = {
         ...this.record.componentProps,
-        argv: filter(argv.slice(2), o => !includes([componentName, command], o)),
+        args: filter(argv.slice(2), o => !includes([componentName, command], o)),
       };
       try {
         // Execute the command for the component with the prepared inputs.
-        return await instance[command](newInputs);
+        await instance[command](newInputs);
+        return;
       } catch (e) {
         const error = e as Error;
         // Check if the failure is allowed based on the record's allowFailure setting.
@@ -308,7 +309,7 @@ You can still use them now, but we suggest to modify them.`)
           data: get(e, 'data'),
           stack: error.stack,
           exitCode: EXIT_CODE.COMPONENT,
-          prefix: `${this.record.lable} ${hook.hookType}-action failed to [${this.record.command}]:`,
+          prefix: `${this.record.label} ${hook.hookType}-action failed to [${this.record.command}]:`,
           trackerType: ETrackerType.runtimeException,
         });
       }
@@ -323,9 +324,9 @@ You can still use them now, but we suggest to modify them.`)
     // 方法不存在，此时系统将会认为是未找到组件方法，系统的exit code为100；
     throw new DevsError(`The [${command}] command was not found.`, {
       exitCode: EXIT_CODE.DEVS,
-      prefix: `${this.record.lable} ${hook.hookType}-action failed to [${this.record.command}]:`,
+      prefix: `${this.record.label} ${hook.hookType}-action failed to [${this.record.command}]:`,
       tips: `Please check the component ${componentName} has the ${command} command. Serverless Devs documents：${chalk.underline(
-        'https://github.com/Serverless-Devs/Serverless-Devs/blob/master/docs/zh/command',
+        'https://manual.serverless-devs.com/',
       )}`,
       trackerType: ETrackerType.parseException,
     });

packages/engine/src/index.ts

@@ -12,6 +12,7 @@ import Logger, { ILoggerInstance } from '@serverless-devs/logger';
 import { DevsError, ETrackerType, emoji, getAbsolutePath, getRootHome, getUserAgent, traceid } from '@serverless-devs/utils';
 import { EXIT_CODE } from './constants';
 import assert from 'assert';
+import Ajv from 'ajv';
 export * from './types';
 export { verify, preview } from './utils';
 
@@ -91,7 +92,7 @@ class Engine {
       return this.context;
     }
     const { steps: _steps, yaml, command, access = yaml.access } = this.spec;
-    this.logger.write(`${emoji('⌛')} Steps for [${command}] of [${get(this.spec, 'yaml.appName')}]\n${chalk.gray('====================')}`);
+    this.logger.write(chalk.gray(`${emoji('⌛')} Steps for [${command}] of [${get(this.spec, 'yaml.appName')}]\n${chalk.gray('====================')}`));
     // 初始化全局的 action
     this.globalActionInstance = new Actions(yaml.actions, {
       hookLevel: IActionLevel.GLOBAL,
@@ -206,14 +207,54 @@ class Engine {
    */
   private async validate() {
     const { steps, command, projectName } = this.spec;
+    let errorsList: any[] = [];
+    const ajv = new Ajv({ allErrors: true });
+    assert(!isEmpty(steps), 'Step is required');
     for (const step of steps) {
-      const instance = await loadComponent(step.component, { engineLogger: this.logger });
+      const instance = await loadComponent(step.component, { logger: this.logger, engineLogger: this.logger });
       if (projectName && keys(get(instance, 'commands')).includes(projectName)) {
-        assert(!projectName, `The name of the project [${projectName}] overlaps with a command, please change it's name`);
+        throw new DevsError(`The name of the project [${projectName}] overlaps with a command, please change it's name.`, {
+          exitCode: EXIT_CODE.DEVS,
+          trackerType: ETrackerType.parseException,
+          prefix: `[${projectName}] failed to [${command}]:`
+        });
+      }
+      // schema validation
+      if (get(this.spec, 'yaml.use3x') && get(this.spec, 'yaml.content.validation')) {
+        const schema = await this.getSchemaByInstance(instance);
+        if (isEmpty(schema)) continue;
+        const validate = ajv.compile(JSON.parse(schema));
+        if (!validate(step.props)) {
+          const errors = validate.errors;
+          if (!errors) continue;
+          for (const j of errors) {
+            j.instancePath = step.projectName + '/props' + j.instancePath;
+            if (j.keyword === 'enum') {
+              j.message = j.message + ': ' + j.params.allowedValues.join(', ');
+            }
+          }
+          errorsList = errorsList.concat(errors);
+        }
       }
     }
-    assert(!isEmpty(steps), 'Step is required');
     assert(command, 'Command is required');
+    if (!isEmpty(errorsList)) {
+      throw new DevsError(`${ajv.errorsText(errorsList, { dataVar: '', separator: '\n' })}`, {
+        exitCode: EXIT_CODE.DEVS,
+        trackerType: ETrackerType.parseException,
+        prefix: 'Function props validation error:'
+      });
+    }
+  }
+
+  /**
+   * Get schema by existing instance, avoid loading components.
+   * @param instance loadComponent instance
+   * @param logger Logger
+   */
+  private getSchemaByInstance(instance: any) {
+    if (!instance || !instance.getSchema) return null;
+    return instance.getSchema();
   }
 
   /**
@@ -298,7 +339,7 @@ class Engine {
       cwd: path.dirname(this.spec.yaml.path),
       vars: this.spec.yaml.vars,
       resources: {},
-      __runtime: this.options.verify ? 'enigne' : 'parse',
+      __runtime: this.options.verify ? 'engine' : 'parse',
       __steps: this.context.steps,
     } as Record<string, any>;
     for (const obj of this.context.steps) {
@@ -599,7 +640,7 @@ class Engine {
       throw new DevsError(`The [${command}] command was not found.`, {
         exitCode: EXIT_CODE.DEVS,
         tips: `Please check the component ${item.component} has the ${command} command. Serverless Devs documents：${chalk.underline(
-          'https://github.com/Serverless-Devs/Serverless-Devs/blob/master/docs/zh/command',
+          'https://manual.serverless-devs.com/',
         )}`,
         prefix: `[${item.projectName}] failed to [${command}]:`,
         trackerType: ETrackerType.parseException,
@@ -629,7 +670,7 @@ class Engine {
     // 方法不存在，进行警告，但是并不会报错，最终的exit code为0；
     this.logger.tips(
       `The [${command}] command was not found.`,
-      `Please check the component ${item.component} has the ${command} command. Serverless Devs documents：https://github.com/Serverless-Devs/Serverless-Devs/blob/master/docs/zh/command`,
+      `Please check the component ${item.component} has the ${command} command. Serverless Devs documents：https://manual.serverless-devs.com/`,
     );
   }
 

packages/load-application/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@serverless-devs/load-application",
-  "version": "0.0.13-beta.1",
+  "version": "0.0.13-beta.6",
   "description": "load application for serverless-devs",
   "main": "lib/index.js",
   "scripts": {

packages/load-application/src/constant.ts

@@ -1,12 +1,15 @@
 import chalk from 'chalk';
+import { getGlobalConfig } from '@serverless-devs/utils';
 
 export const gray = chalk.hex('#8c8d91');
 export const RANDOM_PATTERN = '${default-suffix}';
 export const DEVSAPP = 'devsapp';
+export const GITHUB_REGISTRY = 'https://api.github.com/repos';
 
 export const REGISTRY = {
   V2: 'https://registry.devsapp.cn/simple',
   V3: 'https://api.devsapp.cn/v3',
+  CUSTOM_URL: getGlobalConfig('registry'),
 };
 
 export const CONFIGURE_LATER = 'configure_later';

packages/load-application/src/index.ts

@@ -2,21 +2,24 @@ import V3 from './v3';
 import V2 from './v2';
 import assert from 'assert';
 import { IOptions } from './types';
-import { includes } from 'lodash';
+import { includes, get } from 'lodash';
+import { REGISTRY } from './constant';
 const debug = require('@serverless-cd/debug')('serverless-devs:load-appliaction');
 
 export default async (template: string, options: IOptions = {}) => {
   debug(`load application, template: ${template}, options: ${JSON.stringify(options)}`);
+  const { logger } = options;
   if (options.uri) {
     return await v3(template, options);
   }
   assert(template, 'template is required');
-  if (includes(template, '/')) {
+  if ((includes(template, '/') && REGISTRY.CUSTOM_URL === REGISTRY.V3) || (REGISTRY.CUSTOM_URL === REGISTRY.V2)) {
     return await v2(template, options);
   }
   try {
     return await v3(template, options);
   } catch (error) {
+    logger.warn(get(error, 'message') + ', try to load from v2 registry.');
     debug(`v3 error, ${error}`);
     return await v2(template, options);
   }

packages/load-application/src/utils/index.ts

@@ -1,5 +1,5 @@
-import { endsWith, keys, replace } from 'lodash';
-import { RANDOM_PATTERN, REGISTRY } from '../constant';
+import { keys, replace, split } from 'lodash';
+import { RANDOM_PATTERN, REGISTRY, GITHUB_REGISTRY } from '../constant';
 import Credential from '@serverless-devs/credential';
 
 export { default as getInputs } from './get-inputs';
@@ -10,7 +10,15 @@ export const tryfun = async (fn: Function, ...args: any[]) => {
   } catch (ex) {}
 };
 
-export const getUrlWithLatest = (name: string) => `${REGISTRY.V3}/packages/${name}/release/latest`;
+export const getUrlWithLatest = (name: string) => {
+  if (REGISTRY.CUSTOM_URL === GITHUB_REGISTRY) {
+    if (split(name, '/').length === 1) {
+      return `${REGISTRY.CUSTOM_URL}/devsapp/${name}`;
+    }
+    return `${REGISTRY.CUSTOM_URL}/${name}`;
+  }
+  return `${REGISTRY.V3}/packages/${name}/release/latest`
+};
 export const getUrlWithVersion = (name: string, versionId: string) => `${REGISTRY.V3}/packages/${name}/release/tags/${versionId}`;
 
 export const randomId = () => Math.random().toString(36).substring(2, 6);
@@ -22,5 +30,5 @@ export const getAllCredential = async ({ logger }: any) => {
 
 export const getDefaultValue = (value: any) => {
   if (typeof value !== 'string') return;
-  return endsWith(value, RANDOM_PATTERN) ? replace(value, RANDOM_PATTERN, randomId()) : value;
+  return replace(value, RANDOM_PATTERN, randomId());
 };

packages/load-application/src/v2.ts

@@ -4,7 +4,7 @@ import axios from 'axios';
 import download from '@serverless-devs/downloads';
 import artTemplate from 'art-template';
 import { getYamlContent, isCiCdEnvironment, getYamlPath } from '@serverless-devs/utils';
-import { isEmpty, includes, split, get, has, set, sortBy, map, concat, keys, find } from 'lodash';
+import { isEmpty, includes, split, get, has, set, sortBy, map, concat, keys, find, startsWith } from 'lodash';
 import parse from './parse';
 import { IProvider, IOptions } from './types';
 import { CONFIGURE_LATER, DEFAULT_MAGIC_ACCESS, REGISTRY } from './constant';
@@ -374,13 +374,31 @@ class LoadApplication {
   private async doLoad() {
     const { logger } = this.options;
     const zipball_url = this.version ? await this.doZipballUrlWithVersion() : await this.doZipballUrl();
-    await download(zipball_url, {
-      dest: this.tempPath,
-      logger,
-      extract: true,
-      strip: 1,
-      filename: this.name,
-    });
+    try {
+      await download(zipball_url, {
+        dest: this.tempPath,
+        logger,
+        extract: true,
+        strip: 1,
+        filename: this.name,
+      });
+    } catch(e) {
+      logger.debug(e);
+      // if https, try http
+      if (startsWith(zipball_url, 'https')) {
+        logger.debug('https error, try http');
+        const newZipballUrl = zipball_url.replace('https://', 'http://');
+        await download(newZipballUrl, {
+          dest: this.tempPath,
+          logger,
+          extract: true,
+          strip: 1,
+          filename: this.name,
+        });
+      } else {
+        throw e;
+      }
+    }
   }
   private async doZipballUrl() {
     const maps = {