Merge pull request #154 from aliyun/tanhehe/refactor_docker_env_from_…

…array_to_obj

* refactor env from array to obj
* optimize docker stop operation
* fix InitializationTimeout bug when using fun local
* refactor nas browser example using fun install

examples/nas_browser/fun.yml

@@ -0,0 +1,3 @@
+runtime: python3
+tasks:
+  - pip: Flask

examples/nas_browser/template.yml

@@ -23,8 +23,8 @@ Resources:
       Type: 'Aliyun::Serverless::Function'
       Properties:
         Handler: index.handler
-        Runtime: python2.7
-        CodeUri: './src'
+        Runtime: python3
+        CodeUri: './'
         Timeout: 100
         EnvironmentVariables:
           ROOT_DIR: /mnt/nas

lib/debug.js

@@ -150,17 +150,17 @@ function generateDebugEnv(runtime, debugPort) {
 
   switch (runtime) {
   case 'nodejs8':
-    return `DEBUG_OPTIONS=--inspect-brk=0.0.0.0:${debugPort}`;
+    return { 'DEBUG_OPTIONS': `--inspect-brk=0.0.0.0:${debugPort}` };
   case 'nodejs6':
-    return `DEBUG_OPTIONS=--debug-brk=${debugPort}`;
+    return { 'DEBUG_OPTIONS': `--debug-brk=${debugPort}` };
   case 'python2.7':
   case 'python3':
-    return `DEBUG_OPTIONS=-m ptvsd --host 0.0.0.0 --port ${debugPort} --wait`;
+    return { 'DEBUG_OPTIONS': `-m ptvsd --host 0.0.0.0 --port ${debugPort} --wait` };
   case 'java8':
-    return `DEBUG_OPTIONS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,quiet=y,address=${debugPort}`;
+    return { 'DEBUG_OPTIONS': `-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,quiet=y,address=${debugPort}` };
   case 'php7.2':
     console.log(`using remote_ip ${remoteIp}`);
-    return `XDEBUG_CONFIG=remote_enable=1 remote_autostart=1 remote_port=${debugPort} remote_host=${remoteIp}`;
+    return { 'XDEBUG_CONFIG': `remote_enable=1 remote_autostart=1 remote_port=${debugPort} remote_host=${remoteIp}` };
   default:
     console.error(red('could not found runtime.'));
     process.exit(-1);

lib/docker-opts.js

@@ -6,6 +6,9 @@ const {
 
 const debug = require('debug')('docker-opts');
 const nestedObjectAssign = require('nested-object-assign');
+const { addEnv } = require('./install/env');
+
+const _ = require('lodash');
 
 const { red } = require('colors');
 
@@ -18,6 +21,10 @@ const runtimeImageMap = {
   'php7.2': 'php7.2'
 };
 
+function resolveDockerEnv(envs = {}) {
+  return _.map(addEnv(envs || {}), (v, k) => `${k}=${v}`);
+}
+
 function resolveRuntimeToDockerImage(runtime, isBuild) {
   if (runtimeImageMap[runtime]) {
     const name = runtimeImageMap[runtime];
@@ -79,7 +86,7 @@ async function generateLocalInvokeOpts(runtime, containerName, mounts, cmd, debu
 
   const opts = nestedObjectAssign(
     {
-      Env: envs,
+      Env: resolveDockerEnv(envs),
       Image: imageName,
       name: containerName,
       Cmd: cmd,
@@ -118,7 +125,7 @@ async function generateLocalStartRunOpts(runtime, name, mounts, cmd, debugPort,
 
   const opts = nestedObjectAssign(
     {
-      Env: envs,
+      Env: resolveDockerEnv(envs),
       Image: imageName,
       name,
       Cmd: cmd,
@@ -136,5 +143,5 @@ async function generateLocalStartRunOpts(runtime, name, mounts, cmd, debugPort,
 module.exports = {
   generateLocalInvokeOpts, resolveRuntimeToDockerImage, 
   generateInstallOpts, generateLocalStartRunOpts,
-  resolveMockScript
+  resolveMockScript, resolveDockerEnv
 };

lib/docker.js

@@ -16,7 +16,6 @@ const dockerOpts = require('./docker-opts');
 
 var containers = new Set();
 
-const { addEnv } = require('./install/env');
 const devnull = require('dev-null');
 
 // exit container, when use ctrl + c
@@ -35,8 +34,7 @@ function waitingForContainerStopped() {
   }
 
   let stopping = false;
-  let success = true;
-
+
   process.on('SIGINT', async () => {
 
     debug('containers length: ', containers.length);
@@ -60,18 +58,17 @@ function waitingForContainerStopped() {
 
     console.log(`\nreceived canncel request, stopping running containers.....`);
 
+    const jobs = [];
+
     for (let container of containers) {
       console.log(`stopping container ${container}`);
-
-      try {
-        await docker.getContainer(container).stop();
-      } catch (error) {
-        success = false;
-        console.error(error);
-      }
+      jobs.push(docker.getContainer(container).stop());
     }
 
-    if (!success) {
+    try {
+      await Promise.all(jobs);
+    } catch (error) {
+      console.error(error);
       process.exit(-1);
     }
   });
@@ -197,8 +194,9 @@ function generateDockerCmd(functionProps, httpMode) {
 
   const initializationTimeout = functionProps.InitializationTimeout;
 
+  // initializationTimeout is defined as integer, see lib/validate/schema/function.js
   if (initializationTimeout) {
-    cmd.push('--initializationTimeout', initializationTimeout);
+    cmd.push('--initializationTimeout', initializationTimeout.toString());
   }
 
   debug(`docker cmd: ${cmd}`);
@@ -258,44 +256,43 @@ async function pullImage(imageName) {
 }
 
 function generateFunctionEnvs(functionProps) {
-  const environmentVariables = addEnv(functionProps.EnvironmentVariables);
-
-  if (!environmentVariables) { return []; }
-
-  let envs = [];
-
-  for (const [envName, envValue] of Object.entries(environmentVariables)) {
-    envs.push(`${envName}=${envValue}`);
-  }
+  const environmentVariables = functionProps.EnvironmentVariables;
 
-  debug(`load function env: ${envs}`);
+  if (!environmentVariables) { return {}; }
 
-  return envs;
+  return Object.assign({}, environmentVariables);
 }
 
 function generateRamdomContainerName() {
   return `fun_local_${new Date().getTime()}_${Math.random().toString(36).substr(2, 7)}`;
 }
 
 async function generateDockerEnvs(functionProps, debugPort, httpParams) {
-  const envs = [];
+  const envs = {};
 
   if (httpParams) {
-    envs.push(`FC_HTTP_PARAMS=${httpParams}`);
+    Object.assign(envs, {
+      'FC_HTTP_PARAMS': httpParams
+    });
   }
 
   if (debugPort) {
-    const env = generateDebugEnv(functionProps.Runtime, debugPort);
+    const debugEnv = generateDebugEnv(functionProps.Runtime, debugPort);
 
-    debug('debug env: ' + env);
-    envs.push(env);
-  }
+    debug('debug env: ' + debugEnv);
 
-  envs.push(...generateFunctionEnvs(functionProps));
+    Object.assign(envs, debugEnv);
+  }
 
+  Object.assign(envs, generateFunctionEnvs(functionProps));
+
   const profile = await getProfile();
 
-  envs.push('local=true', `FC_ACCESS_KEY_ID=${profile.accessKeyId}`, `FC_ACCESS_KEY_SECRET=${profile.accessKeySecret}`);
+  Object.assign(envs, {
+    'local': true,
+    'FC_ACCESS_KEY_ID': profile.accessKeyId,
+    'FC_ACCESS_KEY_SECRET': profile.accessKeySecret
+  });
 
   return envs;
 }
@@ -417,10 +414,10 @@ async function startContainer(opts) {
       containers.delete(container.id);
     },
 
-    exec: async (cmd, {cwd = '', env = [], event, outputStream, errorStream, verbose = false} = {}) => {      
+    exec: async (cmd, {cwd = '', env = {}, event, outputStream, errorStream, verbose = false} = {}) => {      
       const options = {
         Cmd: cmd,
-        Env: env || [],
+        Env: dockerOpts.resolveDockerEnv(env),
         Tty: false,
         AttachStdin: true,  
         AttachStdout: true,

lib/install/install.js

@@ -4,25 +4,21 @@ const { AptTask, PipTask, ShellTask } = require('./task');
 const Context = require('./context');
 const { FunModule } = require('./module');
 const path = require('path');
-const { addEnv } = require('../install/env');
-const _ = require('lodash');
 
 async function installPackage(runtime, pkgType, pkgName, options) {
 
   const ctx = await new Context(runtime, options.codeUri);
 
   try {
-    const envs = _.map(addEnv(options.env || {}), (v, k) => `${k}=${v}`);
-
     switch (pkgType) {
 
     case 'apt':
       await new AptTask(options.name, runtime, options.codeUri, 
-        pkgName, options.local, envs, ctx, options.verbose).run();
+        pkgName, options.local, options.env, ctx, options.verbose).run();
       break;
     case 'pip':
       await new PipTask(options.name, runtime, options.codeUri, 
-        pkgName, options.local, envs, ctx, options.verbose).run();
+        pkgName, options.local, options.env, ctx, options.verbose).run();
       break;
     case 'module':
       // TODO
@@ -46,19 +42,18 @@ async function installFromYaml(file, verbose) {
   try {
     for (const t of funModule.tasks) {
 
-      const envs = _.map(addEnv(t.attrs.env || {}), (v, k) => `${k}=${v}`);
 
       if (t.type === 'pip') {
         const pipTask = new PipTask(t.attrs.name, runtime, codeUri, 
-          t.attrs.pip, t.attrs.local, envs,  ctx, verbose);
+          t.attrs.pip, t.attrs.local, t.attrs.env,  ctx, verbose);
         await pipTask.run();
       } else if (t.type === 'apt') {
         const aptTask = new AptTask(t.attrs.name, runtime, codeUri, 
-          t.attrs.apt, t.attrs.local, envs, ctx, verbose);
+          t.attrs.apt, t.attrs.local, t.attrs.env, ctx, verbose);
         await aptTask.run();
       } else if (t.type === 'shell') {
         const shellTask = new ShellTask(t.attrs.name, runtime, codeUri, 
-          t.attrs.shell, t.attrs.cwd, envs, ctx, verbose);
+          t.attrs.shell, t.attrs.cwd, t.attrs.env, ctx, verbose);
         await shellTask.run();
       } else {
         console.error('unkown task %s', t);

lib/install/task.js

@@ -4,7 +4,7 @@ const { startInstallationContainer } = require('../docker');
 const { cyan, green } = require('colors');
 
 class Task {
-  constructor(name, runtime, codeUri, env = [], context = null, verbose = false) {
+  constructor(name, runtime, codeUri, env = {}, context = null, verbose = false) {
     this.name = name;
     this.runtime = runtime;
     this.codeUri = codeUri;
@@ -62,7 +62,9 @@ class PipTask extends InstallTask {
       console.log(green('     => ') + cyan('PYTHONUSERBASE=/code/.fun/python pip install --user %s'), this.pkgName);
 
       await this._exec(['pip', 'install', '--user', '--no-warn-script-location', this.pkgName],
-        ['PIP_DISABLE_PIP_VERSION_CHECK=1', ...this.env]);
+        Object.assign({
+          'PIP_DISABLE_PIP_VERSION_CHECK': '1',
+        }, this.env));
     } else {
       console.log(green('     => ') + cyan('pip install %s'), this.pkgName);
       await this._exec(['pip', 'install', this.pkgName]);

lib/local/http-invoke.js

@@ -36,7 +36,7 @@ class HttpInvoke extends Invoke {
       }
 
       console.log(`detect code changes, file is ${name}, event is ${evt}, auto reloading...`);
-      
+
       oldRunner = this.runner;
 
       this.runner = null;
@@ -102,38 +102,40 @@ class HttpInvoke extends Invoke {
   }
 
   async doInvoke(req, res) {
-    const outputStream = new streams.WritableStream();
-    const errorStream = new streams.WritableStream();
+    // only one invoke can be processed
+    await lock.acquire('invoke', async () => {
+      debug('http doInvoke, aquire invoke lock success, processing...');
 
-    const event = await getHttpRawBody(req);
+      const outputStream = new streams.WritableStream();
+      const errorStream = new streams.WritableStream();
 
-    const httpParams = generateHttpParams(req, this.endpointPrefix);
+      const event = await getHttpRawBody(req);
 
-    const envs = await docker.generateDockerEnvs(this.functionProps, this.debugPort, httpParams);
+      const httpParams = generateHttpParams(req, this.endpointPrefix);
 
-    if (this.debugPort) {
-      // don't reuse container
-      const cmd = docker.generateDockerCmd(this.functionProps, true);
+      const envs = await docker.generateDockerEnvs(this.functionProps, this.debugPort, httpParams);
 
-      const opts = await dockerOpts.generateLocalInvokeOpts(
-        this.runtime,
-        this.containerName,
-        this.mounts,
-        cmd,
-        this.debugPort,
-        envs,
-        this.dockerUser);
+      if (this.debugPort) {
+        // don't reuse container
+        const cmd = docker.generateDockerCmd(this.functionProps, true);
 
-      await docker.run(opts,
-        event,
-        outputStream, errorStream);
-    } else {
-      // reuse container
-      debug('http doInvoke, acquire invoke lock');
+        this.containerName = docker.generateRamdomContainerName();
 
-      // only one invoke can be processed
-      await lock.acquire('invoke', async () => {
-        debug('http doInvoke, aquire invoke lock success, processing...');
+        const opts = await dockerOpts.generateLocalInvokeOpts(
+          this.runtime,
+          this.containerName,
+          this.mounts,
+          cmd,
+          this.debugPort,
+          envs,
+          this.dockerUser);
+
+        await docker.run(opts,
+          event,
+          outputStream, errorStream);
+      } else {
+        // reuse container
+        debug('http doInvoke, acquire invoke lock');
 
         const cmd = [dockerOpts.resolveMockScript(this.runtime), ...docker.generateDockerCmd(this.functionProps, true)];
 
@@ -144,19 +146,19 @@ class HttpInvoke extends Invoke {
           if (!await validateSignature(req, res, req.method)) { return; }
         }
 
-        await this.runner.exec(cmd, { 
-          env: envs, 
+        await this.runner.exec(cmd, {
+          env: envs,
           event,
-          outputStream, 
-          errorStream, 
+          outputStream,
+          errorStream,
           verbose: true
         });
 
         debug('http doInvoke exec end, begin to response');
-      });
-    }
+      }
 
-    this.response(outputStream, errorStream, res);
+      this.response(outputStream, errorStream, res);
+    });
   }
 
   // responseHttpTriggers