Update V3 staging

CHANGELOG.md

@@ -3,6 +3,22 @@
 All notable changes to the Zlux Server Framework package will be documented in this file..
 This repo is part of the app-server Zowe Component, and the change logs here may appear on Zowe.org in that section.
 
+## 2.17.0
+- Enhancement: Added function `isClientAttls(zoweConfig)` within `libs/util.js`. Whenever a plugin makes a network request, it should always use this to determine if a normally HTTPS request should instead be made as HTTP due to AT-TLS handling the TLS when enabled. (#544)
+- Bugfix: Fixed function `isServerAttls(zoweConfig)` within `libs/util.js`, which was preventing using AT-TLS with app-server. (#544)
+
+## 2.15.0
+- Bugfix: App-server could not run in HTTP mode for AT-TLS setup because it was not able to merge HTTPS and HTTP addresses. (#984)
+
+## 2.14.0
+- Bugfix: App-server could not load when multiple discovery servers were present and the app-server was unable to reach the first one specified. Now, the app-server will iterate through the list of servers until an accessible one is reached. (#522)
+- Bugfix: App-server would not correctly detect when it was running in a high-availability configuration environment. (#521)
+- Bugfix: A call to GET /plugins would trigger an authorization check regardless of if rbac was set on or off (#523)
+
+## 2.13.0
+- Added support for using zowe.network and components.app-server.zowe.network to set listener IP and TLS properties including max and min version, ciphers, and ECDH curves. (#511)
+- Enhanced cipher customization to allow for ciphers to be specified in IANA format in addition to the existing OpenSSL format. (#511)
+
 ## 2.12.0
 
 - Enhancement: Auth plugins that are not requested by any dataservice found at startup are no longer loaded by the server.

lib/apiml.js

@@ -12,6 +12,7 @@ const Promise = require("bluebird");
 const eureka = require('@rocketsoftware/eureka-js-client').Eureka;
 const zluxUtil = require('./util');
 const https = require('https');
+const http = require('http');
 
 const log = zluxUtil.loggers.apiml;
 
@@ -77,10 +78,10 @@ const MEDIATION_LAYER_INSTANCE_DEFAULTS = (zluxProto, zluxHostname, zluxPort) =>
   }
 }};
 
-function ApimlConnector({ hostName, port, isHttps, discoveryHost,
-    discoveryPort, tlsOptions, eurekaOverrides }) {
-  Object.assign(this, { hostName, port, isHttps, discoveryHost,
-    discoveryPort, tlsOptions, eurekaOverrides });
+function ApimlConnector({ hostName, port, discoveryUrls,
+                          discoveryPort, tlsOptions, eurekaOverrides, isClientAttls }) {
+  Object.assign(this, { hostName, port, discoveryUrls,
+                        discoveryPort, tlsOptions, eurekaOverrides, isClientAttls });
   this.vipAddress = hostName;
 }
 
@@ -110,29 +111,21 @@ ApimlConnector.prototype = {
     const end = Date.now() + timer;
 
     return new Promise((resolve, reject) => {
-      const options = Object.assign({
-        host: this.discoveryHost,
-        port: this.discoveryPort,
-        method: 'GET',
-        path: `/eureka/apps/${serviceName}`,
-        headers: {'accept':'application/json'}
-      }, this.tlsOptions);
-
-      if (!this.tlsOptions.rejectUnauthorized) {
-        //Keeping these certs causes an openssl error 46, unknown cert error in a dev environment
-        delete options.cert;
-        delete options.key;
-      } //else, apiml expects a cert and will give a 403.
+      const optionsArray = this.getRequestOptionsArray('GET', `/eureka/apps/${serviceName}`);
+      let optionsIndex = 0;
 
       const issueRequest = () => {
+        const options = optionsArray[optionsIndex];
         if (Date.now() > end) {
-          log.warn(`ZWED0045`, this.discoveryHost, this.discoveryPort);
+          log.warn(`ZWED0045W`, this.discoveryHost, this.discoveryPort);
           return reject(new Error(`Call timeout when fetching agent status from APIML`));
         }
 
         let data = [];
+
+        let httpModule = this.isClientAttls ? http : https;
 
-        const req = https.request(options, (res) => {
+        const req = httpModule.request(options, (res) => {
           res.on('data', (chunk) => data.push(chunk));
           res.on('end', () => {
             log.debug(`Query rc=`,res.statusCode);
@@ -156,7 +149,9 @@ ApimlConnector.prototype = {
           reject(new Error(`Call timeout when fetching agent status from APIML`));
         });
         req.on('error', (error) => {
-          log.warn("APIML query error:", error.message);
+          log.warn("ZWED0180W", options.host, options.port, error.message);
+          //
+          optionsIndex = (optionsIndex+1) % optionsArray.length;
           setTimeout(issueRequest, AGENT_CHECK_RECONNECT_DELAY);
         });
         req.end();
@@ -173,10 +168,10 @@ ApimlConnector.prototype = {
       // If the HTTP port is set to 0 then the API ML doesn't load zlux
       httpPort: Number(this.port),
       httpsPort: Number(this.port),
-      httpEnabled: !this.isHttps,
-      httpsEnabled: this.isHttps
+      httpEnabled: false,
+      httpsEnabled: true
     };
-    const proto = this.isHttps? 'https' : 'http';
+    const proto = 'https';
 
     log.debug("ZWED0141I", proto, this.port); //"Protocol:", proto, "Port", port);
     log.debug("ZWED0142I", JSON.stringify(protocolObject)); //"Protocol Object:", JSON.stringify(protocolObject));
@@ -249,8 +244,7 @@ ApimlConnector.prototype = {
     }
     log.debug("ZWED0144I", JSON.stringify(zluxProxyServerInstanceConfig, null, 2)); //log.debug("zluxProxyServerInstanceConfig: " 
         //+ JSON.stringify(zluxProxyServerInstanceConfig, null, 2))
-    const defaultUrl = `https://${this.discoveryHost}:${this.discoveryPort}/eureka/apps`;
-    const serviceUrls = this.getServiceUrls(defaultUrl);
+    const serviceUrls = this.getServiceUrls();
     zluxProxyServerInstanceConfig.eureka.serviceUrls = { default: serviceUrls };
     log.info(`ZWED0020I`, serviceUrls.join(',')); //log.info(`Registering at ${url}...`);
     log.debug("ZWED0145I", JSON.stringify(zluxProxyServerInstanceConfig)); //log.debug(`zluxProxyServerInstanceConfig ${JSON.stringify(zluxProxyServerInstanceConfig)}`)
@@ -285,17 +279,29 @@ ApimlConnector.prototype = {
     });
   },
 
-  getServiceUrls(defaultUrl) {
-    const discoveryServiceList = process.env['ZWE_DISCOVERY_SERVICES_LIST'] || '';
-    const serviceUrls = discoveryServiceList
-      .split(',')
-      .map(url => url.trim())
-      .filter(url => url.length > 0)
-      .map(url => url + (url.endsWith('/') ? '' : '/') + 'apps');
-    if (serviceUrls.length === 0) {
-      serviceUrls.push(defaultUrl);
-    }
-    return serviceUrls;
+  getServiceUrls() {
+    return this.discoveryUrls.map(url => url + (url.endsWith('/') ? '' : '/') + 'apps');
+  },
+
+  getRequestOptionsArray(method, path) {
+    return this.discoveryUrls.map((url)=>{
+      //in the form of https://host:port/eureka/, trim from https:// and following slash.
+      const hostAndPort = url.substring(8, url.indexOf('/', 8)).split(':');
+      const options = Object.assign({
+        host: hostAndPort[0],
+        port: hostAndPort[1],
+        method: method,
+        path: path,
+        headers: {'accept':'application/json'}
+      }, this.tlsOptions);
+
+      if (!this.tlsOptions.rejectUnauthorized) {
+        //Keeping these certs causes an openssl error 46, unknown cert error in a dev environment
+        delete options.cert;
+        delete options.key;
+      } //else, apiml expects a cert and will give a 403.
+      return options;
+    });
   }
 
   /*

lib/apimlStorage.ts

@@ -16,10 +16,17 @@ import { AxiosInstance } from 'axios';
 let apimlClient: AxiosInstance;
 
 export function configure(settings: ApimlStorageSettings) {
-  apimlClient = axios.create({
-    baseURL: `https://${settings.host}:${settings.port}`,
-    httpsAgent: new https.Agent(settings.tlsOptions)
-  });
+  if (settings.isHttps) {
+    apimlClient = axios.create({
+      baseURL: `https://${settings.host}:${settings.port}`,
+      httpsAgent: new https.Agent(settings.tlsOptions)
+    });
+  } else {
+    apimlClient = axios.create({
+      baseURL: `https://${settings.host}:${settings.port}`,
+      httpAgent: new http.Agent()
+    });
+  }
 }
 
 export function isConfigured(): boolean {
@@ -30,6 +37,7 @@ export interface ApimlStorageSettings {
   host: string;
   port: number;
   tlsOptions: https.AgentOptions;
+  isHttps?: boolean;
 }
 
 
@@ -162,7 +170,7 @@ async function apimlDoRequest(req: ApimlRequest): Promise<ApimlResponse> {
       headers: req.headers,
     });
     const apimlResponse: ApimlResponse = {
-      headers: response.headers,
+      headers: response.headers as http.IncomingHttpHeaders,
       statusCode: response.status,
       json: response.data
     };
@@ -375,4 +383,4 @@ class ApimlStorage {
   SPDX-License-Identifier: EPL-2.0
 
   Copyright Contributors to the Zowe Project.
-*/
+*/

lib/assets/i18n/log/messages_en.json

@@ -307,6 +307,7 @@
   "ZWED0177W":"Unable to load %s for '%s' into config",
   "ZWED0178W":"Skipping authentication plugin %s because it's not HA compatible",
   "ZWED0179W":"Unable to retrieve the list of certificate authorities from the keyring=%s owner=%s Error: %s",
+  "ZWED0180W":"Failed to query discovery server (%s:%s) for agent access: %s",
 
   "ZWED0001E":"RESERVED: Error: %s",
   "ZWED0002E":"Could not stop language manager for types=%s",

lib/auth-manager.js

@@ -116,7 +116,8 @@ AuthManager.prototype = {
                                               plugin,
                                               this.configuration,
                                               componentConfig,
-                                              new AuthPluginContext(plugin, tlsOptions));
+                                              new AuthPluginContext(plugin, tlsOptions),
+                                              config);
         // at this time we should have resolved plugin configuration to have a 
         // nice list of info about what we are using to authenticate against
         if ((typeof authenticationHandler.authenticate) !== 'function') {

lib/index.js

@@ -218,11 +218,10 @@ Server.prototype = {
         this.apiml = new ApimlConnector({
           hostName: webAppOptions.hostname,
           port: this.port,
-          isHttps: util.isServerHttps(this.zoweConfig),
-          discoveryHost: apimlConfig.server.hostname,
-          discoveryPort: apimlConfig.server.port,
+          discoveryUrls: apimlConfig.server.discoveryUrls || [`https://${apimlConfig.server.hostname}:${apimlConfig.server.port}/eureka/`],
           tlsOptions: this.tlsOptions,
-          eurekaOverrides: apimlConfig.eureka
+          eurekaOverrides: apimlConfig.eureka,
+          isClientAttls: util.isClientAttls(this.zoweConfig)
         });
         yield this.apiml.setBestIpFromConfig(this.componentConfig.node);
         yield this.apiml.registerMainServerInstance();
@@ -236,7 +235,7 @@ Server.prototype = {
          && this.componentConfig.agent.mediationLayer.serviceName
          && this.componentConfig.node.mediationLayer.server?.gatewayPort) {
         //at this point, we expect zss to also be attached to the mediation layer, so lets adjust.
-        webAppOptions.proxiedHost = apimlConfig.server.hostname;
+        webAppOptions.proxiedHost = apimlConfig.server.gatewayHostname;
         webAppOptions.proxiedPort = this.componentConfig.node.mediationLayer.server.gatewayPort;
         if (firstWorker) {
           yield this.apiml.checkAgent(this.componentConfig.agent.handshakeTimeout,
@@ -264,7 +263,7 @@ Server.prototype = {
 
     bootstrapLogger.info('ZWED0302I', util.isHaMode() ? 'enabled' : 'disabled'); // "HA mode is %s"
     if (apimlConfig.cachingService?.enabled) {
-      this.configureApimlStorage(apimlConfig);
+      this.configureApimlStorage(apimlConfig, util.isClientAttls(this.zoweConfig));
     }
 
     const plugins = yield this.loadPlugins();
@@ -336,11 +335,12 @@ Server.prototype = {
     return yield this.pluginLoader.loadPlugins();
   }),
 
-  configureApimlStorage(apimlConfig) {
+  configureApimlStorage(apimlConfig, isHttps) {
     apimlStorage.configure({
       host: apimlConfig.server.gatewayHostname,
       port: apimlConfig.server.gatewayPort,
-      tlsOptions: this.tlsOptions
+      tlsOptions: this.tlsOptions,
+      isHttps: isHttps
     });
     bootstrapLogger.info(`ZWED0300I`); // Caching Service configured
   },

lib/swagger-catalog.js

@@ -15,7 +15,7 @@ const Promise = require('bluebird');
 const zLuxUrl = require('./url')
 const path = require('path');
 const fs = require('fs');
-const jsyaml = require('js-yaml');
+const yaml = require('yaml');
 const swaggerParser = require('swagger-parser')
 const os = require('os');
 const zluxUtil = require('./util');
@@ -201,7 +201,7 @@ function readSingleSwaggerFile (dirName, serviceName, serviceVersion) {
             if (err) {
               return reject(err);
             }
-            let swaggerJson = jsyaml.safeLoad(fileContent);
+            let swaggerJson = yaml.parse(fileContent);
             swaggerParser.validate(swaggerJson).then(function(valid) {
               return resolve(swaggerJson)
             }).catch(function(err) {

lib/tomcatManager.ts

@@ -21,12 +21,10 @@ unpack the wars ahead of time, so the symbolic links are the unpacked dirs
 import { Path, TomcatConfig, TomcatShutdown, TomcatHttps, JavaServerManager, AppServerInfo } from './javaTypes';
 import * as fs from 'graceful-fs';
 import * as path from 'path';
-import * as mkdirp from 'mkdirp';
 import * as child_process from 'child_process';
 //import * as xml2js from 'xml2js';
 import * as yauzl from 'yauzl';
 import * as utils from './util';
-import * as rimraf from 'rimraf';
 
 const log = utils.loggers.langManager;
 const spawn = child_process.spawn;
@@ -63,7 +61,7 @@ export class TomcatManager implements JavaServerManager {
 
   private makeRoot():Promise<void> {
     return new Promise((resolve,reject)=> {
-      mkdirp(this.appdir, {mode: DIR_WRITE_MODE}, (err)=> {
+      fs.mkdir(this.appdir, {recursive: true, mode: DIR_WRITE_MODE}, (err)=> {
         if (err) {
           reject(err);
         } else {
@@ -279,7 +277,7 @@ export class TomcatManager implements JavaServerManager {
     log.info(`ZWED0092I`, this.id); //log.info(`Tomcat Manager ID=${this.id} stopping`);
     TomcatManager.isWindows ? this.stopForWindows() : this.stopForUnix();
     return new Promise((resolve, reject) => {
-      rimraf(this.appdir, (error)=> {
+      fs.rm(this.appdir, { recursive: true, force: true }, (error)=> {
         if (error) {
           reject(error);
         } else {
@@ -368,7 +366,7 @@ export class TomcatManager implements JavaServerManager {
           zipfile.on("entry", function(entry) {
             if (entry.fileName.endsWith('/')) {
               //directory
-              mkdirp(path.join(destPath,entry.fileName), {mode: DIR_WRITE_MODE}, (err)=> {
+              fs.mkdir(path.join(destPath,entry.fileName), {recursive: true, mode: DIR_WRITE_MODE}, (err)=> {
                 if (err) {
                   error = err;                  
                   zipfile.close();
@@ -380,7 +378,7 @@ export class TomcatManager implements JavaServerManager {
               zipfile.readEntry(); //TODO is it correct to skip this?
             } else {
               //file
-              mkdirp(path.join(destPath,path.dirname(entry.fileName)), {mode: DIR_WRITE_MODE}, (err)=> {
+              fs.mkdir(path.join(destPath,path.dirname(entry.fileName)), {recursive: true, mode: DIR_WRITE_MODE}, (err)=> {
                 if (err) {
                   error = err;
                   zipfile.close();

lib/translation-utils.js

@@ -10,8 +10,8 @@
 
 'use strict';
 const path = require('path');
+const fs = require('fs');
 const jsonUtils = require('./jsonUtils.js');
-const glob = require('glob');
 const zluxUtil = require('./util');
 const acceptLanguageParser = require('accept-language-parser');
 
@@ -31,15 +31,22 @@ const utilLog = zluxUtil.loggers.utilLogger;
 */
 function loadTranslations(pluginLocation) {
   const translationMaps = {};
-  const relativePath = 'web/assets/i18n';
   const filePrefix = 'pluginDefinition.i18n.';
   const fileExt = '.json';
-  const pattern = path.join(
-    pluginLocation,
-    relativePath,
-    `${filePrefix}*${fileExt}`
-  );
-  const files = glob.sync(pattern, {});
+  const folder = path.join(pluginLocation, 'web/assets/i18n');
+  let files = [];
+  try {
+    files = fs.readdirSync(folder)
+          .filter((filename)=> {
+            return filename.startsWith(filePrefix) && filename.endsWith(fileExt);
+          })
+          .map((filename)=> {
+            return path.join(folder, filename);
+          });
+  } catch (e) {
+    //A plugin may not have translations and that is OK.
+    return translationMaps;
+  }
   for (const file of files) {
     const basename = path.basename(file);
     const languageCountry = basename.substring(