feat: register gateway to additional discovery service

.github/workflows/ci-tests.yml

@@ -28,13 +28,22 @@ jobs:
 
             -   name: Build with Gradle
                 run: >
-                    ./gradlew clean build runStartUpCheck --info --scan -Partifactory_user=${{ secrets.ARTIFACTORY_USERNAME }} -Partifactory_password=${{ secrets.ARTIFACTORY_PASSWORD }}
+                    ./gradlew clean build --info --scan
+            -   name: Run startup check
+                run: >
+                    ./gradlew runStartUpCheck --info --scan
             -   name: Store results
                 uses: actions/upload-artifact@v2
                 if: always()
                 with:
                     name: CITests-${{ env.JOB_ID }}
                     path: |
                         integration-tests/build/reports/**
+                        gateway-service/build/reports/**
+                        api-catalog-services/build/reports/**
+                        caching-service/build/reports/**
+                        cloud-gateway-service/build/reports/**
+                        discovery-service/build/reports/**
+                        metrics-service/build/reports/**
 
             -   uses: ./.github/actions/teardown

.github/workflows/containers.yml

@@ -55,12 +55,19 @@ jobs:
                 image: ghcr.io/balhar-jakub/discovery-service:${{ github.run_id }}-${{ github.run_number }}
                 volumes:
                     - /api-defs:/api-defs
+            discovery-service-2:
+                image: ghcr.io/balhar-jakub/discovery-service:${{ github.run_id }}-${{ github.run_number }}
+                volumes:
+                    - /api-defs:/api-defs
+                env:
+                    APIML_SERVICE_HOSTNAME: discovery-service-2
             gateway-service:
                 image: ghcr.io/balhar-jakub/gateway-service:${{ github.run_id }}-${{ github.run_number }}
                 env:
                     APIML_SECURITY_AUTH_JWT_CUSTOMAUTHHEADER: customJwtHeader
                     APIML_SECURITY_AUTH_PASSTICKET_CUSTOMUSERHEADER: customUserHeader
                     APIML_SECURITY_AUTH_PASSTICKET_CUSTOMAUTHHEADER: customPassticketHeader
+                    APIML_SERVICE_CENTRALREGISTRYURLS: https://discovery-service-2:10011/eureka
             mock-services:
                 image: ghcr.io/balhar-jakub/mock-services:${{ github.run_id }}-${{ github.run_number }}
             metrics-service:

.run/GatewayApplication.run.xml

@@ -8,7 +8,6 @@
       </param>
     </additionalParameters>
     <option name="ALTERNATIVE_JRE_PATH" value="1.8 (2)" />
-    <option name="ALTERNATIVE_JRE_PATH_ENABLED" value="true" />
     <module name="api-layer.gateway-service.main" />
     <option name="SHORTEN_COMMAND_LINE" value="MANIFEST" />
     <option name="SPRING_BOOT_MAIN_CLASS" value="org.zowe.apiml.gateway.GatewayApplication" />

apiml-common/src/main/java/org/zowe/apiml/product/web/HttpConfig.java

@@ -299,5 +299,9 @@ public EurekaJerseyClient eurekaJerseyClient() {
         return eurekaJerseyClientBuilder.build();
     }
 
+    @Bean
+    public EurekaJerseyClientBuilder eurekaJerseyClientBuilder() {
+        return eurekaJerseyClientBuilder;
+    }
 
 }

config/local/gateway-service.yml

@@ -5,7 +5,9 @@ apiml:
         hostname: localhost
         ipAddress: 127.0.0.1
         port: 10010
+        centralRegistryUrls: https://localhost:10021/eureka/,https://localhost:10031/eureka/
         discoveryServiceUrls: https://localhost:10011/eureka/
+
     security:
         allowTokenRefresh: true
         webfinger:

gateway-package/src/main/resources/bin/start.sh

@@ -207,6 +207,7 @@ _BPX_JOBNAME=${ZWE_zowe_job_prefix}${GATEWAY_CODE} java \
     -Dapiml.service.hostname=${ZWE_haInstance_hostname:-localhost} \
     -Dapiml.service.port=${ZWE_configs_port:-7554} \
     -Dapiml.service.discoveryServiceUrls=${ZWE_DISCOVERY_SERVICES_LIST:-"https://${ZWE_haInstance_hostname:-localhost}:${ZWE_components_discovery_port:-7553}/eureka/"} \
+    -Dapiml.service.centralRegistryUrls=${ZWE_configs_apiml_service_centralRegistryUrls:-} \
     -Dapiml.service.allowEncodedSlashes=${ZWE_configs_apiml_service_allowEncodedSlashes:-true} \
     -Dapiml.service.corsEnabled=${ZWE_configs_apiml_service_corsEnabled:-false} \
     -Dapiml.service.externalUrl="${httpProtocol}://${ZWE_zowe_externalDomains_0}:${ZWE_zowe_externalPort}" \

gateway-service/src/main/java/org/zowe/apiml/gateway/config/DiscoveryClientConfig.java

@@ -14,46 +14,86 @@
 import com.netflix.appinfo.HealthCheckHandler;
 import com.netflix.discovery.AbstractDiscoveryClientOptionalArgs;
 import com.netflix.discovery.EurekaClientConfig;
+import com.netflix.discovery.shared.transport.jersey.EurekaJerseyClientImpl;
 import lombok.RequiredArgsConstructor;
-import org.springframework.aop.support.AopUtils;
+import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.cloud.context.config.annotation.RefreshScope;
+import org.springframework.cloud.netflix.eureka.EurekaClientConfigBean;
+import org.springframework.cloud.netflix.eureka.MutableDiscoveryClientOptionalArgs;
 import org.springframework.cloud.util.ProxyUtils;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.zowe.apiml.gateway.discovery.ApimlDiscoveryClient;
 
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 /**
  * This configuration override bean EurekaClient with custom ApimlDiscoveryClient. This bean offer additional method
  * fetchRegistry. User can call this method to asynchronously fetch new data from discovery service. There is no time
  * to fetching.
  * <p>
- * Configuration also add listeners to call other beans waiting for fetch new registry. It speed up distribution of
+ * Configuration also add listeners to call other beans waiting for fetch new registry. It speeds up distribution of
  * changes in whole gateway.
  */
 @Configuration
 @RequiredArgsConstructor
 public class DiscoveryClientConfig {
     private final ApplicationContext context;
     private final AbstractDiscoveryClientOptionalArgs<?> optionalArgs;
+    private final EurekaJerseyClientImpl.EurekaJerseyClientBuilder eurekaJerseyClientBuilder;
+
+    @Value("${apiml.service.centralRegistryUrls:-}")
+    private String[] centralRegistryUrls;
 
     @Bean(destroyMethod = "shutdown")
     @RefreshScope
-    public ApimlDiscoveryClient eurekaClient(ApplicationInfoManager manager,
-                                             EurekaClientConfig config,
-                                             @Autowired(required = false) HealthCheckHandler healthCheckHandler
+    public ApimlDiscoveryClient primaryApimlEurekaClient(ApplicationInfoManager manager,
+                                                         EurekaClientConfig config,
+                                                         @Autowired(required = false) HealthCheckHandler healthCheckHandler
     ) {
-        ApplicationInfoManager appManager;
-        if (AopUtils.isAopProxy(manager)) {
-            appManager = ProxyUtils.getTargetObject(manager);
-        } else {
-            appManager = manager;
-        }
+        ApplicationInfoManager appManager = ProxyUtils.getTargetObject(manager);
+
         final ApimlDiscoveryClient discoveryClientClient = new ApimlDiscoveryClient(appManager, config, this.optionalArgs, this.context);
         discoveryClientClient.registerHealthCheck(healthCheckHandler);
 
         return discoveryClientClient;
     }
 
+    @Bean(destroyMethod = "shutdown")
+    @ConditionalOnProperty(name = "apiml.service.centralRegistryUrls")
+    @RefreshScope
+    public DiscoveryClientWrapper additionalDiscoveryClientWrapper(ApplicationInfoManager manager,
+                                                                   EurekaClientConfig config,
+                                                                   @Autowired(required = false) HealthCheckHandler healthCheckHandler
+    ) {
+        ApplicationInfoManager appManager = ProxyUtils.getTargetObject(manager);
+        List<ApimlDiscoveryClient> discoveryClientsList = new ArrayList<>();
+
+        for (String url : centralRegistryUrls) {
+
+            EurekaClientConfigBean configBean = new EurekaClientConfigBean();
+            BeanUtils.copyProperties(config, configBean);
+
+            Map<String, String> urls = new HashMap<>();
+            urls.put("defaultZone", url);
+
+            configBean.setServiceUrl(urls);
+
+            MutableDiscoveryClientOptionalArgs args = new MutableDiscoveryClientOptionalArgs();
+            args.setEurekaJerseyClient(eurekaJerseyClientBuilder.build());
+
+            final ApimlDiscoveryClient discoveryClientClient = new ApimlDiscoveryClient(appManager, configBean, args, this.context);
+            discoveryClientClient.registerHealthCheck(healthCheckHandler);
+            discoveryClientsList.add(discoveryClientClient);
+        }
+
+        return new DiscoveryClientWrapper(discoveryClientsList);
+    }
 }

gateway-service/src/main/java/org/zowe/apiml/gateway/config/DiscoveryClientWrapper.java

@@ -0,0 +1,29 @@
+/*
+ * This program and the accompanying materials are made available under the terms of the
+ * Eclipse Public License v2.0 which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-v20.html
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Copyright Contributors to the Zowe Project.
+ */
+
+package org.zowe.apiml.gateway.config;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import org.zowe.apiml.gateway.discovery.ApimlDiscoveryClient;
+
+import java.util.List;
+
+@AllArgsConstructor
+@Getter
+public class DiscoveryClientWrapper {
+    private List<ApimlDiscoveryClient> discoveryClients;
+
+    public void shutdown() {
+        if (discoveryClients != null) {
+            discoveryClients.forEach(ApimlDiscoveryClient::shutdown);
+        }
+    }
+}

gateway-service/src/main/java/org/zowe/apiml/gateway/controllers/CacheServiceController.java

@@ -23,7 +23,7 @@
 
 /**
  * This controller allows control the caches about services. The main purpose is to evict cached data
- * about services when a update happened in discovery service. Discovery service notifies about any
+ * about services when an update happened in discovery service. Discovery service notifies about any
  * change to be sure that cache on gateway is still valid.
  */
 @AllArgsConstructor

gateway-service/src/main/java/org/zowe/apiml/gateway/discovery/ApimlDiscoveryClient.java

@@ -26,7 +26,7 @@
 import java.util.concurrent.TimeUnit;
 
 /**
- * Custom implementation of Eureka client. It support additional feature:
+ * Custom implementation of Eureka client. It supports additional feature:
  * - fetchRegistry - invoke asynchronous task to update registry from discovery client immediatelly
  */
 public class ApimlDiscoveryClient extends CloudEurekaClient {

gateway-service/src/main/java/org/zowe/apiml/gateway/security/service/JwtSecurity.java

@@ -258,7 +258,6 @@ public Optional<JWK> getJwkPublicKey() {
      */
     private void validateInitializationWhenZosmfIsAvailable() {
         zosmfListener.register();
-
         new Thread(() -> {
             try {
                 events.add("Started waiting for z/OSMF instance " + zosmfServiceId + " to be registered and known by the discovery service");
@@ -273,7 +272,6 @@ private void validateInitializationWhenZosmfIsAvailable() {
                     apimlLog.log("org.zowe.apiml.gateway.jwtProducerConfigError", StringUtils.join(events, "\n"));
                 }
                 apimlLog.log("org.zowe.apiml.security.zosmfInstanceNotFound", zosmfServiceId);
-                System.exit(1);
             }
         }).start();
     }
@@ -317,7 +315,7 @@ public void onEvent(EurekaEvent event) {
                         synchronized (events) {
                             apimlLog.log("org.zowe.apiml.gateway.jwtProducerConfigError", StringUtils.join(events, "\n"));
                         }
-                        System.exit(1);
+                        System.exit(1); // TODO remove
                     }
                 } else {
                     events.add("z/OSMF instance " + zosmfServiceId + " is not available and online yet.");

gateway-service/src/main/java/org/zowe/apiml/gateway/security/service/zosmf/AbstractZosmfService.java

@@ -12,16 +12,12 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.netflix.discovery.DiscoveryClient;
+import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.security.authentication.AuthenticationServiceException;
 import org.springframework.security.authentication.BadCredentialsException;
 import org.springframework.security.core.Authentication;
-import org.springframework.web.client.HttpClientErrorException;
-import org.springframework.web.client.ResourceAccessException;
-import org.springframework.web.client.RestClientException;
-import org.springframework.web.client.RestClientResponseException;
-import org.springframework.web.client.RestTemplate;
+import org.springframework.web.client.*;
 import org.zowe.apiml.message.log.ApimlLogger;
 import org.zowe.apiml.product.logging.annotations.InjectApimlLogger;
 import org.zowe.apiml.security.common.config.AuthConfigurationProperties;
@@ -30,14 +26,14 @@
 import org.zowe.apiml.util.EurekaUtils;
 
 import javax.net.ssl.SSLHandshakeException;
-
 import java.net.ConnectException;
 import java.nio.charset.StandardCharsets;
 import java.util.*;
 import java.util.function.Supplier;
 
 import static org.zowe.apiml.security.SecurityUtils.readPassword;
 
+@RequiredArgsConstructor
 @Slf4j
 public abstract class AbstractZosmfService {
 
@@ -54,18 +50,6 @@ public abstract class AbstractZosmfService {
     protected final RestTemplate restTemplateWithoutKeystore;
     protected final ObjectMapper securityObjectMapper;
 
-    protected AbstractZosmfService(
-        AuthConfigurationProperties authConfigurationProperties,
-        DiscoveryClient discovery,
-        @Qualifier("restTemplateWithoutKeystore") RestTemplate restTemplateWithoutKeystore,
-        ObjectMapper securityObjectMapper
-    ) {
-        this.authConfigurationProperties = authConfigurationProperties;
-        this.discovery = discovery;
-        this.restTemplateWithoutKeystore = restTemplateWithoutKeystore;
-        this.securityObjectMapper = securityObjectMapper;
-    }
-
     /**
      * @return serviceId of z/OSMF service from configuration, which is used
      */
@@ -121,7 +105,6 @@ protected String getAuthenticationValue(Authentication authentication) {
      *
      * @param zosmf the z/OSMF service id
      * @return the uri
-     *
      * @throws ServiceNotAccessibleException if z/OSMF is not available in discovery service
      */
     protected String getURI(String zosmf) {

gateway-service/src/main/resources/application.yml

@@ -101,6 +101,9 @@ spring:
     application:
         name: ${apiml.service.id}
     cloud:
+        discovery:
+            reactive:
+                enabled: false
         compatibilityVerifier:
             enabled: false # Should be removed when upgrade to Spring Cloud 3.x
         client:

gateway-service/src/test/java/org/zowe/apiml/gateway/config/DiscoveryClientBeanTest.java

@@ -0,0 +1,51 @@
+/*
+ * This program and the accompanying materials are made available under the terms of the
+ * Eclipse Public License v2.0 which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-v20.html
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Copyright Contributors to the Zowe Project.
+ */
+
+package org.zowe.apiml.gateway.config;
+
+import com.netflix.appinfo.*;
+import com.netflix.discovery.shared.transport.jersey.EurekaJerseyClientImpl;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.springframework.cloud.netflix.eureka.EurekaClientConfigBean;
+import org.springframework.context.ApplicationContext;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class DiscoveryClientBeanTest {
+    DiscoveryClientConfig dcConfig;
+
+    @BeforeEach
+    void setup() {
+        ApplicationContext context = mock(ApplicationContext.class);
+        EurekaJerseyClientImpl.EurekaJerseyClientBuilder builder = mock(EurekaJerseyClientImpl.EurekaJerseyClientBuilder.class);
+        dcConfig = new DiscoveryClientConfig(context, null, builder);
+    }
+
+    @Test
+    void givenListOfCentralRegistryURLs_thenCreateNewDiscoveryClientForEach() {
+        String[] centralRegistryUrls = {"https://host:10021/eureka", "https://host:10011/eureka"};
+        ReflectionTestUtils.setField(dcConfig, "centralRegistryUrls", centralRegistryUrls);
+        ApplicationInfoManager manager = mock(ApplicationInfoManager.class);
+        InstanceInfo info = mock(InstanceInfo.class);
+        when(manager.getInfo()).thenReturn(info);
+        when(info.getIPAddr()).thenReturn("127.0.0.1");
+        when(info.getDataCenterInfo()).thenReturn(new MyDataCenterInfo(DataCenterInfo.Name.MyOwn));
+        LeaseInfo leaseInfo = mock(LeaseInfo.class);
+        when(info.getLeaseInfo()).thenReturn(leaseInfo);
+        EurekaClientConfigBean bean = new EurekaClientConfigBean();
+        DiscoveryClientWrapper wrapper = dcConfig.additionalDiscoveryClientWrapper(manager, bean, null);
+        wrapper.shutdown();
+        assertEquals(2, wrapper.getDiscoveryClients().size());
+    }
+}