KNOX-2990 - Using DerbyDatabaseTSS instead of AliasBasedTSS by default

In addition to the new implementation I deprecated the AliasBased, Zookeeper and JournalBased TSS implementations in 2.1.0.

gateway-provider-security-jwt/src/test/java/org/apache/knox/gateway/provider/federation/TokenIDAsHTTPBasicCredsFederationFilterTest.java

@@ -523,5 +523,10 @@ private Collection<KnoxToken> fetchTokens(String userName, boolean createdBy) {
           });
           return tokens;
         }
+
+        @Override
+        public boolean isMigrationTarget() {
+          return false;
+        }
     }
 }

gateway-server/pom.xml

@@ -215,6 +215,10 @@
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-text</artifactId>
+        </dependency>
         <dependency>
             <groupId>commons-net</groupId>
             <artifactId>commons-net</artifactId>
@@ -467,19 +471,16 @@
         <dependency>
             <groupId>org.apache.knox</groupId>
             <artifactId>gateway-shell</artifactId>
-            <scope>test</scope>
         </dependency>
 
         <dependency>
             <groupId>org.apache.derby</groupId>
             <artifactId>derby</artifactId>
-            <scope>test</scope>
         </dependency>
 
         <dependency>
             <groupId>org.apache.derby</groupId>
             <artifactId>derbynet</artifactId>
-            <scope>test</scope>
         </dependency>
 
         <dependency>

gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java

@@ -282,6 +282,11 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig {
 
   private static final String TOKEN_STATE_SERVER_MANAGED = GATEWAY_CONFIG_FILE_PREFIX + ".knox.token.exp.server-managed";
   private static final String USERS_CAN_SEE_ALL_TOKENS = GATEWAY_CONFIG_FILE_PREFIX + ".knox.token.management.users.can.see.all.tokens";
+  private static final String SKIP_TOKEN_MIGRATION= GATEWAY_CONFIG_FILE_PREFIX + ".knox.token.migration.skip";
+  private static final String ARCHIVE_MIGRATED_TOKENS= GATEWAY_CONFIG_FILE_PREFIX + ".knox.token.migration.archive.tokens";
+  private static final String MIGRATE_EXPIRED_TOKENS= GATEWAY_CONFIG_FILE_PREFIX + ".knox.token.migration.include.expired.tokens";
+  private static final String TOKEN_MIGRATION_PRINTS_VERBOSE_MESSAGES= GATEWAY_CONFIG_FILE_PREFIX + ".knox.token.migration.verbose";
+  private static final String TOKEN_MIGRATION_PROGRESS_COUNT= GATEWAY_CONFIG_FILE_PREFIX + ".knox.token.migration.progress.count";
 
   private static final String CLOUDERA_MANAGER_DESCRIPTORS_MONITOR_INTERVAL = GATEWAY_CONFIG_FILE_PREFIX + ".cloudera.manager.descriptors.monitor.interval";
   private static final String CLOUDERA_MANAGER_ADVANCED_SERVICE_DISCOVERY_CONF_MONITOR_INTERVAL = GATEWAY_CONFIG_FILE_PREFIX + ".cloudera.manager.advanced.service.discovery.config.monitor.interval";
@@ -311,11 +316,11 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig {
   private static final String KNOX_INCOMING_XFORWARDED_ENABLED = "gateway.incoming.xforwarded.enabled";
 
   //Gateway Database related properties
-  private static final String GATEWAY_DATABASE_TYPE = GATEWAY_CONFIG_FILE_PREFIX + ".database.type";
-  private static final String GATEWAY_DATABASE_CONN_URL = GATEWAY_CONFIG_FILE_PREFIX + ".database.connection.url";
-  private static final String GATEWAY_DATABASE_HOST =  GATEWAY_CONFIG_FILE_PREFIX + ".database.host";
-  private static final String GATEWAY_DATABASE_PORT =  GATEWAY_CONFIG_FILE_PREFIX + ".database.port";
-  private static final String GATEWAY_DATABASE_NAME =  GATEWAY_CONFIG_FILE_PREFIX + ".database.name";
+  public static final String GATEWAY_DATABASE_TYPE = GATEWAY_CONFIG_FILE_PREFIX + ".database.type";
+  public static final String GATEWAY_DATABASE_CONN_URL = GATEWAY_CONFIG_FILE_PREFIX + ".database.connection.url";
+  public static final String GATEWAY_DATABASE_HOST =  GATEWAY_CONFIG_FILE_PREFIX + ".database.host";
+  public static final String GATEWAY_DATABASE_PORT =  GATEWAY_CONFIG_FILE_PREFIX + ".database.port";
+  public static final String GATEWAY_DATABASE_NAME =  GATEWAY_CONFIG_FILE_PREFIX + ".database.name";
   private static final String GATEWAY_DATABASE_SSL_ENABLED =  GATEWAY_CONFIG_FILE_PREFIX + ".database.ssl.enabled";
   private static final String GATEWAY_DATABASE_VERIFY_SERVER_CERT =  GATEWAY_CONFIG_FILE_PREFIX + ".database.ssl.verify.server.cert";
   private static final String GATEWAY_DATABASE_TRUSTSTORE_FILE =  GATEWAY_CONFIG_FILE_PREFIX + ".database.ssl.truststore.file";
@@ -1512,4 +1517,29 @@ private Map<String, Collection<String>> getPathAliases(String qualifier) {
     return pathAliases;
   }
 
+  @Override
+  public boolean skipTokenMigration() {
+    return getBoolean(SKIP_TOKEN_MIGRATION, false);
+  }
+
+  @Override
+  public boolean archiveMigratedTokens() {
+    return getBoolean(ARCHIVE_MIGRATED_TOKENS, false);
+  }
+
+  @Override
+  public boolean migrateExpiredTokens() {
+    return getBoolean(MIGRATE_EXPIRED_TOKENS, false);
+  }
+
+  @Override
+  public boolean printVerboseTokenMigrationMessages() {
+    return getBoolean(TOKEN_MIGRATION_PRINTS_VERBOSE_MESSAGES, true);
+  }
+
+  @Override
+  public int getTokenMigrationProgressCount() {
+    return getInt(TOKEN_MIGRATION_PROGRESS_COUNT, 10);
+  }
+
 }

gateway-server/src/main/java/org/apache/knox/gateway/services/CLIGatewayServices.java

@@ -53,6 +53,8 @@ public void init(GatewayConfig config, Map<String,String> options) throws Servic
     addService(ServiceType.CRYPTO_SERVICE, gatewayServiceFactory.create(this, ServiceType.CRYPTO_SERVICE, config, options));
 
     addService(ServiceType.TOPOLOGY_SERVICE, gatewayServiceFactory.create(this, ServiceType.TOPOLOGY_SERVICE, config, options));
+
+    addService(ServiceType.TOKEN_STATE_SERVICE, gatewayServiceFactory.create(this, ServiceType.TOKEN_STATE_SERVICE, config, options));
   }
 
   @Override

gateway-server/src/main/java/org/apache/knox/gateway/services/factory/TokenStateServiceFactory.java

@@ -32,6 +32,7 @@
 import org.apache.knox.gateway.services.ServiceType;
 import org.apache.knox.gateway.services.token.impl.AliasBasedTokenStateService;
 import org.apache.knox.gateway.services.token.impl.DefaultTokenStateService;
+import org.apache.knox.gateway.services.token.impl.DerbyDBTokenStateService;
 import org.apache.knox.gateway.services.token.impl.JDBCTokenStateService;
 import org.apache.knox.gateway.services.token.impl.JournalBasedTokenStateService;
 import org.apache.knox.gateway.services.token.impl.ZookeeperTokenStateService;
@@ -47,7 +48,7 @@ protected Service createService(GatewayServices gatewayServices, ServiceType ser
     if (shouldCreateService(implementation)) {
       if (matchesImplementation(implementation, DefaultTokenStateService.class)) {
         service = new DefaultTokenStateService();
-      } else if (matchesImplementation(implementation, AliasBasedTokenStateService.class, true)) {
+      } else if (matchesImplementation(implementation, AliasBasedTokenStateService.class)) {
         service = new AliasBasedTokenStateService();
         ((AliasBasedTokenStateService) service).setAliasService(getAliasService(gatewayServices));
       } else if (matchesImplementation(implementation, JournalBasedTokenStateService.class)) {
@@ -61,17 +62,32 @@ protected Service createService(GatewayServices gatewayServices, ServiceType ser
           service.init(gatewayConfig, options);
         } catch (ServiceLifecycleException e) {
           LOG.errorInitializingService(implementation, e.getMessage(), e);
-          service = new AliasBasedTokenStateService();
-          ((AliasBasedTokenStateService) service).setAliasService(getAliasService(gatewayServices));
+          service = useDerbyDatabaseTokenStateService(gatewayServices, gatewayConfig, options);
         }
+      } else if (matchesImplementation(implementation, DerbyDBTokenStateService.class, true)) {
+        service = useDerbyDatabaseTokenStateService(gatewayServices, gatewayConfig, options);
       }
 
-      logServiceUsage(isEmptyDefaultImplementation(implementation) ? AliasBasedTokenStateService.class.getName() : implementation, serviceType);
+      logServiceUsage(service.getClass().getName(), serviceType);
     }
 
     return service;
   }
 
+  private Service useDerbyDatabaseTokenStateService(GatewayServices gatewayServices, GatewayConfig gatewayConfig, Map<String, String> options) {
+    Service service;
+    try {
+      service = new DerbyDBTokenStateService();
+      ((DerbyDBTokenStateService) service).setAliasService(getAliasService(gatewayServices));
+      ((DerbyDBTokenStateService) service).setMasterService(getMasterService(gatewayServices));
+      service.init(gatewayConfig, options);
+    } catch (ServiceLifecycleException e) {
+      LOG.errorInitializingService(DerbyDBTokenStateService.class.getName(), e.getMessage(), e);
+      service = new DefaultTokenStateService();
+    }
+    return service;
+  }
+
   @Override
   protected ServiceType getServiceType() {
     return ServiceType.TOKEN_STATE_SERVICE;
@@ -80,6 +96,6 @@ protected ServiceType getServiceType() {
   @Override
   protected Collection<String> getKnownImplementations() {
     return unmodifiableList(asList(DefaultTokenStateService.class.getName(), AliasBasedTokenStateService.class.getName(), JournalBasedTokenStateService.class.getName(),
-        ZookeeperTokenStateService.class.getName(), JDBCTokenStateService.class.getName()));
+        ZookeeperTokenStateService.class.getName(), JDBCTokenStateService.class.getName(), DerbyDBTokenStateService.class.getName()));
   }
 }

gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/AliasBasedTokenStateService.java

@@ -52,6 +52,8 @@
 
 /**
  * A TokenStateService implementation based on the AliasService.
+ *
+ * @deprecated Since 2.1.0
  */
 public class AliasBasedTokenStateService extends AbstractPersistentTokenStateService implements TokenStatePeristerMonitorListener {
 

gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/DefaultTokenStateService.java

@@ -457,4 +457,9 @@ private Collection<KnoxToken> fetchTokens(String userName, boolean createdBy) {
     });
     return tokens;
   }
+
+  @Override
+  public boolean isMigrationTarget() {
+    return false;
+  }
 }

gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/DerbyDBTokenStateService.java

@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.knox.gateway.services.token.impl;
+
+import static org.apache.knox.gateway.config.impl.GatewayConfigImpl.GATEWAY_DATABASE_NAME;
+import static org.apache.knox.gateway.config.impl.GatewayConfigImpl.GATEWAY_DATABASE_TYPE;
+import static org.apache.knox.gateway.services.security.AliasService.NO_CLUSTER_NAME;
+import static org.apache.knox.gateway.util.JDBCUtils.DATABASE_PASSWORD_ALIAS_NAME;
+import static org.apache.knox.gateway.util.JDBCUtils.DATABASE_USER_ALIAS_NAME;
+import static org.apache.knox.gateway.util.JDBCUtils.DERBY_DB_TYPE;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.knox.gateway.config.GatewayConfig;
+import org.apache.knox.gateway.services.ServiceLifecycleException;
+import org.apache.knox.gateway.services.security.MasterService;
+import org.apache.knox.gateway.shell.jdbc.derby.DerbyDatabase;
+import org.apache.knox.gateway.util.FileUtils;
+
+public class DerbyDBTokenStateService extends JDBCTokenStateService {
+
+  public static final String DEFAULT_TOKEN_DB_USER_NAME = "knox";
+  public static final String DB_NAME = "tokens";
+
+  private DerbyDatabase derbyDatabase;
+  private Path derbyDatabaseFolder;
+  private MasterService masterService;
+
+  public void setMasterService(MasterService masterService) {
+    this.masterService = masterService;
+  }
+
+  @Override
+  public void init(GatewayConfig config, Map<String, String> options) throws ServiceLifecycleException {
+    try {
+      derbyDatabaseFolder = Paths.get(config.getGatewaySecurityDir(), DB_NAME);
+      startDerby();
+      ((Configuration) config).set(GATEWAY_DATABASE_TYPE, DERBY_DB_TYPE);
+      ((Configuration) config).set(GATEWAY_DATABASE_NAME, derbyDatabaseFolder.toString());
+      getAliasService().addAliasForCluster(NO_CLUSTER_NAME, DATABASE_USER_ALIAS_NAME, getDatabaseUserName());
+      getAliasService().addAliasForCluster(NO_CLUSTER_NAME, DATABASE_PASSWORD_ALIAS_NAME, getDatabasePassword());
+      super.init(config, options);
+
+      // we need the "x" permission too to be able to browse that folder (600 is not enough)
+      if (Files.exists(derbyDatabaseFolder)) {
+        FileUtils.chmod("700", derbyDatabaseFolder.toFile());
+      }
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw new ServiceLifecycleException("Error while initiating DerbyDBTokenStateService: " + e, e);
+    }
+  }
+
+  private void startDerby() throws Exception {
+    derbyDatabase = new DerbyDatabase(derbyDatabaseFolder.toString());
+    derbyDatabase.create();
+    TimeUnit.SECONDS.sleep(1); // give a bit of time for the server to start
+  }
+
+  private String getDatabasePassword() throws Exception {
+    final char[] dbPasswordAliasValue = getAliasService().getPasswordFromAliasForGateway(DATABASE_PASSWORD_ALIAS_NAME);
+    return dbPasswordAliasValue != null ? new String(dbPasswordAliasValue) : new String(masterService.getMasterSecret());
+  }
+
+  private String getDatabaseUserName() throws Exception {
+    final char[] dbUserAliasValue = getAliasService().getPasswordFromAliasForGateway(DATABASE_USER_ALIAS_NAME);
+    return dbUserAliasValue != null ? new String(dbUserAliasValue) : DEFAULT_TOKEN_DB_USER_NAME;
+  }
+
+  @Override
+  public void stop() throws ServiceLifecycleException {
+    try {
+      if (derbyDatabase != null) {
+        derbyDatabase.shutdown();
+      }
+    } catch (Exception e) {
+      throw new ServiceLifecycleException("Error while shutting down Derby Database", e);
+    }
+  }
+
+}

gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/JDBCTokenStateService.java

@@ -37,6 +37,7 @@
 import org.apache.knox.gateway.services.security.token.TokenStateServiceException;
 import org.apache.knox.gateway.services.security.token.UnknownTokenException;
 import org.apache.knox.gateway.util.JDBCUtils;
+import org.apache.knox.gateway.util.TokenMigrationTool;
 import org.apache.knox.gateway.util.Tokens;
 
 public class JDBCTokenStateService extends AbstractPersistentTokenStateService {
@@ -46,10 +47,20 @@ public class JDBCTokenStateService extends AbstractPersistentTokenStateService {
   private Lock initLock = new ReentrantLock(true);
   private Lock addMetadataLock = new ReentrantLock(true);
 
+  private boolean skipTokenMigration;
+  private boolean archiveMigratedTokens;
+  private boolean migrateExpiredTokens;
+  private boolean verboseTokenMigration;
+  private int tokenMigrationProgressCount;
+
   public void setAliasService(AliasService aliasService) {
     this.aliasService = aliasService;
   }
 
+  protected AliasService getAliasService() {
+    return aliasService;
+  }
+
   @Override
   public void init(GatewayConfig config, Map<String, String> options) throws ServiceLifecycleException {
     if (!initialized.get()) {
@@ -65,12 +76,33 @@ public void init(GatewayConfig config, Map<String, String> options) throws Servi
         } catch (Exception e) {
           throw new ServiceLifecycleException("Error while initiating JDBCTokenStateService: " + e, e);
         }
+
+        this.skipTokenMigration = config.skipTokenMigration();
+        this.archiveMigratedTokens = config.archiveMigratedTokens();
+        this.migrateExpiredTokens = config.migrateExpiredTokens();
+        this.verboseTokenMigration = config.printVerboseTokenMigrationMessages();
+        this.tokenMigrationProgressCount = config.getTokenMigrationProgressCount();
       } finally {
         initLock.unlock();
       }
     }
   }
 
+  @Override
+  public void start() throws ServiceLifecycleException {
+    super.start();
+    if (skipTokenMigration) {
+      log.skipTokenMigration();
+    } else {
+      final TokenMigrationTool tokenMigrationTool = new TokenMigrationTool(aliasService, this, null);
+      tokenMigrationTool.setArchiveMigratedTokens(archiveMigratedTokens);
+      tokenMigrationTool.setProgressCount(tokenMigrationProgressCount);
+      tokenMigrationTool.setVerbose(verboseTokenMigration);
+      tokenMigrationTool.setMigrateExpiredTokens(migrateExpiredTokens);
+      tokenMigrationTool.migrateTokensFromGatewayCredentialStore();
+    }
+  }
+
   @Override
   public void addToken(String tokenId, long issueTime, long expiration, long maxLifetimeDuration) {
     try {
@@ -323,4 +355,9 @@ public Collection<KnoxToken> getDoAsTokens(String createdBy) {
       return Collections.emptyList();
     }
   }
+
+  @Override
+  public boolean isMigrationTarget() {
+    return true;
+  }
 }

gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/JournalBasedTokenStateService.java

@@ -32,6 +32,9 @@
 import java.util.Map;
 import java.util.Set;
 
+/**
+ * @deprecated Since 2.1.0
+ */
 public class JournalBasedTokenStateService extends AbstractPersistentTokenStateService {
 
     private TokenStateJournal journal;

gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/TokenStateServiceMessages.java

@@ -261,4 +261,11 @@ public interface TokenStateServiceMessages {
 
   @Message(level = MessageLevel.ERROR, text = "An error occurred while fetching impersonation tokens for user {0} from the database : {1}")
   void errorFetchingDoAsTokensForUserFromDatabase(String userName, String errorMessage, @StackTrace(level = MessageLevel.DEBUG) Exception e);
+
+  @Message(level = MessageLevel.INFO, text = "Skipping token migration!")
+  void skipTokenMigration();
+
+  @Message(level = MessageLevel.INFO, text = "{0}")
+  void info(String message);
+
 }

gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/ZookeeperTokenStateService.java

@@ -38,6 +38,8 @@
  * A Zookeeper Token State Service is actually an Alias based TSS where the 'alias service' happens to be the 'zookeeper' implementation.
  * This means the only important thing that should be overridden here is the init method where the underlying alias service is configured
  * properly.
+ *
+ * @deprecated Since 2.1.0
  */
 public class ZookeeperTokenStateService extends AliasBasedTokenStateService implements RemoteTokenStateChangeListener {