diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index e62bf104ae..7c9b1d1fa6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -29,6 +29,10 @@ import java.util.Iterator; import java.util.List; +import io.hops.security.HopsUtil; +import io.hops.security.SuperuserKeystoresLoader; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configuration; @@ -36,6 +40,9 @@ import org.apache.hadoop.http.HttpConfig.Policy; import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory; +import org.apache.hadoop.security.ssl.SSLFactory; +import org.apache.hadoop.security.ssl.X509SecurityMaterial; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.HAUtil; @@ -61,6 +68,8 @@ public class WebAppUtils { public static final String HTTPS_PREFIX = "https://"; public static final String HTTP_PREFIX = "http://"; + private static final Log LOG = LogFactory.getLog(WebAppUtils.class.getName()); + public static void setRMWebAppPort(Configuration conf, int port) { String hostname = getRMWebAppURLWithoutScheme(conf); hostname = @@ -428,9 +437,20 @@ public static HttpServer2.Builder loadSslConfiguration( if (conf != null) { sslConf.addResource(conf); } - boolean needsClientAuth = YarnConfiguration.YARN_SSL_CLIENT_HTTPS_NEED_AUTH_DEFAULT; + + String keystoreLocation = sslConf.get( + FileBasedKeyStoresFactory.resolvePropertyName(SSLFactory.Mode.SERVER, + FileBasedKeyStoresFactory.SSL_KEYSTORE_LOCATION_TPL_KEY), null); + builder.needsClientAuth(YarnConfiguration.YARN_SSL_CLIENT_HTTPS_NEED_AUTH_DEFAULT); + if (keystoreLocation != null) { + // ssl-server.xml has been populated, load material from there + return loadFromSSLServerConf(builder, sslConf); + } + return loadFromSuperUserLoader(builder, sslConf); + } + + private static HttpServer2.Builder loadFromSSLServerConf(HttpServer2.Builder builder, Configuration sslConf) { return builder - .needsClientAuth(needsClientAuth) .keyPassword(getPassword(sslConf, WEB_APP_KEY_PASSWORD_KEY)) .keyStore(sslConf.get("ssl.server.keystore.location"), getPassword(sslConf, WEB_APP_KEYSTORE_PASSWORD_KEY), @@ -438,8 +458,25 @@ public static HttpServer2.Builder loadSslConfiguration( .trustStore(sslConf.get("ssl.server.truststore.location"), getPassword(sslConf, WEB_APP_TRUSTSTORE_PASSWORD_KEY), sslConf.get("ssl.server.truststore.type", "jks")) - .excludeCiphers( - sslConf.get("ssl.server.exclude.cipher.list")); + .excludeCiphers(sslConf.get("ssl.server.exclude.cipher.list")); + } + + private static HttpServer2.Builder loadFromSuperUserLoader(HttpServer2.Builder builder, Configuration sslConf) { + try { + SuperuserKeystoresLoader loader = new SuperuserKeystoresLoader(sslConf); + X509SecurityMaterial material = loader.loadSuperUserMaterial(); + String password = HopsUtil.readCryptoMaterialPassword(material.getPasswdLocation().toFile()); + return builder + .keyPassword(password) + .keyStore(material.getKeyStoreLocation().toString(), password, + sslConf.get("ssl.server.keystore.type", "jks")) + .trustStore(material.getTrustStoreLocation().toString(), password, + sslConf.get("ssl.server.truststore.type", "jks")) + .excludeCiphers(sslConf.get("ssl.server.exclude.cipher.list")); + } catch (IOException ex) { + LOG.fatal("Could not system user x.509 material with SuperuserLoader", ex); + throw new RuntimeException(ex); + } } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java index d6f78b18c1..9de9b5ed30 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java @@ -24,16 +24,25 @@ import java.io.File; import java.io.IOException; import java.net.UnknownHostException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; + +import io.hops.security.SuperuserKeystoresLoader; +import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.Path; import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.http.HttpServer2.Builder; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.apache.hadoop.security.alias.JavaKeyStoreProvider; +import org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory; +import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -107,16 +116,18 @@ public void testGetPassword() throws Exception { } @Test - public void testLoadSslConfiguration() throws Exception { + public void testLoadFromSSLServerConf() throws Exception { Configuration conf = provisionCredentialsForSSL(); + conf.set(FileBasedKeyStoresFactory.resolvePropertyName(SSLFactory.Mode.SERVER, + FileBasedKeyStoresFactory.SSL_KEYSTORE_LOCATION_TPL_KEY), "kStore.jks"); + TestBuilder builder = (TestBuilder) new TestBuilder(); - builder = (TestBuilder) WebAppUtils.loadSslConfiguration( - builder, conf); + builder = (TestBuilder) WebAppUtils.loadSslConfiguration(builder, conf); String keypass = "keypass"; String storepass = "storepass"; - String trustpass = "trustpass"; + String trustpass = "trustpass"; // make sure we get the right passwords in the builder assertEquals(keypass, ((TestBuilder)builder).keypass); @@ -124,6 +135,30 @@ public void testLoadSslConfiguration() throws Exception { assertEquals(trustpass, ((TestBuilder)builder).truststorePassword); } + @Test + public void testLoadFromSuperUserLoader() throws Exception { + Configuration conf = provisionCredentialsForSSL(); + + String tmp = System.getProperty("test.build.data", "target/test-dir"); + conf.set(CommonConfigurationKeysPublic.HOPS_TLS_SUPER_MATERIAL_DIRECTORY, tmp); + + String password = "password"; + + SuperuserKeystoresLoader loader = new SuperuserKeystoresLoader(conf); + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + java.nio.file.Path passwd = Paths.get(tmp, loader.getSuperMaterialPasswdFilename(ugi.getUserName())); + FileUtils.writeStringToFile(passwd.toFile(), password, StandardCharsets.UTF_8); + + TestBuilder builder = (TestBuilder) new TestBuilder(); + + builder = (TestBuilder) WebAppUtils.loadSslConfiguration(builder, conf); + + // make sure we get the right passwords in the builder + assertEquals(password, ((TestBuilder)builder).keypass); + assertEquals(password, ((TestBuilder)builder).keystorePassword); + assertEquals(password, ((TestBuilder)builder).truststorePassword); + } + protected Configuration provisionCredentialsForSSL() throws IOException, Exception { File testDir = new File(System.getProperty("test.build.data",