Skip to content

Commit

Permalink
Merge pull request cloudera#7 from abayer/master
Browse files Browse the repository at this point in the history
Hue and Oozie added, cut down on API host listing calls
  • Loading branch information
ggear committed Mar 26, 2013
2 parents daed701 + d31fd48 commit b864495
Show file tree
Hide file tree
Showing 8 changed files with 168 additions and 6 deletions.
35 changes: 31 additions & 4 deletions src/main/java/com/cloudera/whirr/cm/api/CmServerApi.java
Original file line number Diff line number Diff line change
Expand Up @@ -468,6 +468,15 @@ private ApiService buildServices(final CmServerCluster cluster, CmServerServiceT
apiServiceConfig.add(new ApiConfig("hdfs_service", cluster.getServiceName(CmServerServiceType.HDFS)));
apiServiceConfig.add(new ApiConfig("zookeeper_service", cluster.getServiceName(CmServerServiceType.ZOOKEEPER)));
break;
case HUE:
apiServiceConfig.add(new ApiConfig("hue_webhdfs", cluster.getServiceName(CmServerServiceType.HDFS_NAMENODE)));
apiServiceConfig.add(new ApiConfig("hive_service", cluster.getServiceName(CmServerServiceType.HIVE)));
apiServiceConfig.add(new ApiConfig("oozie_service", cluster.getServiceName(CmServerServiceType.OOZIE)));
break;
case OOZIE:
apiServiceConfig.add(new ApiConfig("mapreduce_yarn_service", cluster
.getServiceName(CmServerServiceType.MAPREDUCE)));
break;
case HIVE:
apiServiceConfig.add(new ApiConfig("mapreduce_yarn_service", cluster
.getServiceName(CmServerServiceType.MAPREDUCE)));
Expand Down Expand Up @@ -515,9 +524,9 @@ private ApiService buildServices(final CmServerCluster cluster, CmServerServiceT
apiRoleConfigGroup.setBase(false);
apiRoleConfigGroups.add(apiRoleConfigGroup);
}

Set<ApiHost> hosts = hosts();
for (CmServerService subService : cluster.getServices(type)) {
String hostId = getHostIdForInstance(subService.getHost());
String hostId = getHostIdForInstance(subService.getHost(), hosts);
ApiRole apiRole = new ApiRole();
apiRole.setName(subService.getName());
apiRole.setType(subService.getType().getLabel());
Expand All @@ -532,9 +541,8 @@ private ApiService buildServices(final CmServerCluster cluster, CmServerServiceT
return apiService;
}

private String getHostIdForInstance(Instance instance) throws IOException, CmServerApiException {
private String getHostIdForInstance(Instance instance, Set<ApiHost> hosts) throws IOException {
String hostId;
Set<ApiHost> hosts = hosts();

final String hostName = instance.getPublicHostName();
final String privateIp = instance.getPrivateIp();
Expand Down Expand Up @@ -576,6 +584,25 @@ private void initPreStartServices(final CmServerCluster cluster, CmServerService
execute(apiResourceRoot.getClustersResource().getServicesResource(getName(cluster))
.hiveCreateMetastoreDatabaseTablesCommand(cluster.getServiceName(CmServerServiceType.HIVE)));
break;
case HUE:
CmServerService hueService = cluster.getService(CmServerServiceType.HUE_SERVER);
if (hueService != null) {
ApiRoleNameList syncList = new ApiRoleNameList();
syncList.add(hueService.getName());
execute(
apiResourceRoot.getClustersResource().getServicesResource(getName(cluster))
.getRoleCommandsResource(cluster.getServiceName(CmServerServiceType.HUE)).syncHueDbCommand(syncList),
false);
}
execute(apiResourceRoot.getClustersResource().getServicesResource(getName(cluster))
.createBeeswaxWarehouseCommand(cluster.getServiceName(CmServerServiceType.HUE)));
break;
case OOZIE:
execute(apiResourceRoot.getClustersResource().getServicesResource(getName(cluster))
.installOozieShareLib(cluster.getServiceName(CmServerServiceType.OOZIE)));
execute(apiResourceRoot.getClustersResource().getServicesResource(getName(cluster))
.createOozieDb(cluster.getServiceName(CmServerServiceType.OOZIE)));
break;
case HBASE:
execute(apiResourceRoot.getClustersResource().getServicesResource(getName(cluster))
.createHBaseRootCommand(cluster.getServiceName(CmServerServiceType.HBASE)));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,14 @@ public enum CmServerServiceType {
HIVE(CLUSTER, "HIVE", CmServerServiceTypeRepository.CDH), HIVE_METASTORE(HIVE, "HIVEMETASTORE",
CmServerServiceTypeRepository.CDH),

// Hue
HUE(CLUSTER, "HUE", CmServerServiceTypeRepository.CDH), HUE_SERVER(HUE, "HUE_SERVER",
CmServerServiceTypeRepository.CDH), HUE_BEESWAX_SERVER(HUE, "BEESWAX_SERVER", CmServerServiceTypeRepository.CDH),

// Oozie
OOZIE(CLUSTER, "OOZIE", CmServerServiceTypeRepository.CDH), OOZIE_SERVER(OOZIE, "OOZIE_SERVER",
CmServerServiceTypeRepository.CDH),

// Impala
IMPALA(CLUSTER, "IMPALA", CmServerServiceTypeRepository.IMPALA), IMPALA_STATE_STORE(IMPALA, "STATESTORE",
CmServerServiceTypeRepository.IMPALA), IMPALA_DAEMON(IMPALA, "IMPALAD", CmServerServiceTypeRepository.IMPALA),
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
/**
* Licensed to Cloudera, Inc. under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Cloudera, Inc. licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.cloudera.whirr.cm.cdh;

import com.cloudera.whirr.cm.api.CmServerServiceType;

public class CmCdhHueBeeswaxServerHandler extends BaseHandlerCmCdh {

public static final String ROLE = "cmcdh-hue-beeswaxserver";
public static final CmServerServiceType TYPE = CmServerServiceType.HUE_BEESWAX_SERVER;

@Override
public String getRole() {
return ROLE;
}

@Override
public CmServerServiceType getType() {
return TYPE;
}

}
37 changes: 37 additions & 0 deletions src/main/java/com/cloudera/whirr/cm/cdh/CmCdhHueServerHandler.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
/**
* Licensed to Cloudera, Inc. under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Cloudera, Inc. licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.cloudera.whirr.cm.cdh;

import com.cloudera.whirr.cm.api.CmServerServiceType;

public class CmCdhHueServerHandler extends BaseHandlerCmCdh {

public static final String ROLE = "cmcdh-hue-server";
public static final CmServerServiceType TYPE = CmServerServiceType.HUE_SERVER;

@Override
public String getRole() {
return ROLE;
}

@Override
public CmServerServiceType getType() {
return TYPE;
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
/**
* Licensed to Cloudera, Inc. under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Cloudera, Inc. licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.cloudera.whirr.cm.cdh;

import com.cloudera.whirr.cm.api.CmServerServiceType;

public class CmCdhOozieServerHandler extends BaseHandlerCmCdh {

public static final String ROLE = "cmcdh-oozie-server";
public static final CmServerServiceType TYPE = CmServerServiceType.OOZIE_SERVER;

@Override
public String getRole() {
return ROLE;
}

@Override
public CmServerServiceType getType() {
return TYPE;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -37,3 +37,8 @@ com.cloudera.whirr.cm.cdh.CmCdhHBaseRegionServerHandler

com.cloudera.whirr.cm.cdh.CmCdhImpalaDaemonHandler
com.cloudera.whirr.cm.cdh.CmCdhImpalaStateStoreHandler

com.cloudera.whirr.cm.cdh.CmCdhHueServerHandler
com.cloudera.whirr.cm.cdh.CmCdhHueBeeswaxServerHandler

com.cloudera.whirr.cm.cdh.CmCdhOozieServerHandler
9 changes: 7 additions & 2 deletions src/test/java/com/cloudera/whirr/cm/CmServerHandlerTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,13 @@
import com.cloudera.whirr.cm.cdh.CmCdhHBaseMasterHandler;
import com.cloudera.whirr.cm.cdh.CmCdhHBaseRegionServerHandler;
import com.cloudera.whirr.cm.cdh.CmCdhHiveMetaStoreHandler;
import com.cloudera.whirr.cm.cdh.CmCdhHueServerHandler;
import com.cloudera.whirr.cm.cdh.CmCdhHueBeeswaxServerHandler;
import com.cloudera.whirr.cm.cdh.CmCdhImpalaDaemonHandler;
import com.cloudera.whirr.cm.cdh.CmCdhImpalaStateStoreHandler;
import com.cloudera.whirr.cm.cdh.CmCdhJobTrackerHandler;
import com.cloudera.whirr.cm.cdh.CmCdhNameNodeHandler;
import com.cloudera.whirr.cm.cdh.CmCdhOozieServerHandler;
import com.cloudera.whirr.cm.cdh.CmCdhSecondaryNameNodeHandler;
import com.cloudera.whirr.cm.cdh.CmCdhTaskTrackerHandler;
import com.cloudera.whirr.cm.cdh.CmCdhZookeeperServerHandler;
Expand Down Expand Up @@ -85,14 +88,16 @@ public void testNodesAndAgentsAndCluster() throws Exception {
.of("whirr.instance-templates",
"1 " + CmServerHandler.ROLE + "+" + CmAgentHandler.ROLE + ",1 " + CmAgentHandler.ROLE + "+"
+ CmCdhNameNodeHandler.ROLE + "+" + CmCdhSecondaryNameNodeHandler.ROLE + "+"
+ CmCdhHueServerHandler.ROLE + "+" + CmCdhHueBeeswaxServerHandler.ROLE + "+"
+ CmCdhJobTrackerHandler.ROLE + "+" + CmCdhHBaseMasterHandler.ROLE + "+"
+ CmCdhHiveMetaStoreHandler.ROLE + "+" + CmCdhImpalaStateStoreHandler.ROLE + ",3 "
+ CmCdhHiveMetaStoreHandler.ROLE + "+" + CmCdhImpalaStateStoreHandler.ROLE + "+"
+ CmCdhOozieServerHandler.ROLE + ",3 "
+ CmAgentHandler.ROLE + "+" + CmCdhDataNodeHandler.ROLE + "+" + CmCdhTaskTrackerHandler.ROLE + "+"
+ CmCdhZookeeperServerHandler.ROLE + "+" + CmCdhHBaseRegionServerHandler.ROLE + "+"
+ CmCdhImpalaDaemonHandler.ROLE,
BaseHandler.CONFIG_WHIRR_CM_PREFIX + "REMOTE_PARCEL_REPO_URLS",
"http://10.178.197.160/tmph3l7m2vv103/cloudera-repos/cdh4/parcels/4.2.0.10/\\,http://10.178.197.160/tmph3l7m2vv103/cloudera-repos/impala/parcels/0.6.109/"))));
Assert.assertEquals(6, BaseHandlerCmCdh.CmServerClusterSingleton.getInstance().getServiceTypes().size());
Assert.assertEquals(8, BaseHandlerCmCdh.CmServerClusterSingleton.getInstance().getServiceTypes().size());
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,12 @@ public static void provisionCluster() throws CmServerApiException {
cluster = new CmServerCluster();
cluster.add(new CmServerService(CmServerServiceType.HIVE_METASTORE, CLUSTER_TAG, "1",
dummyInstance(CDH_MASTER_HOST, CDH_MASTER_IP)));
cluster.add(new CmServerService(CmServerServiceType.HUE_SERVER, CLUSTER_TAG, "1",
dummyInstance(CDH_MASTER_HOST, CDH_MASTER_IP)));
cluster.add(new CmServerService(CmServerServiceType.HUE_BEESWAX_SERVER, CLUSTER_TAG, "1",
dummyInstance(CDH_MASTER_HOST, CDH_MASTER_IP)));
cluster.add(new CmServerService(CmServerServiceType.OOZIE_SERVER, CLUSTER_TAG, "1",
dummyInstance(CDH_MASTER_HOST, CDH_MASTER_IP)));
cluster.add(new CmServerService(CmServerServiceType.HBASE_MASTER, CLUSTER_TAG, "1",
dummyInstance(CDH_MASTER_HOST, CDH_MASTER_IP)));
cluster.add(new CmServerService(CmServerServiceType.HDFS_NAMENODE, CLUSTER_TAG, "1",
Expand Down

0 comments on commit b864495

Please sign in to comment.