diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/CoordinatorGrpcService.java b/coordinator/src/main/java/org/apache/uniffle/coordinator/CoordinatorGrpcService.java index 33e08c4742..e494300013 100644 --- a/coordinator/src/main/java/org/apache/uniffle/coordinator/CoordinatorGrpcService.java +++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/CoordinatorGrpcService.java @@ -537,7 +537,8 @@ private ServerNode toServerNode(ShuffleServerHeartBeatRequest request) { request.getServerId().getJettyPort(), request.getStartTimeMs(), request.getVersion(), - request.getGitCommitId()); + request.getGitCommitId(), + request.getApplicationInfoList()); } /** diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/ServerNode.java b/coordinator/src/main/java/org/apache/uniffle/coordinator/ServerNode.java index ad992f0bc3..d43808380d 100644 --- a/coordinator/src/main/java/org/apache/uniffle/coordinator/ServerNode.java +++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/ServerNode.java @@ -17,14 +17,18 @@ package org.apache.uniffle.coordinator; +import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.apache.uniffle.common.ServerStatus; import org.apache.uniffle.common.storage.StorageInfo; +import org.apache.uniffle.proto.RssProtos; import org.apache.uniffle.proto.RssProtos.ShuffleServerId; public class ServerNode implements Comparable { @@ -46,6 +50,7 @@ public class ServerNode implements Comparable { private long startTime = -1; private String version; private String gitCommitId; + Map appIdToInfos; public ServerNode(String id) { this(id, "", 0, 0, 0, 0, 0, Sets.newHashSet(), ServerStatus.EXCLUDED); @@ -181,7 +186,8 @@ public ServerNode( jettyPort, startTime, "", - ""); + "", + Collections.EMPTY_LIST); } public ServerNode( @@ -199,7 +205,8 @@ public ServerNode( int jettyPort, long startTime, String version, - String gitCommitId) { + String gitCommitId, + List appInfos) { this.id = id; this.ip = ip; this.grpcPort = grpcPort; @@ -221,6 +228,8 @@ public ServerNode( this.startTime = startTime; this.version = version; this.gitCommitId = gitCommitId; + this.appIdToInfos = new ConcurrentHashMap<>(); + appInfos.forEach(appInfo -> appIdToInfos.put(appInfo.getAppId(), appInfo)); } public ShuffleServerId convertToGrpcProto() { @@ -375,4 +384,8 @@ public String getVersion() { public String getGitCommitId() { return gitCommitId; } + + public Map getAppIdToInfos() { + return appIdToInfos; + } } diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/ApplicationResource.java b/coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/ApplicationResource.java index c6cc60bef8..cc2a128d64 100644 --- a/coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/ApplicationResource.java +++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/ApplicationResource.java @@ -36,9 +36,12 @@ import org.apache.uniffle.common.web.resource.Response; import org.apache.uniffle.coordinator.AppInfo; import org.apache.uniffle.coordinator.ApplicationManager; +import org.apache.uniffle.coordinator.CoordinatorServer; +import org.apache.uniffle.coordinator.ServerNode; import org.apache.uniffle.coordinator.metric.CoordinatorMetrics; import org.apache.uniffle.coordinator.web.vo.AppInfoVO; import org.apache.uniffle.coordinator.web.vo.UserAppNumVO; +import org.apache.uniffle.proto.RssProtos; @Produces({MediaType.APPLICATION_JSON}) public class ApplicationResource extends BaseResource { @@ -84,19 +87,44 @@ public Response> getAppInfoList() { List userToAppList = new ArrayList<>(); Map> currentUserAndApp = getApplicationManager().getCurrentUserAndApp(); + List serverNodes = getCoordinatorServer().getClusterManager().list(); for (Map.Entry> userAppIdTimestampMap : currentUserAndApp.entrySet()) { for (Map.Entry appIdTimestampMap : userAppIdTimestampMap.getValue().entrySet()) { + String user = appIdTimestampMap.getKey(); AppInfo appInfo = appIdTimestampMap.getValue(); - userToAppList.add( + AppInfoVO appInfoVO = new AppInfoVO( - userAppIdTimestampMap.getKey(), + user, appInfo.getAppId(), appInfo.getUpdateTime(), appInfo.getRegistrationTime(), appInfo.getVersion(), - appInfo.getGitCommitId())); + appInfo.getGitCommitId(), + 0, + 0, + 0, + 0, + 0, + 0, + 0); + for (ServerNode server : serverNodes) { + Map appIdToInfos = server.getAppIdToInfos(); + if (appIdToInfos.containsKey(appInfoVO.getAppId())) { + RssProtos.ApplicationInfo app = appIdToInfos.get(appInfoVO.getAppId()); + appInfoVO.setPartitionNum(appInfoVO.getPartitionNum() + app.getPartitionNum()); + appInfoVO.setMemorySize(appInfoVO.getMemorySize() + app.getMemorySize()); + appInfoVO.setLocalFileNum(appInfoVO.getLocalFileNum() + app.getLocalFileNum()); + appInfoVO.setLocalTotalSize( + appInfoVO.getLocalTotalSize() + app.getLocalTotalSize()); + appInfoVO.setHadoopFileNum(appInfoVO.getHadoopFileNum() + app.getHadoopFileNum()); + appInfoVO.setHadoopTotalSize( + appInfoVO.getHadoopTotalSize() + app.getHadoopTotalSize()); + appInfoVO.setTotalSize(appInfoVO.getTotalSize() + app.getTotalSize()); + } + } + userToAppList.add(appInfoVO); } } // Display is inverted by the submission time of the application. @@ -109,4 +137,9 @@ private ApplicationManager getApplicationManager() { return (ApplicationManager) servletContext.getAttribute(ApplicationManager.class.getCanonicalName()); } + + private CoordinatorServer getCoordinatorServer() { + return (CoordinatorServer) + servletContext.getAttribute(CoordinatorServer.class.getCanonicalName()); + } } diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/web/vo/AppInfoVO.java b/coordinator/src/main/java/org/apache/uniffle/coordinator/web/vo/AppInfoVO.java index 2d6c8e1d0f..adb713638a 100644 --- a/coordinator/src/main/java/org/apache/uniffle/coordinator/web/vo/AppInfoVO.java +++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/web/vo/AppInfoVO.java @@ -31,6 +31,13 @@ public class AppInfoVO implements Comparable { private long registrationTime; private String version; private String gitCommitId; + private long partitionNum; + private long memorySize; + private long localFileNum; + private long localTotalSize; + private long hadoopFileNum; + private long hadoopTotalSize; + private long totalSize; @Override public int compareTo(AppInfoVO appInfoVO) { diff --git a/dashboard/src/main/webapp/src/pages/ApplicationPage.vue b/dashboard/src/main/webapp/src/pages/ApplicationPage.vue index 3de57355a4..0098902609 100644 --- a/dashboard/src/main/webapp/src/pages/ApplicationPage.vue +++ b/dashboard/src/main/webapp/src/pages/ApplicationPage.vue @@ -89,6 +89,37 @@ label="GitCommitId" min-width="180" /> + + + + + + + + + @@ -97,7 +128,7 @@