YARN-11582. Improve WebUI diagnosticMessage to show AM Container reso…

…urce request size.
apache · Oct 6, 2023 · 4089dfb · 4089dfb
1 parent b871805
commit 4089dfb
Show file tree

Hide file tree

Showing 2 changed files with 80 additions and 0 deletions.
diff --git a/...org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/...org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -1026,6 +1026,9 @@ protected void getActivedAppDiagnosticMessage(
         .append(appAMNodePartitionName.isEmpty()
             ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : appAMNodePartitionName)
         .append(" ; ")
+        .append("AM Resource Request = ")
+        .append(getAMResource(appAMNodePartitionName))
+        .append(" ; ")
         .append("Partition Resource = ")
         .append(rmContext.getNodeLabelManager()
             .getResourceByLabel(appAMNodePartitionName, Resources.none()))

diff --git a/...doop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java b/...doop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
@@ -1027,4 +1027,81 @@ public void testAMLimitByAllResources() throws Exception {
     rm.close();
 
   }
+
+  @Test(timeout = 120000)
+  public void testDiagnosticWhenAMActivated() throws Exception {
+    /*
+     * Test Case:
+     * Verify AM resource limit per partition level and per queue level. So
+     * we use 2 queues to verify this case.
+     * Queue a1 supports labels (x,y). Configure am-resource-limit as 0.2 (x)
+     * Queue c1 supports default label. Configure am-resource-limit as 0.2
+     *
+     * Queue a1 for label X can only support 2Gb AM resource.
+     * Queue c1 (empty label) can support 2Gb AM resource.
+     *
+     * Verify at least one AM is launched, and AM resources should not go more
+     * than 2GB in each queue.
+     */
+
+    simpleNodeLabelMappingToManager();
+    CapacitySchedulerConfiguration config = (CapacitySchedulerConfiguration)
+         TestUtils.getConfigurationWithQueueLabels(conf);
+
+    // After getting queue conf, configure AM resource percent for Queue a1
+    // as 0.2 (Label X) and for Queue c1 as 0.2 (Empty Label)
+    final String a1 = CapacitySchedulerConfiguration.ROOT + ".a" + ".a1";
+    final String c1 = CapacitySchedulerConfiguration.ROOT + ".c" + ".c1";
+    config.setMaximumAMResourcePercentPerPartition(a1, "x", 0.2f);
+    config.setMaximumApplicationMasterResourcePerQueuePercent(c1, 0.2f);
+
+    // Now inject node label manager with this updated config
+    MockRM rm = new MockRM(config) {
+      @Override
+      public RMNodeLabelsManager createNodeLabelManager() {
+        return mgr;
+      }
+    };
+
+    rm.getRMContext().setNodeLabelManager(mgr);
+    rm.start();
+    rm.registerNode("h1:1234", 10 * GB); // label = x
+    rm.registerNode("h2:1234", 10 * GB); // label = y
+    rm.registerNode("h3:1234", 10 * GB); // label = <empty>
+
+    // Submit app1 with 1Gb AM resource to Queue a1 for label X
+    MockRMAppSubmissionData data1 =
+         MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
+             .withAppName("app")
+             .withUser("user")
+             .withAcls(null)
+             .withQueue("a1")
+             .withAmLabel("x")
+             .build();
+    RMApp app1 = MockRMAppSubmitter.submit(rm, data1);
+
+    // Submit app2 with 1Gb AM resource to Queue a1 for label X
+    MockRMAppSubmissionData data2 =
+         MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
+             .withAppName("app")
+             .withUser("user")
+             .withAcls(null)
+             .withQueue("a1")
+             .withAmLabel("x")
+             .build();
+    RMApp app2 = MockRMAppSubmitter.submit(rm, data2);
+
+    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+    LeafQueue leafQueue = (LeafQueue) cs.getQueue("a1");
+    Assert.assertNotNull(leafQueue);
+
+    // Only one AM will be activated here and second AM will be still pending.
+    Assert.assertEquals(2, leafQueue.getNumActiveApplications());
+    String activatedDiagnostics="AM Resource Request = ";
+    Assert.assertTrue("still doesn't show AMResource When Activated", app1.getDiagnostics()
+         .toString().contains(activatedDiagnostics));
+    Assert.assertTrue("still doesn't show AMResource When Activated", app2.getDiagnostics()
+         .toString().contains(activatedDiagnostics));
+    rm.close();
+  }
 }