[HOPS-140] removing containers log and checkpoint from the database w…

…hen the container is in a final state even if there is no quota utilization.
hopshadoop · Dec 5, 2017 · 8d48f80 · 8d48f80
1 parent db9eba4
commit 8d48f80
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 12 deletions.
diff --git a/.../main/java/org/apache/hadoop/yarn/server/resourcemanager/quota/ContainersLogsService.java b/.../main/java/org/apache/hadoop/yarn/server/resourcemanager/quota/ContainersLogsService.java
@@ -19,7 +19,6 @@
 import io.hops.metadata.common.entity.LongVariable;
 import io.hops.metadata.common.entity.Variable;
 import io.hops.metadata.hdfs.dal.VariableDataAccess;
-import io.hops.metadata.yarn.dal.ContainerStatusDataAccess;
 import io.hops.metadata.yarn.dal.quota.ContainersLogsDataAccess;
 import io.hops.metadata.yarn.dal.quota.PriceMultiplicatorDataAccess;
 import io.hops.metadata.yarn.dal.util.YARNOperationType;
@@ -63,7 +62,6 @@ public class ContainersLogsService extends CompositeService {
   private float currentMultiplicator; // This variable will be set/updated by the streaming service.
   private long multiplicatorPeirod;
 
-  ContainerStatusDataAccess containerStatusDA;
   ContainersLogsDataAccess containersLogsDA;
   VariableDataAccess variableDA;
 
@@ -112,8 +110,6 @@ public void serviceInit(Configuration conf) throws Exception {
     currentMultiplicator = 1;
 
     // Initialize DataAccesses
-    containerStatusDA = (ContainerStatusDataAccess) RMStorageFactory.
-            getDataAccess(ContainerStatusDataAccess.class);
     containersLogsDA = (ContainersLogsDataAccess) RMStorageFactory.
             getDataAccess(ContainersLogsDataAccess.class);
     variableDA = (VariableDataAccess) RMStorageFactory.getDataAccess(

diff --git a/...nager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/quota/QuotaService.java b/...nager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/quota/QuotaService.java
@@ -270,12 +270,12 @@ private void computeAndApplyChargeInt(
 
         } else {
           //The container has finished running
-          toBeRemovedContainersLogs.add((ContainerLog) containerLog);
-          if (checkpoint != containerLog.getStart()) {
+          toBeRemovedContainersLogs.add((ContainerLog) containerLog);         
+          if(containersCheckPoints.remove(containerLog.getContainerid())!=null){
             toBeRemovedContainerCheckPoint.add(new ContainerCheckPoint(
-                    containerLog.getContainerid()));
-            containersCheckPoints.remove(containerLog.getContainerid());
+                  containerLog.getContainerid()));
           }
+
           //** ProjectQuota charging**
           LOG.debug("charging project finished " + projectName
                   + " for container " + containerLog.getContainerid()
@@ -290,19 +290,24 @@ private void computeAndApplyChargeInt(
           chargeProjectDailyCost(chargedProjectsDailyCost, projectName,
                   user, curentDay, charge);
         }
-      } else if (checkpoint == containerLog.getStart()){
-        if (containerLog.getExitstatus() == ContainerExitStatus.CONTAINER_RUNNING_STATE) {
+      } else {
+        if (checkpoint == containerLog.getStart() && 
+                containerLog.getExitstatus() == ContainerExitStatus.CONTAINER_RUNNING_STATE) {
           //create a checkPoint at start to store multiplicator.
           ContainerCheckPoint newCheckpoint = new ContainerCheckPoint(
               containerLog.getContainerid(), containerLog.getStart(),
               currentMultiplicator);
           containersCheckPoints.put(containerLog.getContainerid(), newCheckpoint);
           toBePercistedContainerCheckPoint.add(newCheckpoint);
-        } else {
+        } else if(containerLog.getExitstatus() != ContainerExitStatus.CONTAINER_RUNNING_STATE) {
           //the container is not running remove it from db
           toBeRemovedContainersLogs.add((ContainerLog) containerLog);
+          if(containersCheckPoints.remove(containerLog.getContainerid())!=null){
+            toBeRemovedContainerCheckPoint.add(new ContainerCheckPoint(
+                  containerLog.getContainerid()));
+          }
         }
-      }
+      } 
     }
     // Delet the finished ContainersLogs
     ContainersLogsDataAccess csDA = (ContainersLogsDataAccess) RMStorageFactory.