Feature/worker error budget

deploy-service/common/src/main/java/com/pinterest/deployservice/metrics/MeterConstants.java

@@ -0,0 +1,11 @@
+package com.pinterest.deployservice.metrics;
+
+import static com.pinterest.teletraan.universal.metrics.micrometer.PinStatsNamingConvention.CUSTOM_NAME_PREFIX;
+
+public class MeterConstants {
+  public final static String ERROR_BUDGET_METRIC_NAME = CUSTOM_NAME_PREFIX + "error-budget.counters";
+  public final static String ERROR_BUDGET_TAG_NAME_METHOD_NAME = "method_name";
+  public final static String ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE = "response_type";
+  public final static String ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_SUCCESS = "success";
+  public final static String ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_FAILURE = "failure";
+}

deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/worker/AgentJanitor.java

@@ -32,8 +32,10 @@
 import com.pinterest.deployservice.bean.HostAgentBean;
 import com.pinterest.deployservice.bean.HostBean;
 import com.pinterest.deployservice.bean.HostState;
+import com.pinterest.deployservice.metrics.MeterConstants;
 import com.pinterest.deployservice.rodimus.RodimusManager;
 
+import io.micrometer.core.instrument.Counter;
 import io.micrometer.core.instrument.Metrics;
 
 /**
@@ -54,6 +56,8 @@ public class AgentJanitor extends SimpleAgentJanitor {
     private final int agentlessHostBatchSize = 300;
     private final AtomicInteger unreachableHostsCount;
     private final AtomicInteger staleHostsCount;
+    private final Counter errorBudgetSuccess;
+    private final Counter errorBudgetFailure;
     private long janitorStartTime;
 
     public AgentJanitor(ServiceContext serviceContext, int minStaleHostThresholdSeconds,
@@ -63,6 +67,16 @@ public AgentJanitor(ServiceContext serviceContext, int minStaleHostThresholdSeco
         maxLaunchLatencyThreshold = TimeUnit.SECONDS.toMillis(maxLaunchLatencyThresholdSeconds);
         unreachableHostsCount = Metrics.gauge("unreachable_hosts", new AtomicInteger(0));
         staleHostsCount = Metrics.gauge("stale_hosts", new AtomicInteger(0));
+
+        errorBudgetSuccess = Metrics.counter(MeterConstants.ERROR_BUDGET_METRIC_NAME,
+                MeterConstants.ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE,
+                MeterConstants.ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_SUCCESS,
+                MeterConstants.ERROR_BUDGET_TAG_NAME_METHOD_NAME, this.getClass().getSimpleName());
+
+        errorBudgetFailure = Metrics.counter(MeterConstants.ERROR_BUDGET_METRIC_NAME,
+                MeterConstants.ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE,
+                MeterConstants.ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_FAILURE,
+                MeterConstants.ERROR_BUDGET_TAG_NAME_METHOD_NAME, this.getClass().getSimpleName());
     }
 
     @Override
@@ -71,6 +85,7 @@ void processAllHosts() {
         processStaleHosts();
         determineStaleHostCandidates();
         cleanUpAgentlessHosts();
+        errorBudgetSuccess.increment();
     }
 
     private Set<String> getTerminatedHostsFromSource(List<String> staleHostIds) {
@@ -82,6 +97,7 @@ private Set<String> getTerminatedHostsFromSource(List<String> staleHostIds) {
                         .getTerminatedHosts(staleHostIds.subList(i, Math.min(i + batchSize, staleHostIds.size()))));
             } catch (Exception ex) {
                 LOG.error("Failed to get terminated hosts", ex);
+                errorBudgetFailure.increment();
             }
         }
         return terminatedHosts;
@@ -94,6 +110,7 @@ private Long getInstanceLaunchGracePeriod(String clusterName) {
                 launchGracePeriod = rodimusManager.getClusterInstanceLaunchGracePeriod(clusterName);
             } catch (Exception ex) {
                 LOG.error("failed to get launch grace period for cluster {}, exception: {}", clusterName, ex);
+                errorBudgetFailure.increment();
             }
         }
         return launchGracePeriod == null ? maxLaunchLatencyThreshold : TimeUnit.SECONDS.toMillis(launchGracePeriod);
@@ -113,6 +130,8 @@ private boolean isHostStale(HostAgentBean hostAgentBean) {
             hostBean = hostDAO.getHostsByHostId(hostAgentBean.getHost_id()).get(0);
         } catch (Exception ex) {
             LOG.error("failed to get host bean for ({}), {}", hostAgentBean, ex);
+            errorBudgetFailure.increment();
+
             return false;
         }
 
@@ -140,6 +159,7 @@ private Map<String, HostAgentBean> getStaleHostsMap(long minThreshold, long maxT
             }
         } catch (Exception ex) {
             LOG.error("failed to get stale hosts", ex);
+            errorBudgetFailure.increment();
             return staleHostMap;
         }
 
@@ -171,6 +191,7 @@ private void determineStaleHostCandidates() {
                 HostAgentBean host = unreachableHostsMap.get(unreachableId);
                 LOG.info("{} has unreachable host {}", host.getAuto_scaling_group(), host.getHost_id());
             }
+            errorBudgetSuccess.increment();
         }
         this.unreachableHostsCount.set(unreachableHostCount);
     }
@@ -195,6 +216,7 @@ private void processStaleHosts() {
                     LOG.warn("{}:{} is stale (not Pinging Teletraan), but might be running.",
                             hostAgent.getAuto_scaling_group(), hostAgent.getHost_id());
                     staleHostCount++;
+                    errorBudgetSuccess.increment();
                 } else {
                     LOG.debug("host {} is not stale", staleId);
                 }
@@ -217,6 +239,7 @@ private void cleanUpAgentlessHosts() {
             agentlessHosts = hostDAO.getStaleAgentlessHostIds(noUpdateSince, agentlessHostBatchSize);
         } catch (SQLException ex) {
             LOG.error("failed to get agentless hosts", ex);
+            errorBudgetFailure.increment();
             return;
         }
 
@@ -226,6 +249,7 @@ private void cleanUpAgentlessHosts() {
                 removeStaleHost(hostId);
             } else {
                 LOG.warn("Agentless host {} is stale but might be running", hostId);
+                errorBudgetSuccess.increment();
             }
         }
     }

deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/worker/AutoPromoter.java

@@ -27,6 +27,10 @@
 import com.pinterest.deployservice.dao.PromoteDAO;
 import com.pinterest.deployservice.dao.UtilDAO;
 import com.pinterest.deployservice.handler.DeployHandler;
+import com.pinterest.deployservice.metrics.MeterConstants;
+
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.Metrics;
 
 import com.google.common.base.Preconditions;
 import org.apache.commons.lang.StringUtils;
@@ -59,6 +63,8 @@ public class AutoPromoter implements Runnable {
     private BuildTagsManager buildTagsManager;
     private int bufferTimeMinutes;
     private final int maxCheckBuildsOrDeploys = 100;
+    private Counter errorBudgetSuccess;
+    private Counter errorBudgetFailure;
 
 
     public AutoPromoter(ServiceContext serviceContext) {
@@ -70,6 +76,14 @@ public AutoPromoter(ServiceContext serviceContext) {
         buildTagsManager = new BuildTagsManagerImpl(serviceContext.getTagDAO());
         deployHandler = new DeployHandler(serviceContext);
         bufferTimeMinutes = DEFAULT_BUFFER_TIME_MINUTE;
+
+        errorBudgetSuccess = Metrics.counter(MeterConstants.ERROR_BUDGET_METRIC_NAME,
+            MeterConstants.ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE, MeterConstants.ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_SUCCESS,
+            MeterConstants.ERROR_BUDGET_TAG_NAME_METHOD_NAME, this.getClass().getSimpleName());
+
+        errorBudgetFailure = Metrics.counter(MeterConstants.ERROR_BUDGET_METRIC_NAME,
+            MeterConstants.ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE, MeterConstants.ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_FAILURE,
+            MeterConstants.ERROR_BUDGET_TAG_NAME_METHOD_NAME, this.getClass().getSimpleName());
     }
 
     public AutoPromoter withBufferTimeMinutes(int bufferTime) {
@@ -82,16 +96,22 @@ void processBatch() throws Exception {
         List<String> envIds = promoteDAO.getAutoPromoteEnvIds();
         if (envIds.isEmpty()) {
             LOG.debug("AutoPromoter did not find any valid env to work on, exiting.");
+
+            errorBudgetSuccess.increment();
             return;
         }
         Collections.shuffle(envIds);
         for (String envId : envIds) {
             try {
                 LOG.debug("AutoPromoter chooses env {} to work on.", envId);
                 processOnce(envId);
+
+                errorBudgetSuccess.increment();
             } catch (Throwable t) {
                 // Catch all throwable so that subsequent job not suppressed
                 LOG.error("AutoPromoter failed to process {}, Exception: {}", envId, t);
+
+                errorBudgetFailure.increment();
             }
         }
       LOG.info("AutoPromoter processBatch finishes");
@@ -576,6 +596,8 @@ public void run() {
         } catch (Throwable t) {
             // Catch all throwable so that subsequent job not suppressed
             LOG.error("Failed to call AutoPromoter.", t);
+
+            errorBudgetFailure.increment();
         }
     }
 }

deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/worker/BuildJanitor.java

@@ -17,7 +17,9 @@
 
 import com.pinterest.deployservice.dao.BuildDAO;
 import com.pinterest.deployservice.dao.UtilDAO;
+import com.pinterest.deployservice.metrics.MeterConstants;
 import com.pinterest.teletraan.TeletraanServiceContext;
+
 import org.quartz.Job;
 import org.quartz.JobExecutionContext;
 import org.quartz.JobExecutionException;
@@ -29,6 +31,9 @@
 import java.util.Collections;
 import java.util.List;
 
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.Metrics;
+
 /**
  * Remove unused and old builds.
  * <p>
@@ -39,6 +44,8 @@
 public class BuildJanitor implements Job {
     private static final Logger LOG = LoggerFactory.getLogger(BuildJanitor.class);
     private static final long MILLIS_PER_DAY = 86400000;
+    private Counter errorBudgetSuccess;
+    private Counter errorBudgetFailure;
 
     public BuildJanitor() {
         // If using the Job interface, must keep constructor empty.
@@ -66,6 +73,9 @@ void processBuilds(TeletraanServiceContext workerContext) throws Exception {
                         LOG.info(String.format("Successfully removed builds: %s before %d milliseconds has %d.",                            buildName, timeThreshold, numToDelete));
                     } catch (Exception e) {
                         LOG.error("Failed to delete builds from tables.", e);
+
+                        errorBudgetFailure.increment();
+
                     } finally {
                         utilDAO.releaseLock(buildLockName, connection);
                         LOG.info(String.format("DB lock operation is successful: release lock %s", buildLockName));
@@ -84,10 +94,24 @@ public void execute(JobExecutionContext context) throws JobExecutionException {
             LOG.info("Start build janitor process...");
             SchedulerContext schedulerContext = context.getScheduler().getContext();
             TeletraanServiceContext workerContext = (TeletraanServiceContext) schedulerContext.get("serviceContext");
+
+            errorBudgetSuccess = Metrics.counter(MeterConstants.ERROR_BUDGET_METRIC_NAME,
+                MeterConstants.ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE, MeterConstants.ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_SUCCESS,
+                MeterConstants.ERROR_BUDGET_TAG_NAME_METHOD_NAME, this.getClass().getSimpleName());
+
+            errorBudgetFailure = Metrics.counter(MeterConstants.ERROR_BUDGET_METRIC_NAME,
+                MeterConstants.ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE, MeterConstants.ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_FAILURE,
+                MeterConstants.ERROR_BUDGET_TAG_NAME_METHOD_NAME, this.getClass().getSimpleName());
+
             processBuilds(workerContext);
             LOG.info("Stop build janitor process...");
+
+            errorBudgetSuccess.increment();
         } catch (Throwable t) {
             LOG.error("Failed to call build janitor.", t);
+
+            if(errorBudgetFailure != null)
+                errorBudgetFailure.increment();
         }
     }
 }

deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/worker/DeployJanitor.java

@@ -20,6 +20,8 @@
 import com.pinterest.deployservice.dao.DeployDAO;
 import com.pinterest.deployservice.dao.EnvironDAO;
 import com.pinterest.deployservice.dao.UtilDAO;
+import com.pinterest.deployservice.metrics.MeterConstants;
+
 import org.quartz.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -28,6 +30,9 @@
 import java.util.Collections;
 import java.util.List;
 
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.Metrics;
+
 /**
  * Removed unused/old deploys.
  */
@@ -38,6 +43,8 @@ public class DeployJanitor implements Job {
     private EnvironDAO environDAO;
     private DeployDAO deployDAO;
     private UtilDAO utilDAO;
+    private Counter errorBudgetSuccess;
+    private Counter errorBudgetFailure;
 
     public DeployJanitor() {
         // If using the Job interface, must keep constructor empty.
@@ -62,8 +69,12 @@ void processDeploys() throws Exception {
                         deployDAO.deleteUnusedDeploys(envId, timeThreshold, numToDelete);
                         LOG.info(String.format("Successfully removed deploys: %s before %d milliseconds has %d.",
                             envId, timeThreshold, numToDelete));
+
+                        errorBudgetSuccess.increment();
                     } catch (Exception e) {
                         LOG.error("Failed to delete builds from tables.", e);
+
+                        errorBudgetFailure.increment();
                     } finally {
                         utilDAO.releaseLock(deployLockName, connection);
                         LOG.info(String.format("DB lock operation is successful: release lock %s", deployLockName));
@@ -79,10 +90,20 @@ void processDeploys() throws Exception {
     public void execute(JobExecutionContext context) throws JobExecutionException {
         SchedulerContext schedulerContext;
 
+        errorBudgetSuccess = Metrics.counter(MeterConstants.ERROR_BUDGET_METRIC_NAME,
+            MeterConstants.ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE, MeterConstants.ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_SUCCESS,
+            MeterConstants.ERROR_BUDGET_TAG_NAME_METHOD_NAME, this.getClass().getSimpleName());
+
+        errorBudgetFailure = Metrics.counter(MeterConstants.ERROR_BUDGET_METRIC_NAME,
+            MeterConstants.ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE, MeterConstants.ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_FAILURE,
+            MeterConstants.ERROR_BUDGET_TAG_NAME_METHOD_NAME, this.getClass().getSimpleName());
+
         try {
             schedulerContext = context.getScheduler().getContext();
         } catch (SchedulerException e) {
             LOG.error("Cannot retrive job context!", e);
+
+            errorBudgetFailure.increment();
             return;
         }
 
@@ -95,8 +116,12 @@ public void execute(JobExecutionContext context) throws JobExecutionException {
             LOG.info("Start deploy janitor process...");
             processDeploys();
             LOG.info("Stop deploy janitor process...");
+
+            errorBudgetSuccess.increment();
         } catch (Throwable t) {
             LOG.error("Failed to call deploy janitor.", t);
+
+            errorBudgetFailure.increment();
         }
     }
 }