Refactor error budget to use factory (#1393)

commit-id:cc3ae706
pinterest · Feb 2, 2024 · acc89d5 · acc89d5
1 parent 443a627
commit acc89d5
Show file tree

Hide file tree

Showing 10 changed files with 68 additions and 94 deletions.
diff --git a/deploy-service/common/src/main/java/com/pinterest/deployservice/metrics/MeterConstants.java b/deploy-service/common/src/main/java/com/pinterest/deployservice/metrics/MeterConstants.java
diff --git a/...y-service/teletraanservice/src/main/java/com/pinterest/teletraan/worker/AgentJanitor.java b/...y-service/teletraanservice/src/main/java/com/pinterest/teletraan/worker/AgentJanitor.java
@@ -32,8 +32,8 @@
 import com.pinterest.deployservice.bean.HostAgentBean;
 import com.pinterest.deployservice.bean.HostBean;
 import com.pinterest.deployservice.bean.HostState;
-import com.pinterest.deployservice.metrics.MeterConstants;
 import com.pinterest.deployservice.rodimus.RodimusManager;
+import com.pinterest.teletraan.universal.metrics.ErrorBudgetCounterFactory;
 
 import io.micrometer.core.instrument.Counter;
 import io.micrometer.core.instrument.Metrics;
@@ -67,16 +67,9 @@ public AgentJanitor(ServiceContext serviceContext, int minStaleHostThresholdSeco
         maxLaunchLatencyThreshold = TimeUnit.SECONDS.toMillis(maxLaunchLatencyThresholdSeconds);
         unreachableHostsCount = Metrics.gauge("unreachable_hosts", new AtomicInteger(0));
         staleHostsCount = Metrics.gauge("stale_hosts", new AtomicInteger(0));
-
-        errorBudgetSuccess = Metrics.counter(MeterConstants.ERROR_BUDGET_METRIC_NAME,
-                MeterConstants.ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE,
-                MeterConstants.ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_SUCCESS,
-                MeterConstants.ERROR_BUDGET_TAG_NAME_METHOD_NAME, this.getClass().getSimpleName());
 
-        errorBudgetFailure = Metrics.counter(MeterConstants.ERROR_BUDGET_METRIC_NAME,
-                MeterConstants.ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE,
-                MeterConstants.ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_FAILURE,
-                MeterConstants.ERROR_BUDGET_TAG_NAME_METHOD_NAME, this.getClass().getSimpleName());
+        errorBudgetSuccess = ErrorBudgetCounterFactory.createSuccessCounter(this.getClass().getSimpleName());
+        errorBudgetFailure = ErrorBudgetCounterFactory.createFailureCounter(this.getClass().getSimpleName());
     }
 
     @Override

diff --git a/...y-service/teletraanservice/src/main/java/com/pinterest/teletraan/worker/AutoPromoter.java b/...y-service/teletraanservice/src/main/java/com/pinterest/teletraan/worker/AutoPromoter.java
@@ -27,10 +27,9 @@
 import com.pinterest.deployservice.dao.PromoteDAO;
 import com.pinterest.deployservice.dao.UtilDAO;
 import com.pinterest.deployservice.handler.DeployHandler;
-import com.pinterest.deployservice.metrics.MeterConstants;
+import com.pinterest.teletraan.universal.metrics.ErrorBudgetCounterFactory;
 
 import io.micrometer.core.instrument.Counter;
-import io.micrometer.core.instrument.Metrics;
 
 import com.google.common.base.Preconditions;
 import org.apache.commons.lang.StringUtils;
@@ -77,13 +76,8 @@ public AutoPromoter(ServiceContext serviceContext) {
         deployHandler = new DeployHandler(serviceContext);
         bufferTimeMinutes = DEFAULT_BUFFER_TIME_MINUTE;
 
-        errorBudgetSuccess = Metrics.counter(MeterConstants.ERROR_BUDGET_METRIC_NAME,
-            MeterConstants.ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE, MeterConstants.ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_SUCCESS,
-            MeterConstants.ERROR_BUDGET_TAG_NAME_METHOD_NAME, this.getClass().getSimpleName());
-
-        errorBudgetFailure = Metrics.counter(MeterConstants.ERROR_BUDGET_METRIC_NAME,
-            MeterConstants.ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE, MeterConstants.ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_FAILURE,
-            MeterConstants.ERROR_BUDGET_TAG_NAME_METHOD_NAME, this.getClass().getSimpleName());
+        errorBudgetSuccess = ErrorBudgetCounterFactory.createSuccessCounter(this.getClass().getSimpleName());
+        errorBudgetFailure = ErrorBudgetCounterFactory.createFailureCounter(this.getClass().getSimpleName());
     }
 
     public AutoPromoter withBufferTimeMinutes(int bufferTime) {
@@ -596,7 +590,7 @@ public void run() {
         } catch (Throwable t) {
             // Catch all throwable so that subsequent job not suppressed
             LOG.error("Failed to call AutoPromoter.", t);
-            
+
             errorBudgetFailure.increment();
         }
     }

diff --git a/...y-service/teletraanservice/src/main/java/com/pinterest/teletraan/worker/BuildJanitor.java b/...y-service/teletraanservice/src/main/java/com/pinterest/teletraan/worker/BuildJanitor.java
@@ -4,9 +4,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- *  
+ *
  *     http://www.apache.org/licenses/LICENSE-2.0
- *    
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,8 +17,8 @@
 
 import com.pinterest.deployservice.dao.BuildDAO;
 import com.pinterest.deployservice.dao.UtilDAO;
-import com.pinterest.deployservice.metrics.MeterConstants;
 import com.pinterest.teletraan.TeletraanServiceContext;
+import com.pinterest.teletraan.universal.metrics.ErrorBudgetCounterFactory;
 
 import org.quartz.Job;
 import org.quartz.JobExecutionContext;
@@ -32,7 +32,6 @@
 import java.util.List;
 
 import io.micrometer.core.instrument.Counter;
-import io.micrometer.core.instrument.Metrics;
 
 /**
  * Remove unused and old builds.
@@ -94,14 +93,9 @@ public void execute(JobExecutionContext context) throws JobExecutionException {
             LOG.info("Start build janitor process...");
             SchedulerContext schedulerContext = context.getScheduler().getContext();
             TeletraanServiceContext workerContext = (TeletraanServiceContext) schedulerContext.get("serviceContext");
-
-            errorBudgetSuccess = Metrics.counter(MeterConstants.ERROR_BUDGET_METRIC_NAME,
-                MeterConstants.ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE, MeterConstants.ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_SUCCESS,
-                MeterConstants.ERROR_BUDGET_TAG_NAME_METHOD_NAME, this.getClass().getSimpleName());
-
-            errorBudgetFailure = Metrics.counter(MeterConstants.ERROR_BUDGET_METRIC_NAME,
-                MeterConstants.ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE, MeterConstants.ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_FAILURE,
-                MeterConstants.ERROR_BUDGET_TAG_NAME_METHOD_NAME, this.getClass().getSimpleName());
+
+            errorBudgetSuccess = ErrorBudgetCounterFactory.createSuccessCounter(this.getClass().getSimpleName());
+            errorBudgetFailure = ErrorBudgetCounterFactory.createFailureCounter(this.getClass().getSimpleName());
 
             processBuilds(workerContext);
             LOG.info("Stop build janitor process...");

diff --git a/...-service/teletraanservice/src/main/java/com/pinterest/teletraan/worker/DeployJanitor.java b/...-service/teletraanservice/src/main/java/com/pinterest/teletraan/worker/DeployJanitor.java
@@ -4,9 +4,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- *  
+ *
  *     http://www.apache.org/licenses/LICENSE-2.0
- *    
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -20,7 +20,7 @@
 import com.pinterest.deployservice.dao.DeployDAO;
 import com.pinterest.deployservice.dao.EnvironDAO;
 import com.pinterest.deployservice.dao.UtilDAO;
-import com.pinterest.deployservice.metrics.MeterConstants;
+import com.pinterest.teletraan.universal.metrics.ErrorBudgetCounterFactory;
 
 import org.quartz.*;
 import org.slf4j.Logger;
@@ -31,7 +31,6 @@
 import java.util.List;
 
 import io.micrometer.core.instrument.Counter;
-import io.micrometer.core.instrument.Metrics;
 
 /**
  * Removed unused/old deploys.
@@ -90,13 +89,8 @@ void processDeploys() throws Exception {
     public void execute(JobExecutionContext context) throws JobExecutionException {
         SchedulerContext schedulerContext;
 
-        errorBudgetSuccess = Metrics.counter(MeterConstants.ERROR_BUDGET_METRIC_NAME,
-            MeterConstants.ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE, MeterConstants.ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_SUCCESS,
-            MeterConstants.ERROR_BUDGET_TAG_NAME_METHOD_NAME, this.getClass().getSimpleName());
-
-        errorBudgetFailure = Metrics.counter(MeterConstants.ERROR_BUDGET_METRIC_NAME,
-            MeterConstants.ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE, MeterConstants.ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_FAILURE,
-            MeterConstants.ERROR_BUDGET_TAG_NAME_METHOD_NAME, this.getClass().getSimpleName());
+        errorBudgetSuccess = ErrorBudgetCounterFactory.createSuccessCounter(this.getClass().getSimpleName());
+        errorBudgetFailure = ErrorBudgetCounterFactory.createFailureCounter(this.getClass().getSimpleName());
 
         try {
             schedulerContext = context.getScheduler().getContext();

diff --git a/...ervice/teletraanservice/src/main/java/com/pinterest/teletraan/worker/DeployTagWorker.java b/...ervice/teletraanservice/src/main/java/com/pinterest/teletraan/worker/DeployTagWorker.java
@@ -4,8 +4,8 @@
 import com.pinterest.deployservice.bean.*;
 import com.pinterest.deployservice.dao.*;
 import com.pinterest.deployservice.db.DatabaseUtil;
-import com.pinterest.deployservice.metrics.MeterConstants;
 import com.pinterest.deployservice.rodimus.RodimusManager;
+import com.pinterest.teletraan.universal.metrics.ErrorBudgetCounterFactory;
 
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.TransformerUtils;
@@ -19,7 +19,6 @@
 import java.util.*;
 
 import io.micrometer.core.instrument.Counter;
-import io.micrometer.core.instrument.Metrics;
 
 public class DeployTagWorker implements Runnable {
     private static final Logger LOG = LoggerFactory.getLogger(DeployTagWorker.class);
@@ -44,13 +43,8 @@ public DeployTagWorker(ServiceContext serviceContext) {
         dataSource = serviceContext.getDataSource();
         utilDAO = serviceContext.getUtilDAO();
 
-        errorBudgetSuccess = Metrics.counter(MeterConstants.ERROR_BUDGET_METRIC_NAME,
-            MeterConstants.ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE, MeterConstants.ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_SUCCESS,
-            MeterConstants.ERROR_BUDGET_TAG_NAME_METHOD_NAME, this.getClass().getSimpleName());
-
-        errorBudgetFailure = Metrics.counter(MeterConstants.ERROR_BUDGET_METRIC_NAME,
-            MeterConstants.ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE, MeterConstants.ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_FAILURE,
-            MeterConstants.ERROR_BUDGET_TAG_NAME_METHOD_NAME, this.getClass().getSimpleName());
+        errorBudgetSuccess = ErrorBudgetCounterFactory.createSuccessCounter(this.getClass().getSimpleName());
+        errorBudgetFailure = ErrorBudgetCounterFactory.createFailureCounter(this.getClass().getSimpleName());
     }
 
 

diff --git a/...service/teletraanservice/src/main/java/com/pinterest/teletraan/worker/HostTerminator.java b/...service/teletraanservice/src/main/java/com/pinterest/teletraan/worker/HostTerminator.java
@@ -27,9 +27,8 @@
 import com.pinterest.deployservice.dao.HostDAO;
 import com.pinterest.deployservice.dao.UtilDAO;
 import com.pinterest.deployservice.rodimus.RodimusManager;
-
+import com.pinterest.teletraan.universal.metrics.ErrorBudgetCounterFactory;
 import com.pinterest.deployservice.handler.HostHandler;
-import com.pinterest.deployservice.metrics.MeterConstants;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -38,7 +37,6 @@
 import java.util.*;
 
 import io.micrometer.core.instrument.Counter;
-import io.micrometer.core.instrument.Metrics;
 
 public class HostTerminator implements Runnable {
     private static final Logger LOG = LoggerFactory.getLogger(HostTerminator.class);
@@ -59,13 +57,8 @@ public HostTerminator(ServiceContext serviceContext) {
         hostAgentDAO = serviceContext.getHostAgentDAO();
         hostHandler = new HostHandler(serviceContext);
 
-        errorBudgetSuccess = Metrics.counter(MeterConstants.ERROR_BUDGET_METRIC_NAME,
-            MeterConstants.ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE, MeterConstants.ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_SUCCESS,
-            MeterConstants.ERROR_BUDGET_TAG_NAME_METHOD_NAME, this.getClass().getSimpleName());
-
-        errorBudgetFailure = Metrics.counter(MeterConstants.ERROR_BUDGET_METRIC_NAME,
-            MeterConstants.ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE, MeterConstants.ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_FAILURE,
-            MeterConstants.ERROR_BUDGET_TAG_NAME_METHOD_NAME, this.getClass().getSimpleName());
+        errorBudgetSuccess = ErrorBudgetCounterFactory.createSuccessCounter(this.getClass().getSimpleName());
+        errorBudgetFailure = ErrorBudgetCounterFactory.createFailureCounter(this.getClass().getSimpleName());
     }
 
     private void terminateHost(HostBean host) throws Exception {

diff --git a/...eletraanservice/src/main/java/com/pinterest/teletraan/worker/HotfixStateTransitioner.java b/...eletraanservice/src/main/java/com/pinterest/teletraan/worker/HotfixStateTransitioner.java
@@ -22,7 +22,7 @@
 import com.pinterest.deployservice.common.Jenkins;
 import com.pinterest.deployservice.dao.*;
 import com.pinterest.deployservice.handler.CommonHandler;
-import com.pinterest.deployservice.metrics.MeterConstants;
+import com.pinterest.teletraan.universal.metrics.ErrorBudgetCounterFactory;
 
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
@@ -35,7 +35,6 @@
 import java.util.Set;
 
 import io.micrometer.core.instrument.Counter;
-import io.micrometer.core.instrument.Metrics;
 
 /**
  * Check active deploys and push them into their final states
@@ -66,13 +65,8 @@ public HotfixStateTransitioner(ServiceContext serviceContext) {
         jenkinsUrl = serviceContext.getJenkinsUrl();
         jenkinsRemoteToken = serviceContext.getJenkinsRemoteToken();
 
-        errorBudgetSuccess = Metrics.counter(MeterConstants.ERROR_BUDGET_METRIC_NAME,
-            MeterConstants.ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE, MeterConstants.ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_SUCCESS,
-            MeterConstants.ERROR_BUDGET_TAG_NAME_METHOD_NAME, this.getClass().getSimpleName());
-
-        errorBudgetFailure = Metrics.counter(MeterConstants.ERROR_BUDGET_METRIC_NAME,
-            MeterConstants.ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE, MeterConstants.ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_FAILURE,
-            MeterConstants.ERROR_BUDGET_TAG_NAME_METHOD_NAME, this.getClass().getSimpleName());
+        errorBudgetSuccess = ErrorBudgetCounterFactory.createSuccessCounter(this.getClass().getSimpleName());
+        errorBudgetFailure = ErrorBudgetCounterFactory.createFailureCounter(this.getClass().getSimpleName());
     }
 
     void processBatch() throws Exception {

diff --git a/...vice/teletraanservice/src/main/java/com/pinterest/teletraan/worker/StateTransitioner.java b/...vice/teletraanservice/src/main/java/com/pinterest/teletraan/worker/StateTransitioner.java
@@ -4,9 +4,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- *  
+ *
  *     http://www.apache.org/licenses/LICENSE-2.0
- *    
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,7 +18,7 @@
 import com.pinterest.deployservice.ServiceContext;
 import com.pinterest.deployservice.dao.EnvironDAO;
 import com.pinterest.deployservice.handler.CommonHandler;
-import com.pinterest.deployservice.metrics.MeterConstants;
+import com.pinterest.teletraan.universal.metrics.ErrorBudgetCounterFactory;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -27,7 +27,6 @@
 import java.util.List;
 
 import io.micrometer.core.instrument.Counter;
-import io.micrometer.core.instrument.Metrics;
 
 /**
  * Check active deploys and transition them into final states
@@ -43,14 +42,8 @@ public class StateTransitioner implements Runnable {
     public StateTransitioner(ServiceContext serviceContext) {
         environDAO = serviceContext.getEnvironDAO();
         commonHandler = new CommonHandler(serviceContext);
-
-        errorBudgetSuccess = Metrics.counter(MeterConstants.ERROR_BUDGET_METRIC_NAME,
-            MeterConstants.ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE, MeterConstants.ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_SUCCESS,
-            MeterConstants.ERROR_BUDGET_TAG_NAME_METHOD_NAME, this.getClass().getSimpleName());
-
-        errorBudgetFailure = Metrics.counter(MeterConstants.ERROR_BUDGET_METRIC_NAME,
-            MeterConstants.ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE, MeterConstants.ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_FAILURE,
-            MeterConstants.ERROR_BUDGET_TAG_NAME_METHOD_NAME, this.getClass().getSimpleName());
+        errorBudgetSuccess = ErrorBudgetCounterFactory.createSuccessCounter(this.getClass().getSimpleName());
+        errorBudgetFailure = ErrorBudgetCounterFactory.createFailureCounter(this.getClass().getSimpleName());
     }
 
     void processBatch() throws Exception {

diff --git a/...al/src/main/java/com/pinterest/teletraan/universal/metrics/ErrorBudgetCounterFactory.java b/...al/src/main/java/com/pinterest/teletraan/universal/metrics/ErrorBudgetCounterFactory.java
@@ -0,0 +1,36 @@
+package com.pinterest.teletraan.universal.metrics;
+
+import static com.pinterest.teletraan.universal.metrics.micrometer.PinStatsNamingConvention.CUSTOM_NAME_PREFIX;
+
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.MeterRegistry;
+
+public class ErrorBudgetCounterFactory {
+
+  public final static String ERROR_BUDGET_METRIC_NAME = CUSTOM_NAME_PREFIX + "error-budget.counters";
+  public final static String ERROR_BUDGET_TAG_NAME_METHOD_NAME = "method_name";
+  public final static String ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE = "response_type";
+  public final static String ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_SUCCESS = "success";
+  public final static String ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_FAILURE = "failure";
+
+  private ErrorBudgetCounterFactory() {}
+
+  public static Counter createCounter(MeterRegistry registry, String methodName, boolean success) {
+    return Counter.builder(ERROR_BUDGET_METRIC_NAME)
+        .tag(
+            ERROR_BUDGET_TAG_NAME_RESPONSE_TYPE,
+            success
+                ? ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_SUCCESS
+                : ERROR_BUDGET_TAG_VALUE_RESPONSE_TYPE_FAILURE)
+        .tag(ERROR_BUDGET_TAG_NAME_METHOD_NAME, methodName)
+        .register(registry);
+  }
+
+  public static Counter createSuccessCounter(String methodName) {
+    return createCounter(io.micrometer.core.instrument.Metrics.globalRegistry, methodName, true);
+  }
+
+  public static Counter createFailureCounter(String methodName) {
+    return createCounter(io.micrometer.core.instrument.Metrics.globalRegistry, methodName, false);
+  }
+}