Merge branch 'iroquebert/enum-over-two-dicts' of github.com:pinterest…

…/teletraan into iroquebert/enum-over-two-dicts

deploy-agent/deployd/agent.py

@@ -101,8 +101,8 @@ def first_run(self):
 
     def _send_deploy_status_stats(self, deploy_report):
         if not self._response.deployGoal or not deploy_report:
-            return 
-        
+            return
+
         tags = {'first_run': self.first_run}
         if self._response.deployGoal.deployStage:
             tags['deploy_stage'] = self._response.deployGoal.deployStage
@@ -112,10 +112,10 @@ def _send_deploy_status_stats(self, deploy_report):
             tags['stage_name'] = self._response.deployGoal.stageName
         if deploy_report.status_code:
             tags['status_code'] = deploy_report.status_code
-        if self._telefig_version: 
-            tags['telefig_version'] = self._telefig_version    
+        if self._telefig_version:
+            tags['telefig_version'] = self._telefig_version
         create_sc_increment('deployd.stats.deploy.status.sum', tags=tags)
-        
+
     def serve_build(self):
         """This is the main function of the ``DeployAgent``.
         """
@@ -145,7 +145,7 @@ def serve_build(self):
                                 if status.report.wait == None or status.report.wait > 5:
                                     status.report.redeploy = 0
                                     status.report.wait = 0
-                                else: 
+                                else:
                                     status.report.wait = status.report.wait + 1
                     else:
                         status.report.state = None
@@ -158,7 +158,7 @@ def serve_build(self):
             self._env_status.dump_envs(self._envs)
         # start to ping server to get the latest deploy goal
         self._response = self._client.send_reports(self._envs)
-        # we only need to send RESET once in one deploy-agent run 
+        # we only need to send RESET once in one deploy-agent run
         if len(self._envs) > 0:
             for status in self._envs.values():
                 if status.report.state == "RESET_BY_SYSTEM":
@@ -203,7 +203,7 @@ def serve_build(self):
 
             if PingStatus.PING_FAILED == self.update_deploy_status(deploy_report):
                 return
-                
+
             if deploy_report.status_code in [AgentStatus.AGENT_FAILED,
                                              AgentStatus.TOO_MANY_RETRY,
                                              AgentStatus.SCRIPT_TIMEOUT]:
@@ -284,6 +284,7 @@ def process_deploy(self, response):
             '''
             # pause stat_time_elapsed_internal so that external actions are not counted
             self.stat_time_elapsed_internal.pause()
+            log.info(f"The current deploy stage is: {curr_stage}")
             if curr_stage == DeployStage.DOWNLOADING:
                 return self._executor.run_cmd(self.get_download_script(deploy_goal=deploy_goal))
             elif curr_stage == DeployStage.STAGING:
@@ -522,9 +523,9 @@ def main():
     is_serverless_mode = AgentRunMode.is_serverless(args.mode)
     if args.daemon and is_serverless_mode:
         raise ValueError("daemon and serverless mode is mutually exclusive.")
-    
+
     config = Config(filenames=args.config_file)
-    
+
     if IS_PINTEREST:
         import pinlogger
 
@@ -535,10 +536,10 @@ def main():
         logging.basicConfig(filename=log_filename, level=config.get_log_level(),
                             format='%(asctime)s %(name)s:%(lineno)d %(levelname)s %(message)s')
 
-    if not check_prereqs(config): 
+    if not check_prereqs(config):
         log.warning("Deploy agent cannot start because the prerequisites on puppet did not meet.")
         sys.exit(0)
-        
+
     log.info("Start to run deploy-agent.")
     # timing stats - agent start time
     create_sc_timing('deployd.stats.internal.time_start_sec',

deploy-agent/tests/unit/deploy/client/test_base_client.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 
 import unittest
+from abc import ABCMeta, abstractmethod
+from future.utils import with_metaclass
 from tests import TestCase
 
 from deployd.client.base_client import BaseClient
@@ -28,6 +30,16 @@ class MyClient(BaseClient):
         with self.assertRaises(TypeError):
             myclient = MyClient()
 
+    def test_abc_equivalent_to_old(self):
+        """
+        Make sure that new changes to base client extend the original class
+        """
+        class OldBaseClient(with_metaclass(ABCMeta, object)):
+            @abstractmethod
+            def send_reports(self, env_reports=None):
+                pass
+        self.assertLessEqual(set(dir(OldBaseClient)), set(dir(BaseClient)))
+
 
 if __name__ == '__main__':
     unittest.main()

deploy-agent/tests/unit/deploy/client/test_client.py

@@ -0,0 +1,33 @@
+import mock
+import unittest
+from tests import TestCase
+
+from deployd.client.client import Client 
+from deployd.client.base_client import BaseClient
+from deployd.common.config import Config
+
+
+class TestClient(TestCase):
+    def test_extends_base_client(self):
+        self.assertTrue(issubclass(Client, BaseClient))
+
+    def test_no_config_provided(self):
+        with self.assertRaises(AttributeError):
+            Client()
+
+    def test_read_host_info(self):
+        client = Client(config=Config())
+        client._ec2_tags = {}
+        client._availability_zone = "us-east-1"
+        return_value: bool = client._read_host_info()
+        self.assertIsNotNone(client._hostname)
+        self.assertIsNotNone(client._ip)
+        self.assertTrue(return_value)
+
+    def test_read_host_info_no_ec2_tags_provided(self):
+        client = Client(config=Config())
+        with self.assertRaises(AttributeError):
+            client._read_host_info()
+
+if __name__ == '__main__':
+    unittest.main()

deploy-agent/tests/unit/deploy/client/test_serverless_client.py

@@ -108,6 +108,14 @@ def test_unknow_status_cause_retry(self):
     def test_deploy_stage_transitions(self):
         expected: dict[int, int] = dict([(i, i+1) for i in range(DeployStage.PRE_DOWNLOAD.value, DeployStage.SERVING_BUILD.value)])
         self.assertDictEqual(_DEPLOY_STAGE_TRANSITIONS, expected)
+
+    def test_create_response_last_deploy_stage(self):
+        report = self._new_report()
+        report.deployId = self.client._deploy_id
+        report.status = "SUCCEEDED"
+        report.deployStage = "SERVING_BUILD" 
+        response: PingResponse = self.client._create_response(report)
+        self.assertEqual(response.opCode, "NOOP")
 
 
 if __name__ == '__main__':

deploy-agent/tests/unit/deploy/common/test_config.py

@@ -20,7 +20,7 @@
 import tests
 
 from deployd.common.config import Config
-from deployd.common.types import DeployStatus, OpCode, DeployStage
+from deployd.common.types import DeployStatus, DeployType, OpCode, DeployStage
 from deployd.types.ping_response import PingResponse
 
 
@@ -58,6 +58,42 @@ def test_get_target(self):
         self.assertEqual(os.environ['COMPUTE_ENV_TYPE'], 'DEFAULT')
         self.assertEqual(self.config.get_target(), '/tmp/pinboard')
 
+    def test_init(self):
+        Config()
+
+        with self.assertRaises(TypeError):
+            Config(filenames=[
+                os.path.join(self.dirname, "test_file1.conf"),
+                os.path.join(self.dirname, "test_file2.conf"),
+                ])
+
+        open(os.path.join(self.dirname, "test_file1.conf"), "w")
+        Config(filenames=os.path.join(self.dirname, "test_file1.conf"))
+
+        os.remove(os.path.join(self.dirname, "test_file1.conf"))
+        with self.assertRaises(SystemExit), mock.patch('builtins.print') as print_mock:
+            Config(filenames=os.path.join(self.dirname, "test_file1.conf"))
+            print_mock.assert_called_once_with(f"Cannot find config files: {os.path.join(self.dirname, 'test_file1.conf')}")
+
+    def test_get_config_filename(self):
+        filename = os.path.join(self.dirname, "test_file1.conf")
+        open(os.path.join(self.dirname, "test_file1.conf"), "w")
+
+        config = Config(filenames=filename) 
+        self.assertEqual(config.get_config_filename(), filename)
+
+    def test_get_deploy_type_from_op_code(self):
+        config = Config()
+        self.assertEqual(config._get_deploy_type_from_opcode(opCode="NOOP"), DeployType.REGULAR)
+        self.assertEqual(config._get_deploy_type_from_opcode(opCode="DEPLOY"), DeployType.REGULAR)
+        self.assertEqual(config._get_deploy_type_from_opcode(opCode="UPDATE"), DeployType.REGULAR)
+        self.assertEqual(config._get_deploy_type_from_opcode(opCode="RESTART"), DeployType.RESTART)
+        self.assertEqual(config._get_deploy_type_from_opcode(opCode="DELETE"), DeployType.REGULAR)
+        self.assertEqual(config._get_deploy_type_from_opcode(opCode="TERMINATE"), DeployType.STOP)
+        self.assertEqual(config._get_deploy_type_from_opcode(opCode="WAIT"), DeployType.REGULAR)
+        self.assertEqual(config._get_deploy_type_from_opcode(opCode="ROLLBACK"), DeployType.ROLLBACK)
+        self.assertEqual(config._get_deploy_type_from_opcode(opCode="STOP"), DeployType.STOP)
+
 
 if __name__ == '__main__':
     unittest.main()

deploy-agent/tests/unit/deploy/types/test_ping_report.py

@@ -0,0 +1,23 @@
+from deployd.types.ping_report import PingReport 
+import unittest
+
+
+class TestPingReport(unittest.TestCase):
+    def test_init_enums(self):
+        report = PingReport(jsonValue={
+            "deployStage": 0,
+            "status": 0,
+            })
+        self.assertEqual(report.deployStage, "UNKNOWN") 
+        self.assertEqual(report.status, "SUCCEEDED") 
+
+        report = PingReport(jsonValue={
+            "deployStage": 10,
+            "status": 7,
+            })
+        self.assertEqual(report.deployStage, "STOPPED") 
+        self.assertEqual(report.status, "TOO_MANY_RETRY") 
+
+
+if __name__ == "__main__":
+    unittest.main()

deploy-agent/tests/unit/deploy/types/test_ping_request.py

@@ -0,0 +1,27 @@
+from deployd.types.ping_request import PingRequest 
+from deployd.types.ping_report import PingReport 
+import unittest
+
+
+class TestPingRequest(unittest.TestCase):
+    def test_to_json_enums(self):
+        reports = [
+            PingReport({
+                "deployStage": 7,
+                "status": 4,
+            }),
+            PingReport({
+                "deployStage": "POST_RESTART",
+                "status": "SCRIPT_FAILED",
+            }),
+        ]
+        request = PingRequest(reports=reports)
+        json_request = request.to_json()
+        self.assertEqual(json_request["reports"][0]["deployStage"], "POST_RESTART")
+        self.assertEqual(json_request["reports"][0]["agentStatus"], "SCRIPT_FAILED")
+        self.assertEqual(json_request["reports"][1]["deployStage"], "POST_RESTART")
+        self.assertEqual(json_request["reports"][1]["agentStatus"], "SCRIPT_FAILED")
+
+
+if __name__ == "__main__":
+    unittest.main()

deploy-board/deploy_board/templates/configs/env_config.tmpl

@@ -267,9 +267,7 @@
                             </span>
                         </div>
                         <div class="col-xs-12" id="stageTypeHelpInfo" hidden="true">
-                            DEV - Used for manual testing</br>
                             LATEST - Deploy the latest available build</br>
-                            STAGING - Deploy the latest available artifacts from master, then verify with testing (integration, stress, smoke etc)</br>
                             CANARY - Deploy the build after tests, then verify the ACA together with the CONTROL</br>
                             CONTROL - Deploy the build as the production, used with CANARY</br>
                             PRODUCTION - Deploy the build to production

deploy-board/deploy_board/templates/environs/new_stage_modal.tmpl

@@ -47,9 +47,7 @@
                                 </span>
                             </div>
                             <div class="col-xs-12" id="newStageTypeHelpInfo" hidden="true">
-                                DEV - Used for manual testing</br>
                                 LATEST - Deploy the latest available build</br>
-                                STAGING - Deploy the latest available artifacts from master, then verify with testing (integration, stress, smoke etc)</br>
                                 CANARY - Deploy the build after tests, then verify the ACA together with the CONTROL</br>
                                 CONTROL - Deploy the build as the production, used with CANARY</br>
                                 PRODUCTION - Deploy the build to production

deploy-board/deploy_board/webapp/env_views.py

@@ -867,6 +867,7 @@ def post_create_env(request):
     data = request.POST
     env_name = data["env_name"]
     stage_name = data["stage_name"]
+    stage_type = data.get("stageType")
     clone_env_name = data.get("clone_env_name")
     clone_stage_name = data.get("clone_stage_name")
     description = data.get('description')
@@ -876,7 +877,7 @@ def post_create_env(request):
         try:
             external_id = environs_helper.create_identifier_for_new_stage(request, env_name, stage_name)
             common.clone_from_stage_name(request, env_name, stage_name, clone_env_name,
-                                        clone_stage_name, description, external_id)
+                                        clone_stage_name, stage_type, description, external_id)
         except TeletraanException as detail:
             message = 'Failed to create identifier for {}/{}: {}'.format(env_name, stage_name, detail)
             log.error(message)
@@ -938,7 +939,7 @@ def post_add_stage(request, name):
     if from_stage:
         try:
             external_id = environs_helper.create_identifier_for_new_stage(request, name, stage)
-            common.clone_from_stage_name(request, name, stage, name, stage_type, from_stage, description, external_id)
+            common.clone_from_stage_name(request, name, stage, name, from_stage, stage_type, description, external_id)
         except TeletraanException as detail:
             message = 'Failed to create stage {}/{}: {}'.format(name, stage, detail)
             log.error(message)

deploy-board/deploy_board/webapp/helpers/environs_helper.py

@@ -54,7 +54,7 @@
 DEPLOY_CONSTRAINT_TYPES = ['GROUP_BY_GROUP', 'ALL_GROUPS_IN_PARALLEL']
 
 # Fetch from backend to avoid maintainng at multiple places?
-STAGE_TYPES = ['DEFAULT', 'DEV', 'LATEST', 'STAGING', 'CANARY', 'CONTROL', 'PRODUCTION']
+STAGE_TYPES = ['DEFAULT', 'LATEST', 'CANARY', 'CONTROL', 'PRODUCTION']
 
 deployclient = DeployClient()
 

deploy-service/common/src/main/java/com/pinterest/deployservice/ServiceContext.java

@@ -16,14 +16,6 @@
 package com.pinterest.deployservice;
 
 
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-
-import org.apache.commons.dbcp.BasicDataSource;
-
 import com.pinterest.deployservice.allowlists.Allowlist;
 import com.pinterest.deployservice.buildtags.BuildTagsManager;
 import com.pinterest.deployservice.chat.ChatManager;
@@ -57,6 +49,11 @@
 import com.pinterest.deployservice.scm.SourceControlManagerProxy;
 import com.pinterest.teletraan.universal.events.AppEventPublisher;
 
+import org.apache.commons.dbcp.BasicDataSource;
+
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+
 public class ServiceContext {
     private BasicDataSource dataSource;
     private BuildDAO buildDAO;
@@ -105,7 +102,6 @@ public class ServiceContext {
     private Long agentCountCacheTtl;
     private Long maxParallelThreshold;
     private BuildEventPublisher buildEventPublisher;
-    private Set<String> accountAllowList;
 
     // Publishers & Listeners
     private AppEventPublisher appEventPublisher;
@@ -351,6 +347,7 @@ public void setRodimusManager(RodimusManager rodimusManager) {
         this.rodimusManager = rodimusManager;
     }
 
+
     public void setBuildCacheEnabled(boolean buildCacheEnabled) {
         this.buildCacheEnabled = buildCacheEnabled;
     }
@@ -472,12 +469,4 @@ public void setBuildEventPublisher(
         BuildEventPublisher buildEventPublisher) {
         this.buildEventPublisher = buildEventPublisher;
     }
-
-    public Set<String> getAccountAllowList() { 
-        return accountAllowList;
-    }
-
-    public void setAccountAllowList(Collection<String> accountAllowList) {
-        this.accountAllowList = new HashSet<String>(accountAllowList);
-    }
 }

deploy-service/common/src/main/java/com/pinterest/deployservice/bean/EnvType.java

@@ -24,19 +24,13 @@
  *      Control stage type
  * CANARY:
  *      Canary stage type
- * STAGING:
- *    Staging stage type
  * LATEST:
  *      LATEST stage type
- * DEV:
- *      DEV stage type
  */
 public enum EnvType {
     DEFAULT,
     PRODUCTION,
     CONTROL,
     CANARY,
-    STAGING,
-    LATEST,
-    DEV;
+    LATEST;
 }