move the script for the gather scene out

.github/workflows/build_package.yml

@@ -8,6 +8,7 @@ on:
     branches:
       - master
       - v3.0-dev
+      - enhancement-optimize_external_script
 env:
   ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 

handler/gather/gather_scenes.py

@@ -42,7 +42,7 @@ def __init__(self, context, gather_pack_dir='./', tasks_base_path="~/.obdiag/gat
         self.gather_pack_dir = gather_pack_dir
         self.report_path = None
         self.yaml_tasks = {}
-        self.code_tasks = []
+        self.code_tasks = {}
         self.env = {}
         self.scene = "observer.base"
         self.tasks_base_path = tasks_base_path
@@ -88,8 +88,8 @@ def execute(self):
             self.stdio.verbose("execute_tasks. the number of tasks is {0} ,tasks is {1}".format(len(self.yaml_tasks.keys()), self.yaml_tasks.keys()))
             for key, value in zip(self.yaml_tasks.keys(), self.yaml_tasks.values()):
                 self.__execute_yaml_task_one(key, value)
-            for task in self.code_tasks:
-                self.__execute_code_task_one(task)
+            for key, value in zip(self.code_tasks.keys(), self.code_tasks.values()):
+                self.__execute_code_task_one(key, value)
         except Exception as e:
             self.stdio.error("Internal error :{0}".format(e))
 
@@ -116,11 +116,11 @@ def __execute_yaml_task_one(self, task_name, task_data):
             self.stdio.error("__execute_yaml_task_one Exception : {0}".format(e))
 
     # execute code task
-    def __execute_code_task_one(self, task_name):
+    def __execute_code_task_one(self, task_name, task_data):
         try:
             self.stdio.verbose("execute tasks is {0}".format(task_name))
-            scene = {"name": task_name}
-            task = SceneBase(context=self.context, scene=scene, report_dir=self.report_path, env=self.env, mode='code', task_type=task_name)
+            task = task_data["module"]
+            task.init(self.context, task_name, self.report_path)
             self.stdio.verbose("{0} execute!".format(task_name))
             task.execute()
             self.stdio.verbose("execute tasks end : {0}".format(task_name))
@@ -133,21 +133,18 @@ def __init_task_names(self):
             items = re.split(r'[;,]', new)
             scene = GatherScenesListHandler(self.context)
             for item in items:
-                yaml_task_data = scene.get_one_yaml_task(item)
-                is_code_task = scene.is_code_task(item)
-                if is_code_task:
-                    self.code_tasks.append(item)
+                task_data = scene.get_one_task(item)
+                if task_data["task_type"] == 'py':
+                    self.code_tasks[item] = task_data
+                elif task_data["task_type"] == 'yaml':
+                    self.yaml_tasks[item] = task_data
                 else:
-                    if yaml_task_data:
-                        self.yaml_tasks[item] = yaml_task_data
-                    else:
-                        self.stdio.error("Invalid Task :{0}. Please check the task is exist.".format(item))
-                        if ".yaml" in item:
-                            self.stdio.suggest("'.yaml' in task :{0}. Maybe you can remove it. use '--scene={1}'".format(item, item.replace(".yaml", "")))
+                    self.stdio.error("Invalid Task :{0}. Please check the task is exist.".format(item))
+                    if ".yaml" in item:
+                        self.stdio.suggest("'.yaml' in task :{0}. Maybe you can remove it. use '--scene={1}'".format(item, item.replace(".yaml", "")))
             # hard code add gather observer.base
             if len(self.code_tasks) > 0:
-                yaml_task_base = scene.get_one_yaml_task("observer.base")
-                self.yaml_tasks["observer.base"] = yaml_task_base
+                self.yaml_tasks["observer.base"] = scene.get_one_task("observer.base")
         else:
             self.stdio.error("get task name failed")
 

handler/gather/scenes/base.py

@@ -19,9 +19,6 @@
 from common.scene import filter_by_version
 from handler.gather.step.base import Base
 from common.tool import StringUtils
-from handler.gather.scenes.sql_problem import SQLProblemScene
-from handler.gather.scenes.cpu_high import CPUHighScene
-from handler.gather.scenes.px_collect_log import SQLPXCollectLogScene
 
 
 class SceneBase(SafeStdio):
@@ -90,23 +87,26 @@ def __execute_yaml_mode(self, nodes):
         self.stdio.verbose("run scene excute yaml mode in node")
 
     def __execute_code_mode(self):
-        skip_type = self.context.get_variable("gather_skip_type", None)
-        if skip_type:
-            self.stdio.verbose("needs to be filtered out and not gather type is {0}".format(skip_type))
-        if self.scene["name"] == "observer.perf_sql" or self.scene["name"] == "observer.sql_err":
-            scene = SQLProblemScene(self.context, self.scene["name"], self.report_dir, self.scene_variable_dict, self.env)
-        elif self.scene["name"] == "observer.cpu_high" and (skip_type != "ssh"):
-            scene = CPUHighScene(self.context, self.report_dir, self.scene_variable_dict, self.env)
-        elif self.scene["name"] == "observer.px_collect_log" and (skip_type != "ssh"):
-            scene = SQLPXCollectLogScene(self.context, self.scene["name"], self.report_dir, self.scene_variable_dict, self.env)
-        else:
-            scene_names = ["observer.perf_sql", "observer.cpu_high", "observer.px_collect_log"]
-            if self.scene["name"] not in scene_names:
-                self.stdio.error("unsupported hard code scene {0}".format(self.scene["name"]))
-            return
-        try:
-            self.stdio.verbose("hard code scene {0} execute start".format(self.scene["name"]))
-            scene.execute()
-            self.stdio.verbose("hard code scene {0} execute end".format(self.scene["name"]))
-        except Exception as e:
-            self.stdio.error("hard code scene execute failed, error :{0}".format(e))
+        pass
+
+    # def __execute_code_mode(self):
+    #     skip_type = self.context.get_variable("gather_skip_type", None)
+    #     if skip_type:
+    #         self.stdio.verbose("needs to be filtered out and not gather type is {0}".format(skip_type))
+    #     if self.scene["name"] == "observer.perf_sql" or self.scene["name"] == "observer.sql_err":
+    #         scene = SQLProblemScene(self.context, self.scene["name"], self.report_dir, self.scene_variable_dict, self.env)
+    #     elif self.scene["name"] == "observer.cpu_high" and (skip_type != "ssh"):
+    #         scene = CPUHighScene(self.context, self.report_dir, self.scene_variable_dict, self.env)
+    #     elif self.scene["name"] == "observer.px_collect_log" and (skip_type != "ssh"):
+    #         scene = SQLPXCollectLogScene(self.context, self.scene["name"], self.report_dir, self.scene_variable_dict, self.env)
+    #     else:
+    #         scene_names = ["observer.perf_sql", "observer.cpu_high", "observer.px_collect_log"]
+    #         if self.scene["name"] not in scene_names:
+    #             self.stdio.error("unsupported hard code scene {0}".format(self.scene["name"]))
+    #         return
+    #     try:
+    #         self.stdio.verbose("hard code scene {0} execute start".format(self.scene["name"]))
+    #         scene.execute()
+    #         self.stdio.verbose("hard code scene {0} execute end".format(self.scene["name"]))
+    #     except Exception as e:
+    #         self.stdio.error("hard code scene execute failed, error :{0}".format(e))

handler/gather/scenes/list.py

@@ -22,7 +22,7 @@
 from stdio import SafeStdio
 from common.tool import YamlUtils
 from handler.gather.scenes.register import hardcode_scene_list
-from common.tool import Util
+from common.tool import Util, DynamicLoading
 
 
 class GatherScenesListHandler(SafeStdio):
@@ -89,7 +89,7 @@ def __get_hardcode_task(self, scene):
             "info_cn": scene.info_cn,
         }
 
-    def get_one_yaml_task(self, name):
+    def get_one_task(self, name):
         try:
             task_data = None
             current_path = self.yaml_tasks_base_path
@@ -101,6 +101,22 @@ def get_one_yaml_task(self, name):
                         if name == task_name:
                             task_data = YamlUtils.read_yaml_data(os.path.join(root, file))
                             task_data["name"] = task_name
+                            task_data["task_type"] = 'yaml'
+                    if file.endswith('.py'):
+                        folder_name = os.path.basename(root)
+                        task_name = "{}.{}".format(folder_name, file.split('.')[0])
+                        if name == task_name:
+                            DynamicLoading.add_lib_path(root)
+                            task_module = DynamicLoading.import_module(file[:-3], None)
+                            attr_name = name.split('.')[-1]
+                            if not hasattr(task_module, attr_name):
+                                self.stdio.error("{0} import_module failed".format(attr_name))
+                                return
+                            task_data = {}
+                            task_data["name"] = task_name
+                            task_data["module"] = getattr(task_module, attr_name)
+                            task_data["task_type"] = 'py'
+                        pass
             return task_data
         except Exception as e:
             self.stdio.error("get one yaml task failed, error: ", e)

handler/gather/scenes/cpu_high.py → handler/gather/tasks/observer/cpu_high.py

@@ -24,8 +24,8 @@
 from handler.gather.gather_perf import GatherPerfHandler
 
 
-class CPUHighScene(SafeStdio):
-    def __init__(self, context, report_path, task_variable_dict=None, env={}):
+class CPUHigh(SafeStdio):
+    def init(self, context, scene_name, report_path, task_variable_dict=None, env={}):
         self.context = context
         self.stdio = context.stdio
         if task_variable_dict is None:
@@ -89,3 +89,6 @@ def report(self, file_path, command, data):
                 f.write(data + '\n')
         except Exception as e:
             self.stdio.error("report sql result to file: {0} failed, error: ".format(file_path))
+
+
+cpu_high = CPUHigh()

handler/gather/scenes/px_collect_log.py → handler/gather/tasks/observer/px_collect_log.py

@@ -24,8 +24,8 @@
 import datetime
 
 
-class SQLPXCollectLogScene(object):
-    def __init__(self, context, scene_name, report_path, task_variable_dict=None, env=None):
+class PXCollectLog(object):
+    def init(self, context, scene_name, report_path, task_variable_dict=None, env=None):
         self.context = context
         self.stdio = context.stdio
         if task_variable_dict is None:
@@ -189,3 +189,6 @@ def __parse_env(self):
             return True
         except Exception as e:
             self.stdio.error("Parse env fail. Exception : {0} .".format(e))
+
+
+px_collect_log = PXCollectLog()

handler/gather/scenes/sql_problem.py → handler/gather/tasks/observer/sql_problem.py

@@ -23,8 +23,8 @@
 from common.command import find_home_path_by_port
 
 
-class SQLProblemScene(SafeStdio):
-    def __init__(self, context, scene_name, report_path, task_variable_dict=None, env={}):
+class SQLProblem(SafeStdio):
+    def init(self, context, scene_name, report_path, task_variable_dict=None, env={}):
         self.context = context
         self.stdio = context.stdio
         if task_variable_dict is None:
@@ -130,3 +130,6 @@ def __parse_env(self):
         else:
             self.stdio.error("option env not found, please run 'obdiag gather scene list' to check usage")
             return False
+
+
+sql_problem = SQLProblem()

handler/rca/rca_list.py

@@ -46,7 +46,7 @@ def get_all_scenes(self):
             return
         for scene_file in scenes_files:
             lib_path = self.work_path
-            module_name = os.path.basename(scene_file)[:-9]
+            module_name = os.path.basename(scene_file)[:-3]
             DynamicLoading.add_lib_path(lib_path)
             module = DynamicLoading.import_module(os.path.basename(scene_file)[:-3], None)
             if not hasattr(module, module_name):
@@ -74,6 +74,6 @@ def __find_rca_files(self):
         for file_or_folder in os.listdir(self.work_path):
             full_path = os.path.join(self.work_path, file_or_folder)
             if os.path.isfile(full_path):
-                if full_path.endswith('_scene.py') and len(os.path.basename(full_path)) > 7:
+                if full_path.endswith('.py') and len(os.path.basename(full_path)) > 7:
                     files.append(full_path)
         return files