Added queries collecting (#280)

* Added collection queries. Handler errors

* Added filed with wum time

* fixed reading data from queries

* Increased version

config/config.go

@@ -9,7 +9,7 @@ import (
 )
 
 const (
-	ReleemAgentVersion = "1.13.0.1"
+	ReleemAgentVersion = "1.14.0"
 )
 
 type Config struct {

current_version_agent

@@ -1 +1 @@
-1.13.0.1
+1.14.0

install.sh

@@ -1,13 +1,13 @@
 #!/bin/bash
-# install.sh - Version 1.13.0.1
+# install.sh - Version 1.14.0
 # (C) Releem, Inc 2022
 # All rights reserved
 
 # Releem installation script: install and set up the Releem Agent on supported Linux distributions
 # using the package manager.
 
 set -e
-install_script_version=1.13.0.1
+install_script_version=1.14.0
 logfile="releem-install.log"
 
 WORKDIR="/opt/releem"

main.go

@@ -86,6 +86,7 @@ func (service *Service) Manage(logger logging.Logger, configFile string, command
 	configuration, err := config.LoadConfig(configFile, logger)
 	if err != nil {
 		logger.PrintError("Config load failed", err)
+		os.Exit(0)
 	}
 
 	if len(AgentEvents) > 0 {
@@ -139,7 +140,7 @@ func (service *Service) Manage(logger logging.Logger, configFile string, command
 		logger.Debug("RDS.DescribeDBInstances SUCCESS")
 
 		// Request detailed instance info
-		if len(result.DBInstances) == 1 {
+		if result != nil && len(result.DBInstances) == 1 {
 			//	gatherers = append(gatherers, m.NewAWSRDSMetricsGatherer(nil, cwclient, configuration))
 			//	gatherers = append(gatherers, m.NewAWSRDSInstanceGatherer(nil, rdsclient, ec2client, configuration))
 			configuration.Hostname = configuration.AwsRDSDB
@@ -198,6 +199,7 @@ func (service *Service) Manage(logger logging.Logger, configFile string, command
 	if (Mode.Name == "Configurations" && Mode.ModeType == "get") || Mode.Name == "Events" || Mode.Name == "Task" {
 		gatherers = append(gatherers,
 			m.NewDbConfGatherer(nil, db, configuration),
+			m.NewDbInfoGatherer(nil, db, configuration),
 			m.NewAgentMetricsGatherer(nil, configuration))
 	} else {
 		gatherers = append(gatherers,
@@ -207,7 +209,9 @@ func (service *Service) Manage(logger logging.Logger, configFile string, command
 			m.NewAgentMetricsGatherer(nil, configuration))
 		gatherers_configuration = append(gatherers_configuration, m.NewDbMetricsGatherer(nil, db, configuration))
 	}
-
+	if Mode.Name == "Task" && Mode.ModeType == "collect_queries" {
+		gatherers = append(gatherers, m.NewDbCollectQueries(nil, db, configuration))
+	}
 	m.RunWorker(gatherers, gatherers_configuration, repeaters, nil, configuration, configFile, Mode)
 
 	// never happen, but need to complete code

metrics/Metrics.go

@@ -40,7 +40,8 @@ type Metrics struct {
 		Conf struct {
 			Variables MetricGroupValue
 		}
-		Info MetricGroupValue
+		Info    MetricGroupValue
+		Queries []MetricGroupValue
 	}
 	ReleemAgent struct {
 		Info  MetricGroupValue

metrics/dbCollectQueries.go

@@ -0,0 +1,67 @@
+package metrics
+
+import (
+	"database/sql"
+
+	"github.com/Releem/mysqlconfigurer/config"
+	"github.com/advantageous/go-logback/logging"
+)
+
+type DbCollectQueries struct {
+	logger logging.Logger
+	debug  bool
+	db     *sql.DB
+}
+
+func NewDbCollectQueries(logger logging.Logger, db *sql.DB, configuration *config.Config) *DbCollectQueries {
+
+	if logger == nil {
+		if configuration.Debug {
+			logger = logging.NewSimpleDebugLogger("DbCollectQueries")
+		} else {
+			logger = logging.NewSimpleLogger("DbCollectQueries")
+		}
+	}
+
+	return &DbCollectQueries{
+		logger: logger,
+		debug:  configuration.Debug,
+		db:     db,
+	}
+}
+
+func (DbCollectQueries *DbCollectQueries) GetMetrics(metrics *Metrics) error {
+
+	// Latency
+	{
+		var output []MetricGroupValue
+		var schema_name, query string
+		var calls, avg_time_us, sum_time_us int
+
+		rows, err := DbCollectQueries.db.Query("SELECT IFNULL(schema_name, 'NULL') as schema_name, IFNULL(digest_text, 'NULL') as query, count_star as calls, round(avg_timer_wait/1000000, 0) as avg_time_us, round(SUM_TIMER_WAIT/1000000, 0) as sum_time_us FROM performance_schema.events_statements_summary_by_digest")
+		if err != nil {
+			if err != sql.ErrNoRows {
+				DbCollectQueries.logger.Error(err)
+			}
+		} else {
+			for rows.Next() {
+				err := rows.Scan(&schema_name, &query, &calls, &avg_time_us, &sum_time_us)
+				if err != nil {
+					DbCollectQueries.logger.Error(err)
+					return err
+				}
+				digest := MetricGroupValue{"schema_name": schema_name, "query": query, "calls": calls, "avg_time_us": avg_time_us, "sum_time_us": sum_time_us}
+				output = append(output, digest)
+			}
+		}
+		if len(output) != 0 {
+			metrics.DB.Queries = output
+		} else {
+			metrics.DB.Queries = nil
+		}
+	}
+
+	DbCollectQueries.logger.Debug("collectMetrics ", metrics.DB.Queries)
+	return nil
+
+}

metrics/runner.go

@@ -25,7 +25,7 @@ func makeTerminateChannel() <-chan os.Signal {
 
 func RunWorker(gatherers []MetricsGatherer, gatherers_configuration []MetricsGatherer, repeaters map[string]MetricsRepeater, logger logging.Logger,
 	configuration *config.Config, configFile string, Mode Mode) {
-	var GenerateTimer *time.Timer
+	var GenerateTimer, timer *time.Timer
 	defer HandlePanic(configuration, logger)
 	if logger == nil {
 		if configuration.Debug {
@@ -36,13 +36,14 @@ func RunWorker(gatherers []MetricsGatherer, gatherers_configuration []MetricsGat
 	}
 
 	logger.Debug(configuration)
-	timer := time.NewTimer(1 * time.Second)
 	configTimer := time.NewTimer(configuration.ReadConfigSeconds * time.Second)
 	if (Mode.Name == "Configurations" && Mode.ModeType != "default") || Mode.Name == "Events" || Mode.Name == "Task" {
 		GenerateTimer = time.NewTimer(0 * time.Second)
-
+		timer = time.NewTimer(3600 * time.Second)
 	} else {
 		GenerateTimer = time.NewTimer(configuration.GenerateConfigSeconds * time.Second)
+		timer = time.NewTimer(1 * time.Second)
+
 	}
 
 	terminator := makeTerminateChannel()
@@ -187,6 +188,58 @@ func processTask(metrics Metrics, repeaters map[string]MetricsRepeater, logger l
 			output["task_output"] = task_output + stderr.String()
 			logger.Println(" * Task with id -", TaskID, "and type id -", TaskTypeID, "completed with code", output["task_exit_code"])
 
+			metrics.ReleemAgent.Tasks = output
+			logger.Debug(output)
+			processRepeaters(metrics, repeaters["TaskStatus"], configuration, logger)
+		} else if TaskTypeID == 2 {
+			cmd := exec.Command(configuration.ReleemDir+"/mysqlconfigurer.sh", "-u")
+			cmd.Stdout = &stdout
+			cmd.Stderr = &stderr
+			processRepeaters(metrics, repeaters["TaskStatus"], configuration, logger)
+			err := cmd.Run()
+			task_output := ""
+			if err != nil {
+				task_output = task_output + err.Error()
+				logger.Error(err)
+				if exiterr, ok := err.(*exec.ExitError); ok {
+					output["task_exit_code"] = exiterr.ExitCode()
+				} else {
+					output["task_exit_code"] = 999
+				}
+				output["task_status"] = 4
+			} else {
+				output["task_exit_code"] = 0
+				output["task_status"] = 1
+			}
+			output["task_output"] = task_output + stderr.String()
+			logger.Println(" * Task with id -", TaskID, "and type id -", TaskTypeID, "completed with code", output["task_exit_code"])
+
+			metrics.ReleemAgent.Tasks = output
+			logger.Debug(output)
+			processRepeaters(metrics, repeaters["TaskStatus"], configuration, logger)
+		} else if TaskTypeID == 3 {
+			cmd := exec.Command(configuration.ReleemDir+"/releem-agent", "--task=collect_queries")
+			cmd.Stdout = &stdout
+			cmd.Stderr = &stderr
+			processRepeaters(metrics, repeaters["TaskStatus"], configuration, logger)
+			err := cmd.Run()
+			task_output := ""
+			if err != nil {
+				task_output = task_output + err.Error()
+				logger.Error(err)
+				if exiterr, ok := err.(*exec.ExitError); ok {
+					output["task_exit_code"] = exiterr.ExitCode()
+				} else {
+					output["task_exit_code"] = 999
+				}
+				output["task_status"] = 4
+			} else {
+				output["task_exit_code"] = 0
+				output["task_status"] = 1
+			}
+			output["task_output"] = task_output + stderr.String()
+			logger.Println(" * Task with id -", TaskID, "and type id -", TaskTypeID, "completed with code", output["task_exit_code"])
+
 			metrics.ReleemAgent.Tasks = output
 			logger.Debug(output)
 			processRepeaters(metrics, repeaters["TaskStatus"], configuration, logger)

mysqlconfigurer.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# mysqlconfigurer.sh - Version 1.13.0.1
+# mysqlconfigurer.sh - Version 1.14.0
 # (C) Releem, Inc 2022
 # All rights reserved
 
@@ -12,7 +12,7 @@ MYSQLTUNER_REPORT=$MYSQLCONFIGURER_PATH"mysqltunerreport.json"
 RELEEM_MYSQL_VERSION=$MYSQLCONFIGURER_PATH"mysql_version"
 MYSQLCONFIGURER_CONFIGFILE="${MYSQLCONFIGURER_PATH}${MYSQLCONFIGURER_FILE_NAME}"
 MYSQL_MEMORY_LIMIT=0
-VERSION="1.13.0.1"
+VERSION="1.14.0"
 RELEEM_INSTALL_PATH=$MYSQLCONFIGURER_PATH"install.sh"
 logfile="releem-mysqlconfigurer.log"