Skip to content

Commit

Permalink
Query optimization and query monitoring (#339)
Browse files Browse the repository at this point in the history 
* Added query monitoring

* Refactor Query Optimization

* Added grant privileges for apply without restarts

* increased version

* Fixed error

* Fixed variable name
  • Loading branch information
@kochetovd
kochetovd authored Aug 22, 2024
1 parent 47b49d7 commit b3832f9
Show file tree
Hide file tree
Showing 11 changed files with 198 additions and 134 deletions.
60 changes: 32 additions & 28 deletions config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,35 +10,36 @@ import (
)

const (
ReleemAgentVersion = "1.18.1"
ReleemAgentVersion = "1.19.0"
)

var (
DB *sql.DB
)

type Config struct {
Debug bool `hcl:"debug"`
Env string `hcl:"env"`
Hostname string `hcl:"hostname"`
ApiKey string `hcl:"apikey"`
TimePeriodSeconds time.Duration `hcl:"interval_seconds"`
ReadConfigSeconds time.Duration `hcl:"interval_read_config_seconds"`
GenerateConfigSeconds time.Duration `hcl:"interval_generate_config_seconds"`
MysqlPassword string `hcl:"mysql_password"`
MysqlUser string `hcl:"mysql_user"`
MysqlHost string `hcl:"mysql_host"`
MysqlPort string `hcl:"mysql_port"`
MysqlSslMode bool `hcl:"mysql_ssl_mode"`
CommandRestartService string `hcl:"mysql_restart_service"`
MysqlConfDir string `hcl:"mysql_cnf_dir"`
ReleemConfDir string `hcl:"releem_cnf_dir"`
ReleemDir string `hcl:"releem_dir"`
MemoryLimit int `hcl:"memory_limit"`
InstanceType string `hcl:"instance_type"`
AwsRegion string `hcl:"aws_region"`
AwsRDSDB string `hcl:"aws_rds_db"`
CollectExplain bool `hcl:"collect_explain"`
Debug bool `hcl:"debug"`
Env string `hcl:"env"`
Hostname string `hcl:"hostname"`
ApiKey string `hcl:"apikey"`
MetricsPeriod time.Duration `hcl:"interval_seconds"`
ReadConfigPeriod time.Duration `hcl:"interval_read_config_seconds"`
GenerateConfigPeriod time.Duration `hcl:"interval_generate_config_seconds"`
QueryOptimizationPeriod time.Duration `hcl:"interval_query_optimization_seconds"`
MysqlPassword string `hcl:"mysql_password"`
MysqlUser string `hcl:"mysql_user"`
MysqlHost string `hcl:"mysql_host"`
MysqlPort string `hcl:"mysql_port"`
MysqlSslMode bool `hcl:"mysql_ssl_mode"`
CommandRestartService string `hcl:"mysql_restart_service"`
MysqlConfDir string `hcl:"mysql_cnf_dir"`
ReleemConfDir string `hcl:"releem_cnf_dir"`
ReleemDir string `hcl:"releem_dir"`
MemoryLimit int `hcl:"memory_limit"`
InstanceType string `hcl:"instance_type"`
AwsRegion string `hcl:"aws_region"`
AwsRDSDB string `hcl:"aws_rds_db"`
QueryOptimization bool `hcl:"query_optimization"`
}

func LoadConfig(filename string, logger logging.Logger) (*Config, error) {
Expand All @@ -59,14 +60,17 @@ func LoadConfigFromString(data string, logger logging.Logger) (*Config, error) {
if err != nil {
return nil, err
}
if config.TimePeriodSeconds == 0 {
config.TimePeriodSeconds = 60
if config.MetricsPeriod == 0 {
config.MetricsPeriod = 60
}
if config.ReadConfigSeconds == 0 {
config.ReadConfigSeconds = 3600
if config.ReadConfigPeriod == 0 {
config.ReadConfigPeriod = 3600
}
if config.GenerateConfigSeconds == 0 {
config.GenerateConfigSeconds = 43200
if config.GenerateConfigPeriod == 0 {
config.GenerateConfigPeriod = 43200
}
if config.QueryOptimizationPeriod == 0 {
config.QueryOptimizationPeriod = 3600
}
if config.MysqlHost == "" {
config.MysqlHost = "127.0.0.1"
Expand Down
2 changes: 1 addition & 1 deletion current_version_agent
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1.18.1
1.19.0
14 changes: 10 additions & 4 deletions docker/releem.conf.tpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,22 @@ hostname="${RELEEM_HOSTNAME}"
# Defaults to 0, Mysql memory usage limit.
memory_limit=${MEMORY_LIMIT:-0}

# TimePeriodSeconds time.Duration `hcl:"interval_seconds"`
# MetricsPeriod time.Duration `hcl:"interval_seconds"`
# Defaults to 30 seconds, how often metrics are collected.
interval_seconds=60

# ReadConfigSeconds time.Duration `hcl:"interval_read_config_seconds"`
# ReadConfigPeriod time.Duration `hcl:"interval_read_config_seconds"`
# Defaults to 3600 seconds, how often to update the values from the config.
interval_read_config_seconds=3600

# GenerateConfigSeconds time.Duration `hcl:"interval_generate_config_seconds"`
# GenerateConfigPeriod time.Duration `hcl:"interval_generate_config_seconds"`
# Defaults to 43200 seconds, how often to generate recommend the config.
interval_generate_config_seconds=${RELEEM_INTERVAL_COLLECT_ALL_METRICS:-43200}

# QueryOptimization time.Duration `hcl:"interval_query_optimization_seconds"`
# Defaults to 3600 seconds, how often query metrics are collected.
interval_query_optimization_seconds=3600

# MysqlUser string`hcl:"mysql_user"`
# Mysql user name for collection metrics.
mysql_user="${DB_USER:-releem}"
Expand Down Expand Up @@ -68,4 +72,6 @@ env="${RELEEM_ENV:-prod}"
# Releem Debug messages
debug=${RELEEM_DEBUG:-false}

collect_explain=${RELEEM_QUERY_OPTIMIZATION:-false}
# Collect Explain string `hcl:"query_optimization"`
# Releem collect explain for query
query_optimization=${RELEEM_QUERY_OPTIMIZATION:-false}
19 changes: 16 additions & 3 deletions install.sh
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
#!/bin/bash
# install.sh - Version 1.18.1
# install.sh - Version 1.19.0
# (C) Releem, Inc 2022
# All rights reserved

# Releem installation script: install and set up the Releem Agent on supported Linux distributions
# using the package manager.

set -e
install_script_version=1.18.1
install_script_version=1.19.0
logfile="releem-install.log"

WORKDIR="/opt/releem"
Expand Down Expand Up @@ -321,6 +321,19 @@ else
else
printf "\033[31m\n This database version is too old.\033[0m\n"
fi

if mysql ${root_connection_string} --user=root --password=${RELEEM_MYSQL_ROOT_PASSWORD} -Be "GRANT SYSTEM_VARIABLES_ADMIN ON *.* TO '${RELEEM_MYSQL_LOGIN}'@'${mysql_user_host}';"
then
echo "Successfully GRANT" > /dev/null
else
if mysql ${root_connection_string} --user=root --password=${RELEEM_MYSQL_ROOT_PASSWORD} -Be "GRANT SUPER ON *.* TO '${RELEEM_MYSQL_LOGIN}'@'${mysql_user_host}';"
then
echo "Successfully GRANT" > /dev/null
else
printf "\033[31m\n Error granting privileges to apply without restarting.\033[0m\n"
fi
fi

if [ -n $RELEEM_QUERY_OPTIMIZATION ];
then
mysql ${root_connection_string} --user=root --password=${RELEEM_MYSQL_ROOT_PASSWORD} -Be "GRANT SELECT ON *.* TO '${RELEEM_MYSQL_LOGIN}'@'${mysql_user_host}';"
Expand Down Expand Up @@ -449,7 +462,7 @@ if [ -n "$RELEEM_MYSQL_SSL_MODE" ]; then
echo "mysql_ssl_mode=$RELEEM_MYSQL_SSL_MODE" | $sudo_cmd tee -a $CONF >/dev/null
fi
if [ -n "$RELEEM_QUERY_OPTIMIZATION" ]; then
echo "collect_explain=$RELEEM_QUERY_OPTIMIZATION" | $sudo_cmd tee -a $CONF >/dev/null
echo "query_optimization=$RELEEM_QUERY_OPTIMIZATION" | $sudo_cmd tee -a $CONF >/dev/null
fi
echo "interval_seconds=60" | $sudo_cmd tee -a $CONF >/dev/null
echo "interval_read_config_seconds=3600" | $sudo_cmd tee -a $CONF >/dev/null
Expand Down
33 changes: 18 additions & 15 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ type Service struct {

// Manage by daemon commands or run the daemon
func (service *Service) Manage(logger logging.Logger, configFile string, command []string, TypeConfiguration string, AgentEvent string, AgentTask string) (string, error) {
var gatherers, gatherers_configuration []m.MetricsGatherer
var Mode m.Mode
var gatherers, gatherers_configuration, gatherers_query_optimization []m.MetricsGatherer
var Mode m.ModeT
var configuration *config.Config
usage := "Usage: myservice install | remove | start | stop | status"

Expand Down Expand Up @@ -156,14 +156,18 @@ func (service *Service) Manage(logger logging.Logger, configFile string, command
defer config.DB.Close()

//Init repeaters
repeaters := make(map[string]m.MetricsRepeater)
repeaters["Metrics"] = m.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, m.Mode{Name: "Metrics", ModeType: ""}))
repeaters["Configurations"] = m.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, Mode))
repeaters["Event"] = m.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, Mode))
repeaters["TaskGet"] = m.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, m.Mode{Name: "TaskGet", ModeType: ""}))
repeaters["TaskStatus"] = m.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, m.Mode{Name: "TaskStatus", ModeType: ""}))
repeaters["TaskSet"] = m.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, Mode))
repeaters["GetConfigurationJson"] = m.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, m.Mode{Name: "Configurations", ModeType: "get-json"}))
// repeaters := make(map[string]m.MetricsRepeater)
// repeaters["Metrics"] = m.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, m.Mode{Name: "Metrics", ModeType: ""}))
// repeaters["Configurations"] = m.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, Mode))
// repeaters["Event"] = m.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, Mode))
// repeaters["TaskGet"] = m.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, m.Mode{Name: "TaskGet", ModeType: ""}))
// repeaters["TaskStatus"] = m.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, m.Mode{Name: "TaskStatus", ModeType: ""}))
// repeaters["TaskSet"] = m.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, Mode))
// repeaters["GetConfigurationJson"] = m.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, m.Mode{Name: "Configurations", ModeType: "get-json"}))
// repeaters["QueryOptimization"] = m.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, m.Mode{Name: "Metrics", ModeType: "QuerysOptimization"}))
// repeaters["QueriesOptimization"] = m.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, m.Mode{Name: "TaskSet", ModeType: "queries_optimization"}))
//var repeaters m.MetricsRepeater
repeaters := m.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration))

//Init gatherers
gatherers = append(gatherers,
Expand All @@ -172,10 +176,9 @@ func (service *Service) Manage(logger logging.Logger, configFile string, command
m.NewDbMetricsBaseGatherer(nil, configuration),
m.NewAgentMetricsGatherer(nil, configuration))
gatherers_configuration = append(gatherers_configuration, m.NewDbMetricsGatherer(nil, configuration))
if Mode.Name == "TaskSet" && Mode.ModeType == "collect_queries" {
gatherers = append(gatherers, m.NewDbCollectQueries(nil, configuration))
}
m.RunWorker(gatherers, gatherers_configuration, repeaters, nil, configuration, configFile, Mode)
gatherers_query_optimization = append(gatherers_query_optimization, m.NewDbCollectQueriesOptimization(nil, configuration))

m.RunWorker(gatherers, gatherers_configuration, gatherers_query_optimization, repeaters, nil, configuration, configFile, Mode)

// never happen, but need to complete code
return usage, nil
Expand All @@ -187,7 +190,7 @@ func main() {

configFile := flag.String("config", "/opt/releem/releem.conf", "Releem agent config")
SetConfigRun := flag.Bool("f", false, "Releem agent generate config")
GetConfigRun := flag.Bool("c", false, "Releem agent generate config")
GetConfigRun := flag.Bool("c", false, "Releem agent get config")

AgentEvent := flag.String("event", "", "Releem agent type event")
AgentTask := flag.String("task", "", "Releem agent task name")
Expand Down
4 changes: 2 additions & 2 deletions metrics/Metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ type MetricValue struct {
}
type MetricGroupValue map[string]interface{}

type Mode struct {
type ModeT struct {
Name string
ModeType string
}
Expand Down Expand Up @@ -78,7 +78,7 @@ type MetricsGatherer interface {
}

type MetricsRepeater interface {
ProcessMetrics(context MetricContext, metrics Metrics) (interface{}, error)
ProcessMetrics(context MetricContext, metrics Metrics, Mode ModeT) (interface{}, error)
}

func MapJoin(map1, map2 MetricGroupValue) MetricGroupValue {
Expand Down
Loading

0 comments on commit b3832f9

Please sign in to comment.