From 32a2671e1c8e3179c8f98fa66119da38b0a88b1f Mon Sep 17 00:00:00 2001 From: luowei Date: Tue, 11 Jul 2023 20:00:05 +0800 Subject: [PATCH 01/10] 1. return the specified error type; 2. sqle will not exec show create table again if parse create error to reduce repetitive execution --- sqle/driver/mysql/session/context.go | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/sqle/driver/mysql/session/context.go b/sqle/driver/mysql/session/context.go index 6bd6b34d67..c96114c25f 100644 --- a/sqle/driver/mysql/session/context.go +++ b/sqle/driver/mysql/session/context.go @@ -35,6 +35,8 @@ type TableInfo struct { // OriginalTable save parser object from db by query "show create table ..."; // using in inspect and generate rollback sql OriginalTable *ast.CreateTableStmt + // OriginalTableError save the error about getting original table + OriginalTableError error // todo #1630 临时缓存错误，方便跳过解析建表语句的错误 // MergedTable *ast.CreateTableStmt @@ -503,6 +505,17 @@ func (c *Context) AddSystemVariable(name, value string) { c.sysVars[name] = value } +type ParseShowCreateTableContentErr struct{} // todo #1630 临时返回一个指定的错误类型，方便跳过解析建表语句的错误 + +func (p *ParseShowCreateTableContentErr) Error() string { + return "parse show create table content failed" +} + +func IsParseShowCreateTableContentErr(err error) bool { + var target *ParseShowCreateTableContentErr + return errors.As(err, &target) +} + // GetCreateTableStmt get create table stmtNode for db by query; if table not exist, return null. func (c *Context) GetCreateTableStmt(stmt *ast.TableName) (*ast.CreateTableStmt, bool, error) { exist, err := c.IsTableExist(stmt) @@ -525,6 +538,9 @@ func (c *Context) GetCreateTableStmt(stmt *ast.TableName) (*ast.CreateTableStmt, return nil, false, nil } + if info.OriginalTableError != nil && IsParseShowCreateTableContentErr(err) { // todo #1630 临时减少解析失败时的调用次数 + return nil, false, info.OriginalTableError + } createTableSql, err := c.e.ShowCreateTable(utils.SupplementalQuotationMarks(stmt.Schema.String()), utils.SupplementalQuotationMarks(stmt.Name.String())) if err != nil { return nil, exist, err @@ -535,7 +551,7 @@ func (c *Context) GetCreateTableStmt(stmt *ast.TableName) (*ast.CreateTableStmt, log.Logger().Warnf("parse create table stmt failed. try to parse it as OB-MySQL-Mode. err:%v", err) createStmt, err = c.parseObMysqlCreateTableSql(createTableSql) if err != nil { - return nil, exist, err + return nil, exist, &ParseShowCreateTableContentErr{} } } info.OriginalTable = createStmt @@ -595,8 +611,9 @@ func (c *Context) parseObMysqlCreateTableSql(createTableSql string) (*ast.Create } } } - - return nil, fmt.Errorf("convert OB MySQL create table sql failed") + errMsg := "convert OB MySQL create table sql failed" + log.Logger().Errorf(errMsg) + return nil, errors.New(errMsg) } // GetCollationDatabase get collation database. From c9b95b69078dd4c46e4a1580d394a4c9bf16f501 Mon Sep 17 00:00:00 2001 From: luowei Date: Tue, 11 Jul 2023 20:17:13 +0800 Subject: [PATCH 02/10] return the original error message --- sqle/driver/mysql/session/context.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/sqle/driver/mysql/session/context.go b/sqle/driver/mysql/session/context.go index c96114c25f..b736fe7fbe 100644 --- a/sqle/driver/mysql/session/context.go +++ b/sqle/driver/mysql/session/context.go @@ -505,10 +505,12 @@ func (c *Context) AddSystemVariable(name, value string) { c.sysVars[name] = value } -type ParseShowCreateTableContentErr struct{} // todo #1630 临时返回一个指定的错误类型，方便跳过解析建表语句的错误 +type ParseShowCreateTableContentErr struct { // todo #1630 临时返回一个指定的错误类型，方便跳过解析建表语句的错误 + Msg string +} func (p *ParseShowCreateTableContentErr) Error() string { - return "parse show create table content failed" + return fmt.Sprintf("parse show create table content failed: %v", p.Msg) } func IsParseShowCreateTableContentErr(err error) bool { @@ -545,13 +547,13 @@ func (c *Context) GetCreateTableStmt(stmt *ast.TableName) (*ast.CreateTableStmt, if err != nil { return nil, exist, err } - createStmt, err := util.ParseCreateTableStmt(createTableSql) - if err != nil { + createStmt, errByMysqlParser := util.ParseCreateTableStmt(createTableSql) + if errByMysqlParser != nil { //todo to be compatible with OceanBase-MySQL-Mode log.Logger().Warnf("parse create table stmt failed. try to parse it as OB-MySQL-Mode. err:%v", err) createStmt, err = c.parseObMysqlCreateTableSql(createTableSql) if err != nil { - return nil, exist, &ParseShowCreateTableContentErr{} + return nil, exist, &ParseShowCreateTableContentErr{Msg: errByMysqlParser.Error()} } } info.OriginalTable = createStmt From c895fd7507dd79f48a95ac095cecf422f3792e20 Mon Sep 17 00:00:00 2001 From: luowei Date: Tue, 11 Jul 2023 20:19:04 +0800 Subject: [PATCH 03/10] skip the specified error when audit --- sqle/driver/mysql/audit.go | 7 ++++++- sqle/driver/mysql/mysql.go | 14 ++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/sqle/driver/mysql/audit.go b/sqle/driver/mysql/audit.go index b3a41fc84c..d3e4f09562 100644 --- a/sqle/driver/mysql/audit.go +++ b/sqle/driver/mysql/audit.go @@ -5,6 +5,7 @@ import ( "strings" rulepkg "github.com/actiontech/sqle/sqle/driver/mysql/rule" + "github.com/actiontech/sqle/sqle/driver/mysql/session" "github.com/actiontech/sqle/sqle/driver/mysql/util" driverV2 "github.com/actiontech/sqle/sqle/driver/v2" "github.com/actiontech/sqle/sqle/utils" @@ -34,6 +35,7 @@ const ( ) const CheckInvalidErrorFormat = "预检查失败: %v" +const CheckInvalidError = "预检查失败" func (i *MysqlDriverImpl) CheckInvalid(node ast.Node) error { var err error @@ -65,7 +67,10 @@ func (i *MysqlDriverImpl) CheckInvalid(node ast.Node) error { case *ast.UnparsedStmt: err = i.checkUnparsedStmt(stmt) } - if err != nil { + + if err != nil && session.IsParseShowCreateTableContentErr(err) { + return err // todo #1630 直接返回原始错误类型，方便跳过 + } else if err != nil { return fmt.Errorf(CheckInvalidErrorFormat, err) } return nil diff --git a/sqle/driver/mysql/mysql.go b/sqle/driver/mysql/mysql.go index e572427cb1..6e652dbd5c 100644 --- a/sqle/driver/mysql/mysql.go +++ b/sqle/driver/mysql/mysql.go @@ -309,7 +309,10 @@ func (i *MysqlDriverImpl) audit(ctx context.Context, sql string) (*driverV2.Audi } else { err = i.CheckInvalid(nodes[0]) } - if err != nil { + if err != nil && session.IsParseShowCreateTableContentErr(err) { + i.Logger().Errorf("check invalid failed: %v", err) + i.result.Add(driverV2.RuleLevelWarn, CheckInvalidError, fmt.Sprintf(CheckInvalidErrorFormat, "解析建表语句失败，部分在线审核规则可能失效，请人工确认")) + } else if err != nil { return nil, err } @@ -354,6 +357,11 @@ func (i *MysqlDriverImpl) audit(ctx context.Context, sql string) (*driverV2.Audi } if err := handler.Func(input); err != nil { + // todo #1630 临时跳过解析建表语句失败导致的规则 + if session.IsParseShowCreateTableContentErr(err) { + i.Logger().Errorf("skip rule, rule_desc_name=%v rule_desc=%v err:%v", rule.Name, rule.Desc, err.Error()) + continue + } return nil, err } } @@ -396,7 +404,9 @@ func (i *MysqlDriverImpl) audit(ctx context.Context, sql string) (*driverV2.Audi // print osc oscCommandLine, err := i.generateOSCCommandLine(nodes[0]) - if err != nil { + if err != nil && session.IsParseShowCreateTableContentErr(err) { + i.Logger().Errorf(err.Error()) // todo #1630 临时跳过创表语句解析错误 + } else if err != nil { return nil, err } if oscCommandLine != "" { From efc894c56fb7b0bcdcd9a35fe11cc230c897d401 Mon Sep 17 00:00:00 2001 From: luowei Date: Tue, 11 Jul 2023 20:31:23 +0800 Subject: [PATCH 04/10] skip the specified error when generate rollback sql --- sqle/server/audit.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sqle/server/audit.go b/sqle/server/audit.go index 1c3b48ec7c..85183adf28 100644 --- a/sqle/server/audit.go +++ b/sqle/server/audit.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/actiontech/sqle/sqle/driver" + "github.com/actiontech/sqle/sqle/driver/mysql/session" driverV2 "github.com/actiontech/sqle/sqle/driver/v2" "github.com/actiontech/sqle/sqle/model" "github.com/actiontech/sqle/sqle/utils" @@ -278,7 +279,10 @@ func genRollbackSQL(l *logrus.Entry, task *model.Task, p driver.Plugin) ([]*mode rollbackSQLs := make([]*model.RollbackSQL, 0, len(task.ExecuteSQLs)) for _, executeSQL := range task.ExecuteSQLs { rollbackSQL, reason, err := p.GenRollbackSQL(context.TODO(), executeSQL.Content) - if err != nil { + if err != nil && session.IsParseShowCreateTableContentErr(err) { + l.Errorf("gen rollback sql error, %v", err) // todo #1630 临时跳过创表语句解析错误 + return nil, nil + } else if err != nil { l.Errorf("gen rollback sql error, %v", err) return nil, err } From d9cfbf80267c6782b2bd1329629a4006ec0db6e3 Mon Sep 17 00:00:00 2001 From: luowei Date: Tue, 11 Jul 2023 21:29:27 +0800 Subject: [PATCH 05/10] to avoid panic because original table maybe empty --- sqle/driver/mysql/session/context.go | 3 +++ sqle/driver/mysql/util/parser_helper.go | 16 ++++++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/sqle/driver/mysql/session/context.go b/sqle/driver/mysql/session/context.go index b736fe7fbe..5767629644 100644 --- a/sqle/driver/mysql/session/context.go +++ b/sqle/driver/mysql/session/context.go @@ -372,6 +372,9 @@ func (c *Context) UpdateContext(node ast.Node) { } info.MergedTable, _ = util.MergeAlterToTable(oldTable, s) info.AlterTables = append(info.AlterTables, s) + if info.MergedTable.Table == nil { + return + } // rename table if s.Table.Name.String() != info.MergedTable.Table.Name.String() { schemaName := c.GetSchemaName(s.Table) diff --git a/sqle/driver/mysql/util/parser_helper.go b/sqle/driver/mysql/util/parser_helper.go index 1d587f8e86..b1eaacf027 100644 --- a/sqle/driver/mysql/util/parser_helper.go +++ b/sqle/driver/mysql/util/parser_helper.go @@ -503,13 +503,17 @@ func GetLimitCount(limit *ast.Limit, _default int64) (int64, error) { } func MergeAlterToTable(oldTable *ast.CreateTableStmt, alterTable *ast.AlterTableStmt) (*ast.CreateTableStmt, error) { - newTable := &ast.CreateTableStmt{ - Table: oldTable.Table, - Cols: oldTable.Cols, - Constraints: oldTable.Constraints, - Options: oldTable.Options, - Partition: oldTable.Partition, + newTable := &ast.CreateTableStmt{} + if oldTable != nil { + newTable = &ast.CreateTableStmt{ + Table: oldTable.Table, + Cols: oldTable.Cols, + Constraints: oldTable.Constraints, + Options: oldTable.Options, + Partition: oldTable.Partition, + } } + for _, spec := range GetAlterTableSpecByTp(alterTable.Specs, ast.AlterTableRenameTable) { newTable.Table = spec.NewTable } From 4873f4b87d65999ce21ad19a83b07130c38f9574 Mon Sep 17 00:00:00 2001 From: luowei Date: Wed, 12 Jul 2023 10:18:05 +0800 Subject: [PATCH 06/10] correct error --- sqle/driver/mysql/session/context.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sqle/driver/mysql/session/context.go b/sqle/driver/mysql/session/context.go index 5767629644..3650e7133a 100644 --- a/sqle/driver/mysql/session/context.go +++ b/sqle/driver/mysql/session/context.go @@ -543,7 +543,7 @@ func (c *Context) GetCreateTableStmt(stmt *ast.TableName) (*ast.CreateTableStmt, return nil, false, nil } - if info.OriginalTableError != nil && IsParseShowCreateTableContentErr(err) { // todo #1630 临时减少解析失败时的调用次数 + if info.OriginalTableError != nil && IsParseShowCreateTableContentErr(info.OriginalTableError) { // todo #1630 临时减少解析失败时的调用次数 return nil, false, info.OriginalTableError } createTableSql, err := c.e.ShowCreateTable(utils.SupplementalQuotationMarks(stmt.Schema.String()), utils.SupplementalQuotationMarks(stmt.Name.String())) @@ -556,7 +556,8 @@ func (c *Context) GetCreateTableStmt(stmt *ast.TableName) (*ast.CreateTableStmt, log.Logger().Warnf("parse create table stmt failed. try to parse it as OB-MySQL-Mode. err:%v", err) createStmt, err = c.parseObMysqlCreateTableSql(createTableSql) if err != nil { - return nil, exist, &ParseShowCreateTableContentErr{Msg: errByMysqlParser.Error()} + info.OriginalTableError = &ParseShowCreateTableContentErr{Msg: errByMysqlParser.Error()} + return nil, exist, info.OriginalTableError } } info.OriginalTable = createStmt From adc98b33686f44865770ef9df3fddee8f9f3c598 Mon Sep 17 00:00:00 2001 From: luowei Date: Wed, 12 Jul 2023 10:43:42 +0800 Subject: [PATCH 07/10] modify function name to reduce confusion --- sqle/driver/mysql/session/context.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sqle/driver/mysql/session/context.go b/sqle/driver/mysql/session/context.go index 3650e7133a..0d4635d6f3 100644 --- a/sqle/driver/mysql/session/context.go +++ b/sqle/driver/mysql/session/context.go @@ -553,8 +553,8 @@ func (c *Context) GetCreateTableStmt(stmt *ast.TableName) (*ast.CreateTableStmt, createStmt, errByMysqlParser := util.ParseCreateTableStmt(createTableSql) if errByMysqlParser != nil { //todo to be compatible with OceanBase-MySQL-Mode - log.Logger().Warnf("parse create table stmt failed. try to parse it as OB-MySQL-Mode. err:%v", err) - createStmt, err = c.parseObMysqlCreateTableSql(createTableSql) + log.Logger().Warnf("parse create table stmt failed. try to parse it with compabile method. err:%v", errByMysqlParser) + createStmt, err = c.parseCreateTableSqlCompatibly(createTableSql) if err != nil { info.OriginalTableError = &ParseShowCreateTableContentErr{Msg: errByMysqlParser.Error()} return nil, exist, info.OriginalTableError @@ -608,7 +608,7 @@ partition p15) 建表语句后半段是options，oceanbase mysql模式下的show create table结果返回的options中包含mysql不支持的options, 为了能解析, 方法将会倒着遍历建表语句, 每次找到右括号时截断后面的部分, 然后尝试解析一次, 直到解析成功, 此时剩余的建表语句将不在包含OB特有options */ -func (c *Context) parseObMysqlCreateTableSql(createTableSql string) (*ast.CreateTableStmt, error) { +func (c *Context) parseCreateTableSqlCompatibly(createTableSql string) (*ast.CreateTableStmt, error) { for i := len(createTableSql) - 1; i >= 0; i-- { if createTableSql[i] == ')' { stmt, err := util.ParseCreateTableStmt(createTableSql[0 : i+1]) @@ -617,7 +617,7 @@ func (c *Context) parseObMysqlCreateTableSql(createTableSql string) (*ast.Create } } } - errMsg := "convert OB MySQL create table sql failed" + errMsg := "parse create table sql with compatible method failed" log.Logger().Errorf(errMsg) return nil, errors.New(errMsg) } From edfaae40951d3f687b956577dedbe44fde2c6aef Mon Sep 17 00:00:00 2001 From: luowei Date: Wed, 12 Jul 2023 11:03:26 +0800 Subject: [PATCH 08/10] fix docker check --- sqle/driver/mysql/session/context.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sqle/driver/mysql/session/context.go b/sqle/driver/mysql/session/context.go index 0d4635d6f3..227cdb00b9 100644 --- a/sqle/driver/mysql/session/context.go +++ b/sqle/driver/mysql/session/context.go @@ -508,16 +508,16 @@ func (c *Context) AddSystemVariable(name, value string) { c.sysVars[name] = value } -type ParseShowCreateTableContentErr struct { // todo #1630 临时返回一个指定的错误类型，方便跳过解析建表语句的错误 +type ParseShowCreateTableContentError struct { // todo #1630 临时返回一个指定的错误类型，方便跳过解析建表语句的错误 Msg string } -func (p *ParseShowCreateTableContentErr) Error() string { +func (p *ParseShowCreateTableContentError) Error() string { return fmt.Sprintf("parse show create table content failed: %v", p.Msg) } func IsParseShowCreateTableContentErr(err error) bool { - var target *ParseShowCreateTableContentErr + var target *ParseShowCreateTableContentError return errors.As(err, &target) } @@ -553,10 +553,10 @@ func (c *Context) GetCreateTableStmt(stmt *ast.TableName) (*ast.CreateTableStmt, createStmt, errByMysqlParser := util.ParseCreateTableStmt(createTableSql) if errByMysqlParser != nil { //todo to be compatible with OceanBase-MySQL-Mode - log.Logger().Warnf("parse create table stmt failed. try to parse it with compabile method. err:%v", errByMysqlParser) + log.Logger().Warnf("parse create table stmt failed. try to parse it with compatible method. err:%v", errByMysqlParser) createStmt, err = c.parseCreateTableSqlCompatibly(createTableSql) if err != nil { - info.OriginalTableError = &ParseShowCreateTableContentErr{Msg: errByMysqlParser.Error()} + info.OriginalTableError = &ParseShowCreateTableContentError{Msg: errByMysqlParser.Error()} return nil, exist, info.OriginalTableError } } From 253d429a15fd171ef819d8b78e0fa999b530d387 Mon Sep 17 00:00:00 2001 From: luowei Date: Wed, 12 Jul 2023 14:01:18 +0800 Subject: [PATCH 09/10] detail log --- sqle/driver/mysql/mysql.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqle/driver/mysql/mysql.go b/sqle/driver/mysql/mysql.go index 6e652dbd5c..ef6b141aa9 100644 --- a/sqle/driver/mysql/mysql.go +++ b/sqle/driver/mysql/mysql.go @@ -405,7 +405,7 @@ func (i *MysqlDriverImpl) audit(ctx context.Context, sql string) (*driverV2.Audi // print osc oscCommandLine, err := i.generateOSCCommandLine(nodes[0]) if err != nil && session.IsParseShowCreateTableContentErr(err) { - i.Logger().Errorf(err.Error()) // todo #1630 临时跳过创表语句解析错误 + i.Logger().Errorf("generate osc command failed: %v", err.Error()) // todo #1630 临时跳过创表语句解析错误 } else if err != nil { return nil, err } From 0017cebf207e63787935889cb56ba5b3163965c3 Mon Sep 17 00:00:00 2001 From: luowei Date: Wed, 12 Jul 2023 16:17:14 +0800 Subject: [PATCH 10/10] fix panic --- sqle/driver/mysql/session/context.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqle/driver/mysql/session/context.go b/sqle/driver/mysql/session/context.go index 227cdb00b9..052f2ba2a4 100644 --- a/sqle/driver/mysql/session/context.go +++ b/sqle/driver/mysql/session/context.go @@ -372,7 +372,7 @@ func (c *Context) UpdateContext(node ast.Node) { } info.MergedTable, _ = util.MergeAlterToTable(oldTable, s) info.AlterTables = append(info.AlterTables, s) - if info.MergedTable.Table == nil { + if info.MergedTable == nil || info.MergedTable.Table == nil { return } // rename table