Skip to content

Commit

Permalink
优化运行脚本日志输出,TD数据迁移工具优化,增加TD数据迁移工具说明。
Browse files Browse the repository at this point in the history
  • Loading branch information
microrain authored and microrain committed Jan 10, 2023
1 parent 98a7874 commit 4522fa6
Show file tree
Hide file tree
Showing 7 changed files with 59 additions and 35 deletions.
2 changes: 1 addition & 1 deletion curl.sh
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ function check_pid() {
return
fi

nohup ./$app &> $logfile &
nohup ./$app &>> $logfile &
sleep 1
running=$(ps -p $! | grep -c -v "PID TTY")
if [ "$running" -gt 0 ];then
Expand Down
18 changes: 9 additions & 9 deletions internal/controller/system/sys_plugins_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@ import (
"github.com/sagoo-cloud/sagooiot/internal/service"
)

var SysPluginsConfig = cSystemPlugins_config{}
var SysPluginsConfig = csystempluginsConfig{}

type cSystemPlugins_config struct{}
type csystempluginsConfig struct{}

// GetPluginsConfigList 获取列表
func (u *cSystemPlugins_config) GetPluginsConfigList(ctx context.Context, req *system.GetPluginsConfigListReq) (res *system.GetPluginsConfigListRes, err error) {
func (u *csystempluginsConfig) GetPluginsConfigList(ctx context.Context, req *system.GetPluginsConfigListReq) (res *system.GetPluginsConfigListRes, err error) {
var inputData = new(model.GetPluginsConfigListInput)
if err = gconv.Scan(req, &inputData); err != nil {
return
Expand All @@ -28,23 +28,23 @@ func (u *cSystemPlugins_config) GetPluginsConfigList(ctx context.Context, req *s
}

// GetPluginsConfigById 获取指定ID数据
func (u *cSystemPlugins_config) GetPluginsConfigById(ctx context.Context, req *system.GetPluginsConfigByIdReq) (res *system.GetPluginsConfigByIdRes, err error) {
func (u *csystempluginsConfig) GetPluginsConfigById(ctx context.Context, req *system.GetPluginsConfigByIdReq) (res *system.GetPluginsConfigByIdRes, err error) {
data, err := service.SystemPluginsConfig().GetPluginsConfigById(ctx, req.Id)
res = new(system.GetPluginsConfigByIdRes)
err = gconv.Scan(data, &res)
return
}

// GetPluginsConfigByName 获取指定类型与名称的插件配置数据
func (u *cSystemPlugins_config) GetPluginsConfigByName(ctx context.Context, req *system.GetPluginsConfigByNameReq) (res *system.GetPluginsConfigByNameRes, err error) {
func (u *csystempluginsConfig) GetPluginsConfigByName(ctx context.Context, req *system.GetPluginsConfigByNameReq) (res *system.GetPluginsConfigByNameRes, err error) {
data, err := service.SystemPluginsConfig().GetPluginsConfigByName(ctx, req.Type, req.Name)
res = new(system.GetPluginsConfigByNameRes)
err = gconv.Scan(data, &res)
return
}

// AddPluginsConfig 添加数据
func (u *cSystemPlugins_config) AddPluginsConfig(ctx context.Context, req *system.AddPluginsConfigReq) (res *system.AddPluginsConfigRes, err error) {
func (u *csystempluginsConfig) AddPluginsConfig(ctx context.Context, req *system.AddPluginsConfigReq) (res *system.AddPluginsConfigRes, err error) {
var data = model.PluginsConfigAddInput{}
if err = gconv.Scan(req, &data); err != nil {
return
Expand All @@ -54,7 +54,7 @@ func (u *cSystemPlugins_config) AddPluginsConfig(ctx context.Context, req *syste
}

// EditPluginsConfig 修改数据
func (u *cSystemPlugins_config) EditPluginsConfig(ctx context.Context, req *system.EditPluginsConfigReq) (res *system.EditPluginsConfigRes, err error) {
func (u *csystempluginsConfig) EditPluginsConfig(ctx context.Context, req *system.EditPluginsConfigReq) (res *system.EditPluginsConfigRes, err error) {
var data = model.PluginsConfigEditInput{}
if err = gconv.Scan(req, &data); err != nil {
return
Expand All @@ -64,7 +64,7 @@ func (u *cSystemPlugins_config) EditPluginsConfig(ctx context.Context, req *syst
}

// SavePluginsConfig 修改数据
func (u *cSystemPlugins_config) SavePluginsConfig(ctx context.Context, req *system.SavePluginsConfigReq) (res *system.SavePluginsConfigRes, err error) {
func (u *csystempluginsConfig) SavePluginsConfig(ctx context.Context, req *system.SavePluginsConfigReq) (res *system.SavePluginsConfigRes, err error) {
var data = model.PluginsConfigAddInput{}
if err = gconv.Scan(req, &data); err != nil {
return
Expand All @@ -74,7 +74,7 @@ func (u *cSystemPlugins_config) SavePluginsConfig(ctx context.Context, req *syst
}

// DeletePluginsConfig 删除数据
func (u *cSystemPlugins_config) DeletePluginsConfig(ctx context.Context, req *system.DeletePluginsConfigReq) (res *system.DeletePluginsConfigRes, err error) {
func (u *csystempluginsConfig) DeletePluginsConfig(ctx context.Context, req *system.DeletePluginsConfigReq) (res *system.DeletePluginsConfigRes, err error) {
if len(req.Ids) == 0 {
err = gerror.New("ID参数错误")
}
Expand Down
2 changes: 1 addition & 1 deletion internal/logic/product/dev_device.go
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ func (s *sDevDevice) Del(ctx context.Context, ids []uint) (err error) {
}

num, _ := rs.RowsAffected()
if num > 0 {
if num > 0 && res.MetadataTable == 1 {
// 删除TD子表
err = service.TSLTable().DropTable(ctx, res.Key)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion internal/logic/product/dev_product.go
Original file line number Diff line number Diff line change
Expand Up @@ -282,7 +282,7 @@ func (s *sDevProduct) Del(ctx context.Context, ids []uint) (err error) {
}

num, _ := rs.RowsAffected()
if num > 0 {
if num > 0 && res.MetadataTable == 1 {
// 删除TD表
err = service.TSLTable().DropStable(ctx, res.Key)
if err != nil {
Expand Down
8 changes: 0 additions & 8 deletions internal/model/alarm_level.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,6 @@ package model

import "github.com/sagoo-cloud/sagooiot/internal/model/entity"

const (
AlarmLevel_1 uint = iota + 1 // 告警级别:超紧急
AlarmLevel_2 // 告警级别:紧急
AlarmLevel_3 // 告警级别:严重
AlarmLevel_4 // 告警级别:一般
AlarmLevel_5 // 告警级别:提醒
)

type AlarmLevelOutput struct {
*entity.AlarmLevel
}
Expand Down
41 changes: 26 additions & 15 deletions tools/migration/internal/td_destination.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@ import (
"context"
"database/sql"
"fmt"
"math"
"strconv"
"strings"

"github.com/gogf/gf/v2/container/gvar"
)

type TdDestination struct {
Expand Down Expand Up @@ -51,6 +51,8 @@ func (s *TdDestination) CreateTables() (err error) {
return
}

const pageSize = 3000

func (s *TdDestination) InsertData() (err error) {
tdSrc := NewTdSource()
tables, err := tdSrc.Tables()
Expand All @@ -61,25 +63,34 @@ func (s *TdDestination) InsertData() (err error) {
for _, rs := range tables {
tb := rs["table_name"].String()
count, _ := tdSrc.Count(tb)
if count > 0 {
data, err := tdSrc.Data(tb, 1, 1000)
if count == 0 {
continue
}
println("insert table : " + tb + ", data count: " + strconv.Itoa(count))

for i := 1; i <= int(math.Ceil(float64(count)/pageSize)); i++ {
data, err := tdSrc.Data(tb, i, pageSize)
if err != nil || data.Len() == 0 {
continue
}

var (
fed []string
vle []string
)
fed = data[0].GMap().Keys()

for _, row := range data {
var (
field []string
value []string
)
for k, v := range row {
field = append(field, k)
value = append(value, "'"+gvar.New(v).String()+"'")
}
sqlStr := fmt.Sprintf("insert into %s (%s) values (%s)", tb, strings.Join(field, ","), strings.Join(value, ","))
if _, err = s.db.Exec(sqlStr); err != nil {
return err
var vs []string
for _, k := range fed {
vs = append(vs, "'"+row[k].String()+"'")
}
vle = append(vle, "("+strings.Join(vs, ",")+")")
}

sqlStr := fmt.Sprintf("insert into %s (%s) values %s", tb, strings.Join(fed, ","), strings.Join(vle, ","))
if _, err = GetConn(context.Background(), "tdengineDest").Exec(sqlStr); err != nil {
return err
}
}
}
Expand Down
21 changes: 21 additions & 0 deletions tools/migration/readme.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# TDengine 数据库迁移工具

> 简单的 TDengine 数据库迁移工具
> 该工具在数据迁移时会删除目标数据库的超级表子表的表结构和数据请注意备份!!

使用方式直接动行即可

## TDengine 配置
1. tdengineSrc源数据库连接配置
2. tdengineDest目标数据库连接配置

## 程序运行时迁移步骤说明
1. 创建超级表
- 获取源数据库的超级表结构
- 删除目标数据库超级表结构
- 创建新超级表
2. 创建子表
- 获取源数据库的子表结构
- 删除目标数据库子表结构
- 创建新子表
3. 数据迁移

0 comments on commit 4522fa6

Please sign in to comment.