diff --git a/server/controller/controller/master.go b/server/controller/controller/master.go index ebe75f295461..24f52653185e 100644 --- a/server/controller/controller/master.go +++ b/server/controller/controller/master.go @@ -82,7 +82,10 @@ func checkAndStartMasterFunctions( // - 控制器和数据节点检查 // - license分配和检查 // - resource id manager - // - clean deleted resources + // - clean deleted/dirty resource data + // - prometheus encoder + // - prometheus app label layout updater + // - http resource refresh task manager // 从区域控制器无需判断是否为master controller if !IsMasterRegion(cfg) { @@ -95,6 +98,8 @@ func checkAndStartMasterFunctions( recorderResource := recorder.GetSingletonResource() domainChecker := resoureservice.NewDomainCheck(ctx) prometheus := prometheus.GetSingleton() + prometheus.APPLabelLayoutUpdater.Init(ctx, &cfg.PrometheusCfg) + httpService := http.GetSingleton() masterController := "" @@ -146,6 +151,7 @@ func checkAndStartMasterFunctions( domainChecker.Start() prometheus.Encoder.Start() + prometheus.APPLabelLayoutUpdater.Start() if cfg.DFWebService.Enabled { httpService.TaskManager.Start(ctx, cfg.FPermit, cfg.RedisCfg) @@ -176,6 +182,7 @@ func checkAndStartMasterFunctions( recorderResource.IDManager.Stop() prometheus.Encoder.Stop() + prometheus.APPLabelLayoutUpdater.Stop() if cfg.DFWebService.Enabled { httpService.TaskManager.Stop() diff --git a/server/controller/prometheus/cache/metric_label.go b/server/controller/prometheus/cache/metric_label.go index 01395d029e24..37a2400aca34 100644 --- a/server/controller/prometheus/cache/metric_label.go +++ b/server/controller/prometheus/cache/metric_label.go @@ -72,6 +72,10 @@ func (ml *metricLabel) Add(batch []MetricLabelDetailKey) { } } +func (ml *metricLabel) GetMetricLabelDetailKeys() mapset.Set[MetricLabelDetailKey] { + return ml.metricLabelDetailKeys +} + func (ml *metricLabel) refresh(args ...interface{}) error { metricLabels, err := ml.load() if err != nil { diff --git a/server/controller/prometheus/cache/metric_target.go b/server/controller/prometheus/cache/metric_target.go index b064f87e638c..7a9152bfedff 100644 --- a/server/controller/prometheus/cache/metric_target.go +++ b/server/controller/prometheus/cache/metric_target.go @@ -120,6 +120,10 @@ func (mt *metricTarget) IfLabelIsTargetType(mn, ln string) bool { return false } +func (mt *metricTarget) GetMetricNameToTargetIDs() map[string]mapset.Set[int] { + return mt.metricNameToTargetIDs.Get() +} + func (mt *metricTarget) refresh(args ...interface{}) error { mts, err := mt.load() if err != nil { diff --git a/server/controller/prometheus/cache/target.go b/server/controller/prometheus/cache/target.go index ca1a248bbe2d..4437348b7a25 100644 --- a/server/controller/prometheus/cache/target.go +++ b/server/controller/prometheus/cache/target.go @@ -156,6 +156,10 @@ func (t *target) Add(batch []*controller.PrometheusTarget) { } } +func (t *target) GetTargetIDToLabelNames() map[int]mapset.Set[string] { + return t.targetIDToLabelNames.Get() +} + func (t *target) refresh(args ...interface{}) error { recorderTargets, selfTargets, err := t.load() if err != nil { @@ -212,5 +216,5 @@ func (t *target) load() (recorderTargets, selfTargets []*mysql.PrometheusTarget, } func (t *target) dedup(ids []int) error { - return mysql.Db.Where("id in (?)", ids).Delete(&mysql.PrometheusTarget{}).Error // TODO 强删？ + return mysql.Db.Where("id in (?)", ids).Delete(&mysql.PrometheusTarget{}).Error } diff --git a/server/controller/prometheus/config/config.go b/server/controller/prometheus/config/config.go index d8dff89cc9bd..76850101aa1c 100644 --- a/server/controller/prometheus/config/config.go +++ b/server/controller/prometheus/config/config.go @@ -21,4 +21,5 @@ type Config struct { EncoderCacheRefreshInterval int `default:"3600" yaml:"encoder_cache_refresh_interval"` ResourceMaxID0 int `default:"64000" yaml:"resource_max_id_0"` ResourceMaxID1 int `default:"499999" yaml:"resource_max_id_1"` + APPLabelIndexMax int `default:"256" yaml:"app_label_index"` } diff --git a/server/controller/prometheus/encoder/encoder.go b/server/controller/prometheus/encoder/encoder.go index d8e68ac6c7c2..71af88163bbe 100644 --- a/server/controller/prometheus/encoder/encoder.go +++ b/server/controller/prometheus/encoder/encoder.go @@ -47,7 +47,7 @@ type Encoder struct { metricName *metricName labelName *labelName labelValue *labelValue - labelLayout *labelLayout + LabelLayout *labelLayout label *label metricLabel *metricLabel metricTarget *metricTarget @@ -70,7 +70,7 @@ func (e *Encoder) Init(ctx context.Context, cfg *prometheuscfg.Config) { e.labelName = newLabelName(cfg.ResourceMaxID0) e.labelValue = newLabelValue(cfg.ResourceMaxID1) e.label = newLabel() - e.labelLayout = newLabelLayout() + e.LabelLayout = newLabelLayout(cfg) e.metricLabel = newMetricLabel(e.label) e.target = newTarget(cfg.ResourceMaxID1) e.metricTarget = newMetricTarget(e.target) @@ -90,6 +90,7 @@ func (e *Encoder) Start() error { e.refresh() go func() { ticker := time.NewTicker(e.refreshInterval) + defer ticker.Stop() for { select { case <-e.ctx.Done(): @@ -113,16 +114,19 @@ func (e *Encoder) Stop() { } func (e *Encoder) refresh() error { + log.Info("prometheus encoder refresh started") e.label.refresh() eg := &errgroup.Group{} AppendErrGroup(eg, e.metricName.refresh) AppendErrGroup(eg, e.labelName.refresh) AppendErrGroup(eg, e.labelValue.refresh) - AppendErrGroup(eg, e.labelLayout.refresh) + AppendErrGroup(eg, e.LabelLayout.refresh) AppendErrGroup(eg, e.metricLabel.refresh) AppendErrGroup(eg, e.metricTarget.refresh) AppendErrGroup(eg, e.target.refresh) - return eg.Wait() + err := eg.Wait() + log.Info("prometheus encoder refresh completed") + return err } func (e *Encoder) Encode(req *controller.SyncPrometheusRequest) (*controller.SyncPrometheusResponse, error) { @@ -181,7 +185,7 @@ func (e *Encoder) encodeLabelValue(args ...interface{}) error { func (e *Encoder) encodeLabelIndex(args ...interface{}) error { resp := args[0].(*controller.SyncPrometheusResponse) layouts := args[1].([]*controller.PrometheusMetricAPPLabelLayoutRequest) - lis, err := e.labelLayout.encode(layouts) + lis, err := e.LabelLayout.encode(layouts) if err != nil { return err } diff --git a/server/controller/prometheus/encoder/id_allocator.go b/server/controller/prometheus/encoder/id_allocator.go index 7229193869ef..2c907efef6fa 100644 --- a/server/controller/prometheus/encoder/id_allocator.go +++ b/server/controller/prometheus/encoder/id_allocator.go @@ -30,6 +30,11 @@ type sorter interface { getUsedIDSet(sortedUsableIDs []int, inUseIDSet mapset.Set[int]) mapset.Set[int] } +type rawDataProvider interface { + load() (mapset.Set[int], error) + check([]int) ([]int, error) +} + type idAllocator struct { resourceType string min int @@ -73,14 +78,20 @@ func (ia *idAllocator) allocate(count int) (ids []int, err error) { return } +func (p *idAllocator) recycle(ids []int) { + p.sorter.sort(ids) + p.usableIDs = append(p.usableIDs, ids...) + log.Infof("recycle %s ids: %v", p.resourceType, ids) +} + func (ia *idAllocator) refresh() error { - log.Infof("refresh %s id pools started", ia.resourceType) + log.Debugf("refresh %s id pools started", ia.resourceType) inUseIDSet, err := ia.rawDataProvider.load() if err != nil { return err } ia.usableIDs = ia.getSortedUsableIDs(ia.getAllIDSet(), inUseIDSet) - log.Infof("refresh %s id pools (usable ids count: %d) completed", ia.resourceType, len(ia.usableIDs)) + log.Debugf("refresh %s id pools (usable ids count: %d) completed", ia.resourceType, len(ia.usableIDs)) return nil } @@ -155,11 +166,6 @@ func (ia *ascIDAllocator) sort(ints []int) { sort.Ints(ints) } -type rawDataProvider interface { - load() (mapset.Set[int], error) - check([]int) ([]int, error) -} - type descIDAllocator struct { idAllocator } diff --git a/server/controller/prometheus/encoder/label_layout.go b/server/controller/prometheus/encoder/label_layout.go index dfe88d697a1e..11bbdd9ebac8 100644 --- a/server/controller/prometheus/encoder/label_layout.go +++ b/server/controller/prometheus/encoder/label_layout.go @@ -19,91 +19,202 @@ package encoder import ( "sync" + mapset "github.com/deckarep/golang-set/v2" "github.com/golang/protobuf/proto" "github.com/deepflowio/deepflow/message/controller" "github.com/deepflowio/deepflow/server/controller/db/mysql" - "github.com/deepflowio/deepflow/server/controller/prometheus/cache" + prometheuscfg "github.com/deepflowio/deepflow/server/controller/prometheus/config" ) +type indexAllocator struct { + lock sync.Mutex + resourceType string + metricName string + strToIdx map[string]int + ascIDAllocator +} + +func newIndexAllocator(metricName string, max int) *indexAllocator { + ln := &indexAllocator{ + resourceType: "app_label_index", + metricName: metricName, + strToIdx: make(map[string]int), + } + ln.ascIDAllocator = newAscIDAllocator(ln.resourceType, 1, max) + ln.rawDataProvider = ln + return ln +} + +func (ia *indexAllocator) refresh(labelNameToIdx map[string]int) error { + ia.lock.Lock() + defer ia.lock.Unlock() + ia.strToIdx = labelNameToIdx + return ia.ascIDAllocator.refresh() +} + +func (ia *indexAllocator) load() (ids mapset.Set[int], err error) { + inUseIdxSet := mapset.NewSet[int]() + for _, idx := range ia.strToIdx { + inUseIdxSet.Add(idx) + } + return inUseIdxSet, nil +} + +func (ia *indexAllocator) encode(strs []string) ([]*controller.PrometheusMetricAPPLabelLayout, error) { + ia.lock.Lock() + defer ia.lock.Unlock() + + resp := make([]*controller.PrometheusMetricAPPLabelLayout, 0) + var dbToAdd []*mysql.PrometheusMetricAPPLabelLayout + for i := range strs { + str := strs[i] + if idx, ok := ia.strToIdx[str]; ok { + resp = append(resp, &controller.PrometheusMetricAPPLabelLayout{MetricName: &ia.metricName, AppLabelName: &str, AppLabelColumnIndex: proto.Uint32(uint32(idx))}) + continue + } + dbToAdd = append(dbToAdd, &mysql.PrometheusMetricAPPLabelLayout{MetricName: ia.metricName, APPLabelName: str}) + } + if len(dbToAdd) == 0 { + return resp, nil + } + idxs, err := ia.allocate(len(dbToAdd)) + if err != nil { + return nil, err + } + for i := range idxs { + dbToAdd[i].APPLabelColumnIndex = uint8(idxs[i]) + } + err = addBatch(dbToAdd, ia.resourceType) + if err != nil { + log.Errorf("add %s error: %s", ia.resourceType, err.Error()) + return nil, err + } + for i := range dbToAdd { + idx := dbToAdd[i].APPLabelColumnIndex + str := dbToAdd[i].APPLabelName + ia.strToIdx[str] = int(idx) + resp = append(resp, &controller.PrometheusMetricAPPLabelLayout{MetricName: &ia.metricName, AppLabelName: &str, AppLabelColumnIndex: proto.Uint32(uint32(idx))}) + } + return resp, nil +} + +func (ia *indexAllocator) check(ids []int) (inUseIDs []int, err error) { + var dbItems []*mysql.PrometheusMetricAPPLabelLayout + err = mysql.Db.Where("metric_name = ? AND id IN (?)", ia.metricName, ids).Find(&dbItems).Error + if err != nil { + log.Errorf("db query %s failed: %v", ia.resourceType, err) + return + } + if len(dbItems) != 0 { + for _, item := range dbItems { + inUseIDs = append(inUseIDs, int(item.APPLabelColumnIndex)) + } + log.Infof("%s ids: %+v are in use.", ia.resourceType, inUseIDs) + } + return +} + +func (ia *indexAllocator) release(ids []int) error { + err := mysql.Db.Where("metric_name = ? AND app_label_column_index IN (?)", ia.metricName, ids).Delete(&mysql.PrometheusMetricAPPLabelLayout{}).Error + if err != nil { + return err + } + ia.recycle(ids) + return nil +} + type labelLayout struct { - lock sync.Mutex - resourceType string - layoutKeyToIndex map[cache.LayoutKey]uint8 - metricNameToMaxIndex map[string]uint8 + appLabelIndexMax int + + lock sync.Mutex + resourceType string + metricNameToIdxAllocator map[string]*indexAllocator } -func newLabelLayout() *labelLayout { +func newLabelLayout(cfg *prometheuscfg.Config) *labelLayout { return &labelLayout{ - resourceType: "metric_app_label_layout", - layoutKeyToIndex: make(map[cache.LayoutKey]uint8), - metricNameToMaxIndex: make(map[string]uint8), + appLabelIndexMax: cfg.APPLabelIndexMax, + + resourceType: "metric_app_label_layout", + metricNameToIdxAllocator: make(map[string]*indexAllocator), } } func (ll *labelLayout) refresh(args ...interface{}) error { - ll.lock.Lock() - defer ll.lock.Unlock() - var items []*mysql.PrometheusMetricAPPLabelLayout err := mysql.Db.Find(&items).Error if err != nil { return err } - + mnToLnIdx := make(map[string]map[string]int) for _, item := range items { - ll.layoutKeyToIndex[cache.NewLayoutKey(item.MetricName, item.APPLabelName)] = item.APPLabelColumnIndex - if ll.metricNameToMaxIndex[item.MetricName] < item.APPLabelColumnIndex { - ll.metricNameToMaxIndex[item.MetricName] = item.APPLabelColumnIndex + if _, ok := mnToLnIdx[item.MetricName]; !ok { + mnToLnIdx[item.MetricName] = make(map[string]int) + } + mnToLnIdx[item.MetricName][item.APPLabelName] = int(item.APPLabelColumnIndex) + } + + for mn, lnToIdx := range mnToLnIdx { + ll.metricNameToIdxAllocator[mn], _ = ll.createIndexAllocatorIfNotExists(mn) + ll.metricNameToIdxAllocator[mn].refresh(lnToIdx) + } + for mn := range ll.metricNameToIdxAllocator { + if _, ok := mnToLnIdx[mn]; !ok { + ll.metricNameToIdxAllocator[mn].refresh(make(map[string]int)) } } return nil } func (ll *labelLayout) encode(req []*controller.PrometheusMetricAPPLabelLayoutRequest) ([]*controller.PrometheusMetricAPPLabelLayout, error) { - ll.lock.Lock() - defer ll.lock.Unlock() - - resp := make([]*controller.PrometheusMetricAPPLabelLayout, 0, len(req)) - - tmpMetricNameToMaxIndex := make(map[string]uint8) - for k, v := range ll.metricNameToMaxIndex { - tmpMetricNameToMaxIndex[k] = v - } - var dbToAdd []*mysql.PrometheusMetricAPPLabelLayout - for _, v := range req { - mn := v.GetMetricName() - ln := v.GetAppLabelName() - if idx, ok := ll.layoutKeyToIndex[cache.NewLayoutKey(mn, ln)]; ok { - resp = append(resp, &controller.PrometheusMetricAPPLabelLayout{ - MetricName: &mn, - AppLabelName: &ln, - AppLabelColumnIndex: proto.Uint32(uint32(idx)), - }) - continue + mnToLNs := make(map[string][]string) + for _, item := range req { + mn := item.GetMetricName() + if _, ok := mnToLNs[mn]; !ok { + mnToLNs[mn] = make([]string, 0) } - dbToAdd = append(dbToAdd, &mysql.PrometheusMetricAPPLabelLayout{ - MetricName: mn, - APPLabelName: ln, - APPLabelColumnIndex: tmpMetricNameToMaxIndex[mn] + 1, - }) - tmpMetricNameToMaxIndex[mn]++ - } - err := addBatch(dbToAdd, ll.resourceType) - if err != nil { - log.Errorf("add %s error: %s", ll.resourceType, err.Error()) - return nil, err + mnToLNs[mn] = append(mnToLNs[mn], item.GetAppLabelName()) } - for _, l := range dbToAdd { - resp = append(resp, &controller.PrometheusMetricAPPLabelLayout{ - MetricName: &l.MetricName, - AppLabelName: &l.APPLabelName, - AppLabelColumnIndex: proto.Uint32(uint32(l.APPLabelColumnIndex)), - }) - ll.layoutKeyToIndex[cache.NewLayoutKey(l.MetricName, l.APPLabelName)] = l.APPLabelColumnIndex - if ll.metricNameToMaxIndex[l.MetricName] < l.APPLabelColumnIndex { - ll.metricNameToMaxIndex[l.MetricName] = l.APPLabelColumnIndex + + resp := make([]*controller.PrometheusMetricAPPLabelLayout, 0) + for mn, lns := range mnToLNs { + layouts, err := ll.SingleEncode(mn, lns) + if err != nil { + return nil, err } + resp = append(resp, layouts...) } return resp, nil } + +func (ll *labelLayout) createIndexAllocatorIfNotExists(metricName string) (*indexAllocator, error) { + ll.lock.Lock() + defer ll.lock.Unlock() + if allocator, ok := ll.metricNameToIdxAllocator[metricName]; ok { + return allocator, nil + } + allocator := newIndexAllocator(metricName, ll.appLabelIndexMax) + return allocator, nil +} + +func (ll *labelLayout) getIndexAllocator(metricName string) (*indexAllocator, bool) { + ll.lock.Lock() + defer ll.lock.Unlock() + allocator, ok := ll.metricNameToIdxAllocator[metricName] + return allocator, ok +} + +func (ll *labelLayout) SingleEncode(metricName string, labelNames []string) ([]*controller.PrometheusMetricAPPLabelLayout, error) { + log.Infof("encode metric: %s app label names: %v", metricName, labelNames) + ll.metricNameToIdxAllocator[metricName], _ = ll.createIndexAllocatorIfNotExists(metricName) + return ll.metricNameToIdxAllocator[metricName].encode(labelNames) +} + +func (ll *labelLayout) SingleRelease(metricName string, indexes []int) error { + log.Infof("recycle metric: %s indexes: %v", metricName, indexes) + if allocator, ok := ll.getIndexAllocator(metricName); ok { + return allocator.release(indexes) + } + return nil +} diff --git a/server/controller/prometheus/prometheus.go b/server/controller/prometheus/prometheus.go index 806ef384da68..2cbaa992ddbd 100644 --- a/server/controller/prometheus/prometheus.go +++ b/server/controller/prometheus/prometheus.go @@ -29,15 +29,17 @@ var ( ) type PrometheusManager struct { - Encoder *encoder.Encoder - SynchronizerCache *cache.Cache + Encoder *encoder.Encoder + SynchronizerCache *cache.Cache + APPLabelLayoutUpdater *APPLabelLayoutUpdater } func GetSingleton() *PrometheusManager { prometheusManagerOnce.Do(func() { prometheusManager = &PrometheusManager{ - Encoder: encoder.GetSingleton(), - SynchronizerCache: cache.GetSingleton(), + Encoder: encoder.GetSingleton(), + SynchronizerCache: cache.GetSingleton(), + APPLabelLayoutUpdater: GetAPPLabelLayoutUpdater(), } }) return prometheusManager diff --git a/server/controller/prometheus/updater.go b/server/controller/prometheus/updater.go new file mode 100644 index 000000000000..d4dfedb37329 --- /dev/null +++ b/server/controller/prometheus/updater.go @@ -0,0 +1,185 @@ +/** + * Copyright (c) 2023 Yunshan Networks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package prometheus + +import ( + "context" + "sync" + "time" + + mapset "github.com/deckarep/golang-set/v2" + + "github.com/deepflowio/deepflow/server/controller/db/mysql" + "github.com/deepflowio/deepflow/server/controller/prometheus/cache" + "github.com/deepflowio/deepflow/server/controller/prometheus/common" + prometheuscfg "github.com/deepflowio/deepflow/server/controller/prometheus/config" + "github.com/deepflowio/deepflow/server/controller/prometheus/encoder" +) + +var ( + appLabelLayoutUpdaterOnce sync.Once + appLabelLayoutUpdater *APPLabelLayoutUpdater +) + +type APPLabelLayoutUpdater struct { + ctx context.Context + cancel context.CancelFunc + + refreshInterval time.Duration + + encoder *encoder.Encoder +} + +func GetAPPLabelLayoutUpdater() *APPLabelLayoutUpdater { + appLabelLayoutUpdaterOnce.Do(func() { + appLabelLayoutUpdater = &APPLabelLayoutUpdater{ + encoder: encoder.GetSingleton(), + } + }) + return appLabelLayoutUpdater +} + +func (au *APPLabelLayoutUpdater) Init(ctx context.Context, cfg *prometheuscfg.Config) { + au.ctx, au.cancel = context.WithCancel(ctx) + au.refreshInterval = time.Duration(cfg.SynchronizerCacheRefreshInterval) * time.Second +} + +func (e *APPLabelLayoutUpdater) Start() error { + log.Info("prometheus app label layout updater started") + // it depends on cache module data, so it should not be refresh immediately. + go func() { + ticker := time.NewTicker(e.refreshInterval) + defer ticker.Stop() + for { + select { + case <-e.ctx.Done(): + return + case <-ticker.C: + e.refresh() + } + } + }() + return nil +} + +func (au *APPLabelLayoutUpdater) Stop() { + if au.cancel != nil { + au.cancel() + } + log.Info("prometheus app label layout updater stopped") +} + +func (au *APPLabelLayoutUpdater) refresh() error { + log.Info("prometheus app label layout refresh started") + td := newToolDataSet() + err := td.load() + if err != nil { + return err + } + + for mn, lns := range td.metricNameToLabelNames { + newAPPLabelNames := lns.Difference(mapset.NewSet([]string{common.TargetLabelInstance, common.TargetLabelJob}...)) + if targetLNs, ok := td.metricNameToTargetLabelNames[mn]; ok { + newAPPLabelNames = newAPPLabelNames.Difference(targetLNs) + } + oldAPPLabelNames, ok := td.metricNameToAPPLabelNames[mn] + if !ok { + oldAPPLabelNames = mapset.NewSet[string]() + } + + if appLabelNamesToEncode := newAPPLabelNames.Difference(oldAPPLabelNames).ToSlice(); len(appLabelNamesToEncode) > 0 { + _, err = au.encoder.LabelLayout.SingleEncode(mn, appLabelNamesToEncode) + if err != nil { + log.Debugf("encode metric: %s labels: %#v failed: %s", mn, appLabelNamesToEncode, err.Error()) + } + } + + indexesToRecycle := make([]int, 0) + for _, item := range oldAPPLabelNames.Difference(newAPPLabelNames).ToSlice() { + if idx, ok := td.layoutKeyToIndex[cache.NewLayoutKey(mn, item)]; ok { + indexesToRecycle = append(indexesToRecycle, int(idx)) + } + } + if len(indexesToRecycle) > 0 { + err = au.encoder.LabelLayout.SingleRelease(mn, indexesToRecycle) + if err != nil { + log.Debugf("recycle metric: %s label indexes: %#v failed: %s", mn, indexesToRecycle, err.Error()) + } + } + } + log.Info("prometheus app label layout refresh completed") + return err +} + +type toolData struct { + cache *cache.Cache + + metricNameToLabelNames map[string]mapset.Set[string] + metricNameToTargetLabelNames map[string]mapset.Set[string] + metricNameToAPPLabelNames map[string]mapset.Set[string] + layoutKeyToIndex map[cache.LayoutKey]uint8 +} + +func newToolDataSet() *toolData { + return &toolData{ + cache: cache.GetSingleton(), + + metricNameToLabelNames: make(map[string]mapset.Set[string]), + metricNameToTargetLabelNames: make(map[string]mapset.Set[string]), + metricNameToAPPLabelNames: make(map[string]mapset.Set[string]), + layoutKeyToIndex: make(map[cache.LayoutKey]uint8), + } +} + +func (td *toolData) load() error { + metricNameToTargetIDs := td.cache.MetricTarget.GetMetricNameToTargetIDs() + targetIDToLabelNames := td.cache.Target.GetTargetIDToLabelNames() + for mn, tids := range metricNameToTargetIDs { + for _, tid := range tids.ToSlice() { + lns, ok := targetIDToLabelNames[tid] + if !ok { + continue + } + if _, ok := td.metricNameToTargetLabelNames[mn]; !ok { + td.metricNameToTargetLabelNames[mn] = mapset.NewSet[string]() + } + td.metricNameToTargetLabelNames[mn] = td.metricNameToTargetLabelNames[mn].Union(lns) + } + } + + var layouts []*mysql.PrometheusMetricAPPLabelLayout + err := mysql.Db.Find(&layouts).Error + if err != nil { + return err + } + for _, item := range layouts { + if _, ok := td.metricNameToAPPLabelNames[item.MetricName]; !ok { + td.metricNameToAPPLabelNames[item.MetricName] = mapset.NewSet[string]() + } + td.metricNameToAPPLabelNames[item.MetricName].Add(item.APPLabelName) + td.layoutKeyToIndex[cache.NewLayoutKey(item.MetricName, item.APPLabelName)] = item.APPLabelColumnIndex + } + + metricLabelKeys := td.cache.MetricLabel.GetMetricLabelDetailKeys().ToSlice() + for _, item := range metricLabelKeys { + if _, ok := td.metricNameToLabelNames[item.MetricName]; !ok { + td.metricNameToLabelNames[item.MetricName] = mapset.NewSet[string]() + } + td.metricNameToLabelNames[item.MetricName].Add(item.LabelName) + } + return nil +}