From 221365cfe6bc4a97a27d7ee5a2b908709697322b Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 22 Sep 2023 12:31:10 +0100 Subject: [PATCH 1/3] remove function_name from hydrateMetadata --- plugin/query_data_rate_limiters.go | 3 --- plugin/row_data.go | 1 - plugin/row_with_metadata.go | 1 - 3 files changed, 5 deletions(-) diff --git a/plugin/query_data_rate_limiters.go b/plugin/query_data_rate_limiters.go index 438ef0c3..4537a6b2 100644 --- a/plugin/query_data_rate_limiters.go +++ b/plugin/query_data_rate_limiters.go @@ -141,7 +141,6 @@ func (d *QueryData) resolveListRateLimiters() error { func (d *QueryData) setListLimiterMetadata(fetchDelay time.Duration) { fetchMetadata := &hydrateMetadata{ - FuncName: d.listHydrate.Name, RateLimiters: d.fetchLimiters.rateLimiter.LimiterNames(), ScopeValues: d.fetchLimiters.rateLimiter.ScopeValues, DelayMs: fetchDelay.Milliseconds(), @@ -152,7 +151,6 @@ func (d *QueryData) setListLimiterMetadata(fetchDelay time.Duration) { } else { d.fetchMetadata = &hydrateMetadata{ Type: string(fetchTypeList), - FuncName: d.childHydrate.Name, RateLimiters: d.fetchLimiters.childListRateLimiter.LimiterNames(), ScopeValues: d.fetchLimiters.childListRateLimiter.ScopeValues, } @@ -164,7 +162,6 @@ func (d *QueryData) setListLimiterMetadata(fetchDelay time.Duration) { func (d *QueryData) setGetLimiterMetadata(fetchDelay time.Duration) { d.fetchMetadata = &hydrateMetadata{ Type: string(fetchTypeGet), - FuncName: d.Table.Get.namedHydrate.Name, RateLimiters: d.fetchLimiters.rateLimiter.LimiterNames(), ScopeValues: d.fetchLimiters.rateLimiter.ScopeValues, DelayMs: fetchDelay.Milliseconds(), diff --git a/plugin/row_data.go b/plugin/row_data.go index 14ddbbbe..843e03b6 100644 --- a/plugin/row_data.go +++ b/plugin/row_data.go @@ -88,7 +88,6 @@ func (r *rowData) startAllHydrateCalls(rowDataCtx context.Context, rowQueryData // store the call metadata r.hydrateMetadata = append(r.hydrateMetadata, &hydrateMetadata{ Type: "hydrate", - FuncName: hydrateFuncName, ScopeValues: call.rateLimiter.ScopeValues, RateLimiters: call.rateLimiter.LimiterNames(), DelayMs: rateLimitDelay.Milliseconds(), diff --git a/plugin/row_with_metadata.go b/plugin/row_with_metadata.go index 41c5fa98..b90a6fac 100644 --- a/plugin/row_with_metadata.go +++ b/plugin/row_with_metadata.go @@ -18,7 +18,6 @@ var ValidDiagnosticsLevels = map[string]struct{}{ type hydrateMetadata struct { Type string `json:"type"` - FuncName string `json:"function_name"` ScopeValues map[string]string `json:"scope_values"` RateLimiters []string `json:"rate_limiters"` DelayMs int64 `json:"rate_limiter_delay_ms"` From 6dd1d0b9da1c53ca9647d596c503fa8bab29a500 Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 22 Sep 2023 15:29:37 +0100 Subject: [PATCH 2/3] deprecate STEAMPIPE_DIAGNOSTICS_LEVEL --- plugin/env.go | 51 +++++++++++++++++++++++++------------ plugin/row_with_metadata.go | 18 +------------ 2 files changed, 36 insertions(+), 33 deletions(-) diff --git a/plugin/env.go b/plugin/env.go index 12f39067..85ea1c89 100644 --- a/plugin/env.go +++ b/plugin/env.go @@ -1,21 +1,31 @@ package plugin import ( + "fmt" + "golang.org/x/exp/maps" "log" - "math" "os" "strconv" + "strings" ) const ( envMaxConcurrentConnection = "STEAMPIPE_MAX_CONCURRENT_CONNECTIONS" - envMaxMemoryMb = "STEAMPIPE_MAX_MEMORY_MB" envFreeMemInterval = "STEAMPIPE_FREE_MEM_INTERVAL" - defaultMaxConcurrentConnections = 25 // default to 25 concurrent connections - defaultMaxMemoryMb = math.MaxInt64 // default to no memory limit - defaultFreeMemInterval = 100 // default to freeing memory every 100 rows + defaultMaxConcurrentConnections = 25 // default to 25 concurrent connections + defaultFreeMemInterval = 100 // default to freeing memory every 100 rows + + EnvDiagnosticsLevel = "STEAMPIPE_DIAGNOSTIC_LEVEL" + EnvLegacyDiagnosticsLevel = "STEAMPIPE_DIAGNOSTICS_LEVEL" + DiagnosticsAll = "ALL" + DiagnosticsNone = "NONE" ) +var ValidDiagnosticsLevels = map[string]struct{}{ + DiagnosticsAll: {}, + DiagnosticsNone: {}, +} + func getMaxConcurrentConnections() int { maxConcurrentConnections, _ := strconv.Atoi(os.Getenv(envMaxConcurrentConnection)) if maxConcurrentConnections == 0 { @@ -25,17 +35,6 @@ func getMaxConcurrentConnections() int { return maxConcurrentConnections } -func GetMaxMemoryBytes() int64 { - maxMemoryMb, _ := strconv.Atoi(os.Getenv(envMaxMemoryMb)) - if maxMemoryMb == 0 { - log.Printf("[TRACE] No memory limit set") - maxMemoryMb = defaultMaxMemoryMb - } else { - log.Printf("[TRACE] Setting max memory %dMb", maxMemoryMb) - } - return int64(1024 * 1024 * maxMemoryMb) -} - func GetFreeMemInterval() int64 { freeMemInterval := defaultFreeMemInterval intervalEnv, ok := os.LookupEnv(envFreeMemInterval) @@ -48,3 +47,23 @@ func GetFreeMemInterval() int64 { return int64(freeMemInterval) } + +func loadDiagnosticsEnvVar() string { + // load both the legacy and current diagnostics env vars + diagnostics := strings.ToUpper(os.Getenv(EnvLegacyDiagnosticsLevel)) + if newDiagnostics, isSet := os.LookupEnv(EnvDiagnosticsLevel); isSet { + diagnostics = strings.ToUpper(newDiagnostics) + } + return diagnostics +} + +func ValidateDiagnosticsEnvVar() error { + diagnostics := loadDiagnosticsEnvVar() + if diagnostics == "" { + return nil + } + if _, isValid := ValidDiagnosticsLevels[strings.ToUpper(diagnostics)]; !isValid { + return fmt.Errorf(`invalid value of '%s' (%s), must be one of: %s`, EnvDiagnosticsLevel, diagnostics, strings.Join(maps.Keys(ValidDiagnosticsLevels), ", ")) + } + return nil +} diff --git a/plugin/row_with_metadata.go b/plugin/row_with_metadata.go index b90a6fac..5813d9e6 100644 --- a/plugin/row_with_metadata.go +++ b/plugin/row_with_metadata.go @@ -1,21 +1,5 @@ package plugin -import ( - "os" - "strings" -) - -const ( - EnvDiagnosticsLevel = "STEAMPIPE_DIAGNOSTICS_LEVEL" - DiagnosticsAll = "ALL" - DiagnosticsNone = "NONE" -) - -var ValidDiagnosticsLevels = map[string]struct{}{ - DiagnosticsAll: {}, - DiagnosticsNone: {}, -} - type hydrateMetadata struct { Type string `json:"type"` ScopeValues map[string]string `json:"scope_values"` @@ -37,7 +21,7 @@ func newRowCtxData(rd *rowData) *rowCtxData { Connection: d.Connection.Name, } - if strings.ToUpper(os.Getenv(EnvDiagnosticsLevel)) == DiagnosticsAll { + if loadDiagnosticsEnvVar() == DiagnosticsAll { calls := append([]*hydrateMetadata{d.fetchMetadata}, rd.hydrateMetadata...) if d.parentHydrateMetadata != nil { calls = append([]*hydrateMetadata{d.parentHydrateMetadata}, calls...) From 8330b9782864cd5f1dd05ea54185fbeeeb9ea879 Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 22 Sep 2023 16:07:27 +0100 Subject: [PATCH 3/3] Revert "remove function_name from hydrateMetadata" This reverts commit 221365cfe6bc4a97a27d7ee5a2b908709697322b. --- plugin/query_data_rate_limiters.go | 3 +++ plugin/row_data.go | 1 + plugin/row_with_metadata.go | 1 + 3 files changed, 5 insertions(+) diff --git a/plugin/query_data_rate_limiters.go b/plugin/query_data_rate_limiters.go index 4537a6b2..438ef0c3 100644 --- a/plugin/query_data_rate_limiters.go +++ b/plugin/query_data_rate_limiters.go @@ -141,6 +141,7 @@ func (d *QueryData) resolveListRateLimiters() error { func (d *QueryData) setListLimiterMetadata(fetchDelay time.Duration) { fetchMetadata := &hydrateMetadata{ + FuncName: d.listHydrate.Name, RateLimiters: d.fetchLimiters.rateLimiter.LimiterNames(), ScopeValues: d.fetchLimiters.rateLimiter.ScopeValues, DelayMs: fetchDelay.Milliseconds(), @@ -151,6 +152,7 @@ func (d *QueryData) setListLimiterMetadata(fetchDelay time.Duration) { } else { d.fetchMetadata = &hydrateMetadata{ Type: string(fetchTypeList), + FuncName: d.childHydrate.Name, RateLimiters: d.fetchLimiters.childListRateLimiter.LimiterNames(), ScopeValues: d.fetchLimiters.childListRateLimiter.ScopeValues, } @@ -162,6 +164,7 @@ func (d *QueryData) setListLimiterMetadata(fetchDelay time.Duration) { func (d *QueryData) setGetLimiterMetadata(fetchDelay time.Duration) { d.fetchMetadata = &hydrateMetadata{ Type: string(fetchTypeGet), + FuncName: d.Table.Get.namedHydrate.Name, RateLimiters: d.fetchLimiters.rateLimiter.LimiterNames(), ScopeValues: d.fetchLimiters.rateLimiter.ScopeValues, DelayMs: fetchDelay.Milliseconds(), diff --git a/plugin/row_data.go b/plugin/row_data.go index 843e03b6..14ddbbbe 100644 --- a/plugin/row_data.go +++ b/plugin/row_data.go @@ -88,6 +88,7 @@ func (r *rowData) startAllHydrateCalls(rowDataCtx context.Context, rowQueryData // store the call metadata r.hydrateMetadata = append(r.hydrateMetadata, &hydrateMetadata{ Type: "hydrate", + FuncName: hydrateFuncName, ScopeValues: call.rateLimiter.ScopeValues, RateLimiters: call.rateLimiter.LimiterNames(), DelayMs: rateLimitDelay.Milliseconds(), diff --git a/plugin/row_with_metadata.go b/plugin/row_with_metadata.go index 5813d9e6..3e618bc5 100644 --- a/plugin/row_with_metadata.go +++ b/plugin/row_with_metadata.go @@ -2,6 +2,7 @@ package plugin type hydrateMetadata struct { Type string `json:"type"` + FuncName string `json:"function_name"` ScopeValues map[string]string `json:"scope_values"` RateLimiters []string `json:"rate_limiters"` DelayMs int64 `json:"rate_limiter_delay_ms"`