API and IGP memory usage by host

API & IGP by host
VKCOM · Oct 26, 2024 · 0c7e731 · 0c7e731
1 parent df0208c
commit 0c7e731
Show file tree

Hide file tree

Showing 4 changed files with 215 additions and 15 deletions.
diff --git a/internal/aggregator/ingress_proxy2.go b/internal/aggregator/ingress_proxy2.go
@@ -9,6 +9,7 @@ import (
 	"log"
 	"net"
 	"os"
+	"runtime"
 	"strings"
 	"sync"
 	"time"
@@ -114,6 +115,27 @@ func RunIngressProxy2(ctx context.Context, agent *agent.Agent, config ConfigIngr
 		statshouse.StartRegularMeasurement(func(client *statshouse.Client) {
 			p.сonnectionCountMetric.Count(float64(p.сonnectionCount.Load()))
 			p.requestMemoryMetric.Value(float64(p.requestMemory.Load()))
+			var vmSize, vmRSS float64
+			if st, _ := srvfunc.GetMemStat(0); st != nil {
+				vmSize = float64(st.Size)
+				vmRSS = float64(st.Res)
+			}
+			client.Value(format.BuiltinMetricNameProxyVmSize, statshouse.Tags{1: srvfunc.HostnameForStatshouse()}, vmSize)
+			client.Value(format.BuiltinMetricNameProxyVmRSS, statshouse.Tags{1: srvfunc.HostnameForStatshouse()}, vmRSS)
+			var memStats runtime.MemStats
+			runtime.ReadMemStats(&memStats)
+			client.Value(format.BuiltinMetricNameProxyHeapAlloc, statshouse.Tags{1: srvfunc.HostnameForStatshouse()}, float64(memStats.HeapAlloc))
+			client.Value(format.BuiltinMetricNameProxyHeapSys, statshouse.Tags{1: srvfunc.HostnameForStatshouse()}, float64(memStats.HeapSys))
+			client.Value(format.BuiltinMetricNameProxyHeapIdle, statshouse.Tags{1: srvfunc.HostnameForStatshouse()}, float64(memStats.HeapIdle))
+			client.Value(format.BuiltinMetricNameProxyHeapInuse, statshouse.Tags{1: srvfunc.HostnameForStatshouse()}, float64(memStats.HeapInuse))
+			//-- TODO: remove when deployed
+			client.Value("igp_vm_size", statshouse.Tags{1: srvfunc.HostnameForStatshouse()}, vmSize)
+			client.Value("igp_vm_rss", statshouse.Tags{1: srvfunc.HostnameForStatshouse()}, vmRSS)
+			client.Value("igp_heap_alloc", statshouse.Tags{1: srvfunc.HostnameForStatshouse()}, float64(memStats.HeapAlloc))
+			client.Value("igp_heap_sys", statshouse.Tags{1: srvfunc.HostnameForStatshouse()}, float64(memStats.HeapSys))
+			client.Value("igp_heap_idle", statshouse.Tags{1: srvfunc.HostnameForStatshouse()}, float64(memStats.HeapIdle))
+			client.Value("igp_heap_inuse", statshouse.Tags{1: srvfunc.HostnameForStatshouse()}, float64(memStats.HeapInuse))
+			//--
 		}))
 	// listen on IPv4
 	tcp4 := p.newProxyServer()

diff --git a/internal/api/handler.go b/internal/api/handler.go
@@ -19,6 +19,7 @@ import (
 	"math"
 	"net/http"
 	"os"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -616,12 +617,29 @@ func NewHandler(staticDir fs.FS, jsSettings JSSettings, showInvisible bool, chV1
 		sysMetric := client.MetricRef(format.BuiltinMetricNameUsageCPU, statshouse.Tags{1: strconv.Itoa(format.TagValueIDComponentAPI), 2: strconv.Itoa(format.TagValueIDCPUUsageSys)})
 		sysMetric.Value(sysTime)
 
-		var rss float64
+		var vmSize, vmRSS float64
 		if st, _ := srvfunc.GetMemStat(0); st != nil {
-			rss = float64(st.Res)
-		}
-		memMetric := client.MetricRef(format.BuiltinMetricNameUsageMemory, statshouse.Tags{1: strconv.Itoa(format.TagValueIDComponentAPI)})
-		memMetric.Value(rss)
+			vmSize = float64(st.Size)
+			vmRSS = float64(st.Res)
+		}
+		client.Value(format.BuiltinMetricNameApiVmSize, statshouse.Tags{1: srvfunc.HostnameForStatshouse()}, vmSize)
+		client.Value(format.BuiltinMetricNameApiVmRSS, statshouse.Tags{1: srvfunc.HostnameForStatshouse()}, vmRSS)
+		client.Value(format.BuiltinMetricNameUsageMemory, statshouse.Tags{1: strconv.Itoa(format.TagValueIDComponentAPI)}, vmRSS)
+
+		var memStats runtime.MemStats
+		runtime.ReadMemStats(&memStats)
+		client.Value(format.BuiltinMetricNameApiHeapAlloc, statshouse.Tags{1: srvfunc.HostnameForStatshouse()}, float64(memStats.HeapAlloc))
+		client.Value(format.BuiltinMetricNameApiHeapSys, statshouse.Tags{1: srvfunc.HostnameForStatshouse()}, float64(memStats.HeapSys))
+		client.Value(format.BuiltinMetricNameApiHeapIdle, statshouse.Tags{1: srvfunc.HostnameForStatshouse()}, float64(memStats.HeapIdle))
+		client.Value(format.BuiltinMetricNameApiHeapInuse, statshouse.Tags{1: srvfunc.HostnameForStatshouse()}, float64(memStats.HeapInuse))
+		//-- TODO: remove when deployed
+		client.Value("api_vm_size", statshouse.Tags{1: srvfunc.HostnameForStatshouse()}, vmSize)
+		client.Value("api_vm_rss", statshouse.Tags{1: srvfunc.HostnameForStatshouse()}, vmRSS)
+		client.Value("api_heap_alloc", statshouse.Tags{1: srvfunc.HostnameForStatshouse()}, float64(memStats.HeapAlloc))
+		client.Value("api_heap_sys", statshouse.Tags{1: srvfunc.HostnameForStatshouse()}, float64(memStats.HeapSys))
+		client.Value("api_heap_idle", statshouse.Tags{1: srvfunc.HostnameForStatshouse()}, float64(memStats.HeapIdle))
+		client.Value("api_heap_inuse", statshouse.Tags{1: srvfunc.HostnameForStatshouse()}, float64(memStats.HeapInuse))
+		//--
 
 		writeActiveQuieries := func(ch *util.ClickHouse, versionTag string) {
 			if ch != nil {

diff --git a/internal/format/builtin.go b/internal/format/builtin.go
@@ -135,6 +135,18 @@ const (
 	BuiltinMetricIDAggSamplingEngineTime     = -111
 	BuiltinMetricIDAggSamplingEngineKeys     = -112
 	BuiltinMetricIDProxyAcceptHandshakeError = -113
+	BuiltinMetricIDProxyVmSize               = -114
+	BuiltinMetricIDProxyVmRSS                = -115
+	BuiltinMetricIDProxyHeapAlloc            = -116
+	BuiltinMetricIDProxyHeapSys              = -117
+	BuiltinMetricIDProxyHeapIdle             = -118
+	BuiltinMetricIDProxyHeapInuse            = -119
+	BuiltinMetricIDApiVmSize                 = -120
+	BuiltinMetricIDApiVmRSS                  = -121
+	BuiltinMetricIDApiHeapAlloc              = -122
+	BuiltinMetricIDApiHeapSys                = -123
+	BuiltinMetricIDApiHeapIdle               = -124
+	BuiltinMetricIDApiHeapInuse              = -125
 
 	// [-1000..-2000] reserved by host system metrics
 	// [-10000..-12000] reserved by builtin dashboard
@@ -184,6 +196,18 @@ const (
 	BuiltinMetricAPIBufferBytesTotal            = "__api_buffer_bytes_total"
 	BuiltinMetricNameGCDuration                 = "__gc_duration"
 	BuiltinMetricNameProxyAcceptHandshakeError  = "__igp_accept_handshake_error"
+	BuiltinMetricNameApiVmSize                  = "__api_vm_size"
+	BuiltinMetricNameApiVmRSS                   = "__api_vm_rss"
+	BuiltinMetricNameApiHeapAlloc               = "__api_heap_alloc"
+	BuiltinMetricNameApiHeapSys                 = "__api_heap_sys"
+	BuiltinMetricNameApiHeapIdle                = "__api_heap_idle"
+	BuiltinMetricNameApiHeapInuse               = "__api_heap_inuse"
+	BuiltinMetricNameProxyVmSize                = "__igp_vm_size"
+	BuiltinMetricNameProxyVmRSS                 = "__igp_vm_rss"
+	BuiltinMetricNameProxyHeapAlloc             = "__igp_heap_alloc"
+	BuiltinMetricNameProxyHeapSys               = "__igp_heap_sys"
+	BuiltinMetricNameProxyHeapIdle              = "__igp_heap_idle"
+	BuiltinMetricNameProxyHeapInuse             = "__igp_heap_inuse"
 
 	TagValueIDBadgeAgentSamplingFactor = -1
 	TagValueIDBadgeAggSamplingFactor   = -10
@@ -2303,6 +2327,114 @@ Value is delta between second value and time it was inserted.`,
 				Description: "error",
 			}},
 		},
+		BuiltinMetricIDApiVmSize: {
+			Name:        BuiltinMetricNameApiVmSize,
+			Kind:        MetricKindValue,
+			Description: "StatsHouse API virtual memory size.",
+			Resolution:  60,
+			Tags: []MetricMetaTag{{
+				Description: "host",
+			}},
+		},
+		BuiltinMetricIDApiVmRSS: {
+			Name:        BuiltinMetricNameApiVmRSS,
+			Kind:        MetricKindValue,
+			Description: "StatsHouse API resident set size.",
+			Resolution:  60,
+			Tags: []MetricMetaTag{{
+				Description: "host",
+			}},
+		},
+		BuiltinMetricIDApiHeapAlloc: {
+			Name:        BuiltinMetricNameApiHeapAlloc,
+			Kind:        MetricKindValue,
+			Description: "StatsHouse API bytes of allocated heap objects.",
+			Resolution:  60,
+			Tags: []MetricMetaTag{{
+				Description: "host",
+			}},
+		},
+		BuiltinMetricIDApiHeapSys: {
+			Name:        BuiltinMetricNameApiHeapSys,
+			Kind:        MetricKindValue,
+			Description: "StatsHouse API bytes of heap memory obtained from the OS.",
+			Resolution:  60,
+			Tags: []MetricMetaTag{{
+				Description: "host",
+			}},
+		},
+		BuiltinMetricIDApiHeapIdle: {
+			Name:        BuiltinMetricNameApiHeapIdle,
+			Kind:        MetricKindValue,
+			Description: "StatsHouse API bytes in idle (unused) spans.",
+			Resolution:  60,
+			Tags: []MetricMetaTag{{
+				Description: "host",
+			}},
+		},
+		BuiltinMetricIDApiHeapInuse: {
+			Name:        BuiltinMetricNameApiHeapInuse,
+			Kind:        MetricKindValue,
+			Description: "StatsHouse API bytes in in-use spans.",
+			Resolution:  60,
+			Tags: []MetricMetaTag{{
+				Description: "host",
+			}},
+		},
+		BuiltinMetricIDProxyVmSize: {
+			Name:        BuiltinMetricNameProxyVmSize,
+			Kind:        MetricKindValue,
+			Description: "StatsHouse proxy virtual memory size.",
+			Resolution:  60,
+			Tags: []MetricMetaTag{{
+				Description: "host",
+			}},
+		},
+		BuiltinMetricIDProxyVmRSS: {
+			Name:        BuiltinMetricNameProxyVmRSS,
+			Kind:        MetricKindValue,
+			Description: "StatsHouse proxy resident set size.",
+			Resolution:  60,
+			Tags: []MetricMetaTag{{
+				Description: "host",
+			}},
+		},
+		BuiltinMetricIDProxyHeapAlloc: {
+			Name:        BuiltinMetricNameProxyHeapAlloc,
+			Kind:        MetricKindValue,
+			Description: "StatsHouse proxy bytes of allocated heap objects.",
+			Resolution:  60,
+			Tags: []MetricMetaTag{{
+				Description: "host",
+			}},
+		},
+		BuiltinMetricIDProxyHeapSys: {
+			Name:        BuiltinMetricNameProxyHeapSys,
+			Kind:        MetricKindValue,
+			Description: "StatsHouse proxy bytes of heap memory obtained from the OS.",
+			Resolution:  60,
+			Tags: []MetricMetaTag{{
+				Description: "host",
+			}},
+		},
+		BuiltinMetricIDProxyHeapIdle: {
+			Name:        BuiltinMetricNameProxyHeapIdle,
+			Kind:        MetricKindValue,
+			Description: "StatsHouse proxy bytes in idle (unused) spans.",
+			Resolution:  60,
+			Tags: []MetricMetaTag{{
+				Description: "host",
+			}},
+		},
+		BuiltinMetricIDProxyHeapInuse: {
+			Name:        BuiltinMetricNameProxyHeapInuse,
+			Kind:        MetricKindValue,
+			Description: "StatsHouse proxy bytes in in-use spans.",
+			Resolution:  60,
+			Tags: []MetricMetaTag{{
+				Description: "host",
+			}},
+		},
 	}
 
 	builtinMetricsInvisible = map[int32]bool{
@@ -2344,6 +2476,18 @@ Value is delta between second value and time it was inserted.`,
 		BuiltinMetricIDAutoCreateMetric:           true,
 		BuiltinMetricIDGCDuration:                 true,
 		BuiltinMetricIDProxyAcceptHandshakeError:  true,
+		BuiltinMetricIDProxyVmSize:                true,
+		BuiltinMetricIDProxyVmRSS:                 true,
+		BuiltinMetricIDProxyHeapAlloc:             true,
+		BuiltinMetricIDProxyHeapSys:               true,
+		BuiltinMetricIDProxyHeapIdle:              true,
+		BuiltinMetricIDProxyHeapInuse:             true,
+		BuiltinMetricIDApiVmSize:                  true,
+		BuiltinMetricIDApiVmRSS:                   true,
+		BuiltinMetricIDApiHeapAlloc:               true,
+		BuiltinMetricIDApiHeapSys:                 true,
+		BuiltinMetricIDApiHeapIdle:                true,
+		BuiltinMetricIDApiHeapInuse:               true,
 	}
 
 	builtinMetricsNoSamplingAgent = map[int32]bool{
@@ -2459,6 +2603,18 @@ Value is delta between second value and time it was inserted.`,
 		BuiltinMetricIDRestartTimings:             true,
 		BuiltinMetricIDGCDuration:                 true,
 		BuiltinMetricIDProxyAcceptHandshakeError:  true,
+		BuiltinMetricIDProxyVmSize:                true,
+		BuiltinMetricIDProxyVmRSS:                 true,
+		BuiltinMetricIDProxyHeapAlloc:             true,
+		BuiltinMetricIDProxyHeapSys:               true,
+		BuiltinMetricIDProxyHeapIdle:              true,
+		BuiltinMetricIDProxyHeapInuse:             true,
+		BuiltinMetricIDApiVmSize:                  true,
+		BuiltinMetricIDApiVmRSS:                   true,
+		BuiltinMetricIDApiHeapAlloc:               true,
+		BuiltinMetricIDApiHeapSys:                 true,
+		BuiltinMetricIDApiHeapIdle:                true,
+		BuiltinMetricIDApiHeapInuse:               true,
 	}
 
 	insertKindToValue = map[int32]string{

diff --git a/internal/vkgo/srvfunc/common.go b/internal/vkgo/srvfunc/common.go
@@ -20,9 +20,7 @@ import (
 	"golang.org/x/sys/unix"
 )
 
-const (
-	pagesize = 4096 // замена C.sysconf(C._SC_PAGESIZE)
-)
+var pagesize int
 
 type (
 	// MemStats содержит статистику по использованию памяти в байтах
@@ -54,6 +52,12 @@ type (
 	}
 )
 
+func init() {
+	if pagesize = os.Getpagesize(); pagesize <= 0 {
+		pagesize = 4096
+	}
+}
+
 // SetMaxRLimitNoFile пробует выставить текущие nofile лимиты (ulimit -n) в максимально разрешенные
 // Вернет в случае успеха кортеж (cur, max) значений лимита
 func SetMaxRLimitNoFile() ([]uint64, error) {
@@ -145,13 +149,13 @@ func GetMemStat(pid int) (*MemStats, error) {
 		return nil, err
 	}
 
-	m.Size *= pagesize
-	m.Res *= pagesize
-	m.Share *= pagesize
-	m.Text *= pagesize
-	m.Lib *= pagesize
-	m.Data *= pagesize
-	m.Dt *= pagesize
+	m.Size *= uint64(pagesize)
+	m.Res *= uint64(pagesize)
+	m.Share *= uint64(pagesize)
+	m.Text *= uint64(pagesize)
+	m.Lib *= uint64(pagesize)
+	m.Data *= uint64(pagesize)
+	m.Dt *= uint64(pagesize)
 
 	return &m, nil
 }