Skip to content
This repository has been archived by the owner on May 26, 2023. It is now read-only.

Commit

Permalink
Merge pull request #3 from hendrikKahl/cache-metrics
Browse files Browse the repository at this point in the history
add prometheus metrics to cache
  • Loading branch information
Tim Schrodi authored Feb 5, 2021
2 parents 6d25070 + 37fbece commit 054ead0
Show file tree
Hide file tree
Showing 95 changed files with 19,546 additions and 17 deletions.
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ require (
github.com/docker/distribution v2.7.1+incompatible
github.com/docker/docker v1.4.2-0.20200203170920-46ec8731fbce // indirect
github.com/docker/docker-credential-helpers v0.6.3 // indirect
github.com/gardener/component-spec/bindings-go v0.0.31
github.com/gardener/component-spec/bindings-go v0.0.32
github.com/ghodss/yaml v1.0.0
github.com/go-logr/logr v0.3.0
github.com/go-logr/zapr v0.3.0
Expand All @@ -20,6 +20,7 @@ require (
github.com/onsi/gomega v1.10.1
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.0.1
github.com/prometheus/client_golang v0.9.3
github.com/sirupsen/logrus v1.4.2 // indirect
github.com/spf13/cobra v1.1.1
github.com/spf13/pflag v1.0.5
Expand Down
9 changes: 7 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/b4b4r07/go-pipe v0.0.0-20191010045404-84b446f57366/go.mod h1:1ymsiQNa3qebVEEVtuIdhtAXRfjO4qFCFq1bBUOT2HE=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
Expand Down Expand Up @@ -73,8 +74,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/gardener/component-spec/bindings-go v0.0.31 h1:zkJaUaOIyghTeJPnyFr1Y6lJsnZQSeUlDMd5z2rbXkY=
github.com/gardener/component-spec/bindings-go v0.0.31/go.mod h1:53EIwICsuMgbY/MFCv5/lbOR5e1MQe1HY674Io+nGYE=
github.com/gardener/component-spec/bindings-go v0.0.32 h1:1za/V6R9ORWLdyPu+9VzLxg/JpS0VOMvuos9ncCzMas=
github.com/gardener/component-spec/bindings-go v0.0.32/go.mod h1:53EIwICsuMgbY/MFCv5/lbOR5e1MQe1HY674Io+nGYE=
github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
Expand Down Expand Up @@ -259,13 +260,17 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8=
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM=
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
Expand Down
47 changes: 47 additions & 0 deletions ociclient/cache/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package cache

import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
Expand All @@ -17,6 +18,8 @@ import (
"github.com/mandelsoft/vfs/pkg/projectionfs"
"github.com/mandelsoft/vfs/pkg/vfs"
ocispecv1 "github.com/opencontainers/image-spec/specs-go/v1"

"github.com/gardener/component-cli/ociclient/metrics"
)

type layeredCache struct {
Expand All @@ -25,6 +28,8 @@ type layeredCache struct {

baseFs vfs.FileSystem
overlayFs vfs.FileSystem

basePath string
}

// NewCache creates a new cache with the given options.
Expand All @@ -46,11 +51,36 @@ func NewCache(log logr.Logger, options ...Option) (Cache, error) {
overlay = memoryfs.New()
}

//initialize metrics
metrics.CachedItems.WithLabelValues(opts.BasePath).Set(0)
metrics.CacheDiskUsage.WithLabelValues(opts.BasePath).Set(0)
metrics.CacheHitsDisk.WithLabelValues(opts.BasePath).Add(0)
if opts.InMemoryOverlay {
metrics.CacheMemoryUsage.WithLabelValues(opts.BasePath).Set(0)
metrics.CacheHitsMemory.WithLabelValues(opts.BasePath).Add(0)
}

err = vfs.Walk(base, "/", func(_ string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
metrics.CachedItems.WithLabelValues(opts.BasePath).Inc()
metrics.CacheDiskUsage.WithLabelValues(opts.BasePath).Add(float64(info.Size()))
}
return nil
})

if err != nil {
return nil, fmt.Errorf("error during initialization of cache metrics: %w", err)
}

return &layeredCache{
log: log,
mux: sync.RWMutex{},
baseFs: base,
overlayFs: overlay,
basePath: opts.BasePath,
}, nil
}

Expand Down Expand Up @@ -84,6 +114,7 @@ func (lc *layeredCache) Get(desc ocispecv1.Descriptor) (io.ReadCloser, error) {
if err != nil {
return nil, err
}
metrics.CacheHitsDisk.WithLabelValues(lc.basePath).Inc()
return file, nil
}

Expand All @@ -100,6 +131,16 @@ func (lc *layeredCache) Add(desc ocispecv1.Descriptor, reader io.ReadCloser) err
defer file.Close()

_, err = io.Copy(file, reader)

// in case everything worked well, update metrics
if err == nil {
metrics.CachedItems.WithLabelValues(lc.basePath).Inc()
if fileInfo, metricsErr := file.Stat(); metricsErr == nil {
metrics.CacheDiskUsage.WithLabelValues(lc.basePath).Add(float64(fileInfo.Size()))
} else {
lc.log.V(7).Error(metricsErr, "Failed to access %q", file.Name())
}
}
return err
}

Expand Down Expand Up @@ -136,6 +177,7 @@ func (lc *layeredCache) get(dgst string) (os.FileInfo, vfs.File, error) {
if err != nil {
return nil, nil, err
}
metrics.CacheHitsMemory.WithLabelValues(lc.basePath).Inc()
return info, file, err
}
lc.log.V(7).Info("not found in overlay cache", "dgst", dgst, "digest", dgst)
Expand Down Expand Up @@ -167,6 +209,11 @@ func (lc *layeredCache) get(dgst string) (os.FileInfo, vfs.File, error) {
lc.log.V(5).Info(err.Error())
return info, file, nil
}
if fileInfo, err := lc.overlayFs.Stat(dgst); err == nil {
metrics.CacheMemoryUsage.WithLabelValues(lc.basePath).Add(float64(fileInfo.Size()))
} else {
lc.log.V(7).Error(err, "Failed to access %q", dgst)
}
}
return info, file, nil
}
Expand Down
78 changes: 78 additions & 0 deletions ociclient/metrics/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
// SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors.
//
// SPDX-License-Identifier: Apache-2.0

package metrics

import "github.com/prometheus/client_golang/prometheus"

const (
ociClientNamespaceName = "ociclient"
cacheSubsystemName = "cache"
)

var (
// CacheMemoryUsage discloses memory used by caches
CacheMemoryUsage = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: ociClientNamespaceName,
Subsystem: cacheSubsystemName,
Name: "memory_usage_bytes",
Help: "Bytes in memory currently used by cache instance.",
},
[]string{"path"},
)

// CacheDiskUsage discloses disk used by caches
CacheDiskUsage = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: ociClientNamespaceName,
Subsystem: cacheSubsystemName,
Name: "disk_usage_bytes",
Help: "Bytes on disk currently used by cache instance.",
},
[]string{"path"},
)

// CachedItems discloses the number of items stored by caches
CachedItems = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: ociClientNamespaceName,
Subsystem: cacheSubsystemName,
Name: "items_total",
Help: "Total number of items currently cached by instance.",
},
[]string{"path"},
)

// CacheHitsDisk discloses the number of hits for items cached on disk
CacheHitsDisk = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: ociClientNamespaceName,
Subsystem: cacheSubsystemName,
Name: "disk_hits_total",
Help: "Total number of hits for items cached on disk by an instance.",
},
[]string{"path"},
)

// CacheHitsMemory discloses the number of hits for items cached in memory
CacheHitsMemory = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: ociClientNamespaceName,
Subsystem: cacheSubsystemName,
Name: "memory_hits_total",
Help: "Total number of hits for items cached in memory by an instance.",
},
[]string{"path"},
)
)

// RegisterCacheMetrics allows to register ociclient cache metrics with a given prometheus registerer
func RegisterCacheMetrics(reg prometheus.Registerer) {
reg.MustRegister(CacheHitsDisk)
reg.MustRegister(CacheHitsMemory)
reg.MustRegister(CachedItems)
reg.MustRegister(CacheDiskUsage)
reg.MustRegister(CacheMemoryUsage)
}
20 changes: 20 additions & 0 deletions vendor/github.com/beorn7/perks/LICENSE

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 054ead0

Please sign in to comment.