Log heap dump automatically (#1222)

* log heap dump (run-int-tests)

---------

Co-authored-by: Robert Graeff <[email protected]>

charts/helm-deployer/templates/clusterrole.yaml

@@ -71,4 +71,5 @@ rules:
   - list
   - watch
   - update
+  - delete
 {{- end }}

charts/manifest-deployer/templates/clusterrole.yaml

@@ -70,4 +70,5 @@ rules:
   - list
   - watch
   - update
+  - delete
 {{- end }}

cmd/helm-deployer-controller/app/app.go

@@ -12,6 +12,8 @@ import (
 	"os"
 	"time"
 
+	"github.com/gardener/landscaper/controller-utils/pkg/logging"
+
 	"github.com/spf13/cobra"
 
 	lc "github.com/gardener/landscaper/controller-utils/pkg/logging/constants"
@@ -56,7 +58,8 @@ func (o *options) run(ctx context.Context) error {
 			o.DeployerOptions.Log.Error(err, "helm deployer profiler stopped")
 		}()
 
-		go utils.LogMemStatsPeriodically(ctx, o.DeployerOptions.Log, 60*time.Second)
+		go utils.LogMemStatsPeriodically(logging.NewContext(ctx, o.DeployerOptions.Log), 60*time.Second,
+			o.DeployerOptions.HostUncachedClient, "helm-deployer")
 	}
 
 	o.DeployerOptions.Log.Info("Starting helm deployer manager")

cmd/landscaper-controller/app/app.go

@@ -175,7 +175,7 @@ func (o *Options) startMainController(ctx context.Context,
 		})
 
 		eg.Go(func() error {
-			lsutils.LogMemStatsPeriodically(ctx, setupLogger, 60*time.Second)
+			lsutils.LogMemStatsPeriodically(logging.NewContext(ctx, setupLogger), 60*time.Second, hostUncachedClient, "main-landscaper")
 			return nil
 		})
 	}
@@ -234,7 +234,7 @@ func (o *Options) startCentralLandscaper(ctx context.Context,
 		})
 
 		eg.Go(func() error {
-			lsutils.LogMemStatsPeriodically(ctx, setupLogger, 60*time.Second)
+			lsutils.LogMemStatsPeriodically(logging.NewContext(ctx, setupLogger), 60*time.Second, hostUncachedClient, "central-landscaper")
 			return nil
 		})
 	}

cmd/manifest-deployer-controller/app/app.go

@@ -12,6 +12,8 @@ import (
 	"os"
 	"time"
 
+	"github.com/gardener/landscaper/controller-utils/pkg/logging"
+
 	"github.com/spf13/cobra"
 
 	lc "github.com/gardener/landscaper/controller-utils/pkg/logging/constants"
@@ -57,7 +59,8 @@ func (o *options) run(ctx context.Context) error {
 			o.DeployerOptions.Log.Error(err, "manifest deployer profiler stopped")
 		}()
 
-		go utils.LogMemStatsPeriodically(ctx, o.DeployerOptions.Log, 60*time.Second)
+		go utils.LogMemStatsPeriodically(logging.NewContext(ctx, o.DeployerOptions.Log), 60*time.Second,
+			o.DeployerOptions.HostUncachedClient, "manifest-deployer")
 	}
 
 	o.DeployerOptions.Log.Info("Starting manifest deployer manager")

controller-utils/go.sum

@@ -145,8 +145,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
-golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
 golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
 golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
 golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA=
@@ -190,22 +188,16 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
-golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
 golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk=
-golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
 golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU=
 golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
-golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
 golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
 golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
 golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=

docs/technical/profiling.md

@@ -61,6 +61,52 @@ If the environment variable `ENABLE_PROFILER` is set as described above, you can
    ```shell
    go tool pprof -http=:8082 heap.out
    ```
+
+## Automatic Heap Dump
+
+If the environment variable `ENABLE_PROFILER` is set as described above a heap dump is automatically created if
+
+- there was no automatic heap dump created for the pod since it started and the current HeapInUse is more than 330 MB
+- there was already an automatic heap dump created for the pod since it started and the current HeapInUse is more than
+  10 percentage higher than for the formerly created heap dump. The older heap dump is overwritten by the newer one.
+
+An automatic heap dump is written into a set of secrets, consisting of a base secret with some meta data and several
+data secrets containing the byte data of the heap dump. 
+
+The base secret is stored in the same namespace as the corresponding pod. Its name is as follows for the different 
+pods:
+
+- Landscaper main pod with Installation and Execution controllers: main-landscaper-heap
+- Landscaper pod for rest: central-landscaper-heap
+- Helm deployer pod: helm-deployer-heap
+- Manifest deployer pod: manifest-deployer-heap
+
+A base secret contains the following entries:
+
+- keyNumberOfDataSecrets: number of secrets containing the data of the profile
+- keyHeapInUse: heap in use in bytes
+- keyHeapAlloc: heap alloc in bytes
+- keyPodname: name of the pod for which a heap dump was stored
+- keyStorageDate: timestamp when the heap dump was stored
+
+For every base secret with Name `<prefix>-heap` there is a set of secrets with names `<prefix>-heap-0`, `<prefix>-heap-1`
+etc. containing the heap data. 
+
+To analyse a heap dump you have to execute the following commands:
+
+``` bash
+# store the heap data in different files
+kubectl get secret -n <namespace> <prefix>-heap-0 -o jsonpath="{.data.keyBytes}" | base64 -d > heap0.out
+kubectl get secret -n <namespace> <prefix>-heap-1 -o jsonpath="{.data.keyBytes}" | base64 -d > heap1.out
+...
+
+# concatenate the files
+cat heap0.out heap1.out ... > heap.out
+
+# analyse the result
+go tool pprof -http=:8082 heap.out
+```
+
 ## Some other important commands
 
 Sometimes it might be interesting to see the memory consumption of a pod and the containers running in it.

go.mod

@@ -5,6 +5,7 @@ go 1.22.4
 require (
 	dario.cat/mergo v1.0.0
 	github.com/Masterminds/sprig/v3 v3.2.3
+	github.com/cloudflare/cfssl v1.6.5
 	github.com/containerd/containerd v1.7.18
 	github.com/docker/cli v26.1.5+incompatible
 	github.com/gardener/component-cli v0.44.0
@@ -125,7 +126,6 @@ require (
 	github.com/chai2010/gettext-go v1.0.2 // indirect
 	github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 // indirect
 	github.com/clbanning/mxj/v2 v2.7.0 // indirect
-	github.com/cloudflare/cfssl v1.6.5 // indirect
 	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be // indirect
 	github.com/containerd/errdefs v0.1.0 // indirect