Skip to content

Commit

Permalink
Dynamically start prometheus and service monitor
Browse files Browse the repository at this point in the history
Signed-off-by: Tamal Saha <[email protected]>
  • Loading branch information
tamalsaha committed Jan 16, 2024
1 parent 7e0de7e commit 7e614bb
Show file tree
Hide file tree
Showing 8 changed files with 167 additions and 30 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -342,7 +342,7 @@ lint: $(BUILD_DIRS)
--env GO111MODULE=on \
--env GOFLAGS="-mod=vendor" \
$(BUILD_IMAGE) \
golangci-lint run --enable $(ADDTL_LINTERS) --timeout=10m --skip-files="generated.*\.go$\" --skip-dirs-use-default --skip-dirs=client,vendor
golangci-lint run --enable $(ADDTL_LINTERS) --max-same-issues=100 --timeout=10m --skip-files="generated.*\.go$\" --skip-dirs-use-default --skip-dirs=client,vendor

$(BUILD_DIRS):
@mkdir -p $@
Expand Down
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ require (
github.com/spf13/pflag v1.0.5
github.com/zeebo/xxh3 v1.0.2
go.bytebuilders.dev/license-verifier v0.13.4
go.openviz.dev/apimachinery v0.0.7-0.20240105050246-85dfcd1152cb
go.openviz.dev/apimachinery v0.0.7
go.openviz.dev/grafana-sdk v0.0.4
golang.org/x/text v0.14.0
gomodules.xyz/logs v0.0.7
Expand All @@ -28,7 +28,7 @@ require (
k8s.io/client-go v0.29.0
k8s.io/klog/v2 v2.110.1
kmodules.xyz/authorizer v0.29.0
kmodules.xyz/client-go v0.29.5
kmodules.xyz/client-go v0.29.6
kmodules.xyz/custom-resources v0.29.0
kmodules.xyz/monitoring-agent-api v0.29.0
sigs.k8s.io/controller-runtime v0.16.3
Expand Down
8 changes: 4 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -473,8 +473,8 @@ go.opentelemetry.io/otel/trace v1.20.0 h1:+yxVAPZPbQhbC3OfAkeIVTky6iTFpcr4SiY9om
go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU=
go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
go.openviz.dev/apimachinery v0.0.7-0.20240105050246-85dfcd1152cb h1:1wDqFOBePci3nBcUEMxIa37p/hkSVZczfCvUJjCNHnw=
go.openviz.dev/apimachinery v0.0.7-0.20240105050246-85dfcd1152cb/go.mod h1:cWwPKNiOIOpXV4xwFNJQaeq2+pLKVwg5KhsiCiEDeB8=
go.openviz.dev/apimachinery v0.0.7 h1:LBW/RtI21FA9ILHEfOMCF+YCSBw2UEHd8H+cgmtZaAs=
go.openviz.dev/apimachinery v0.0.7/go.mod h1:OUaaMC3iCOyp7R6j5puYMV7fkSNu6ERW2P+jcJ+n1iE=
go.openviz.dev/grafana-sdk v0.0.4 h1:Tl/lZtHK22j5eGWBQpdi2ErqS5TFlZeICCUyZnIJx1g=
go.openviz.dev/grafana-sdk v0.0.4/go.mod h1:fdBkehPrBOKL53+fCCm95AzK5Vs/5+xhsfuXo8OblaY=
go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY=
Expand Down Expand Up @@ -759,8 +759,8 @@ kmodules.xyz/apiversion v0.2.0 h1:vAQYqZFm4xu4pbB1cAdHbFEPES6EQkcR4wc06xdTOWk=
kmodules.xyz/apiversion v0.2.0/go.mod h1:oPX8g8LvlPdPX3Yc5YvCzJHQnw3YF/X4/jdW0b1am80=
kmodules.xyz/authorizer v0.29.0 h1:ND8YGeyzExdZ8Bq5Z6UdFO794I6+oPuXbUMWyjlsYgM=
kmodules.xyz/authorizer v0.29.0/go.mod h1:UQmE3sNXeliebUqjEeD9QYiY+Na27/C5Bg/ekVRfQ3U=
kmodules.xyz/client-go v0.29.5 h1:iRl4MoV+96TM1csInOCWjn5xSOXzuYlil6CO40vXLHU=
kmodules.xyz/client-go v0.29.5/go.mod h1:pHuzpwzEcDUIGjVVvwz9N8lY+6A7HXwvs2d7NtK7Hho=
kmodules.xyz/client-go v0.29.6 h1:xTVq5LZvsPBUTLY7PORq7zveLOj/vpuTDvkpHWOk3RM=
kmodules.xyz/client-go v0.29.6/go.mod h1:pHuzpwzEcDUIGjVVvwz9N8lY+6A7HXwvs2d7NtK7Hho=
kmodules.xyz/crd-schema-fuzz v0.29.1 h1:zJTlWYOrT5dsVVHW8HGcnR/vaWfxQfNh11QwTtkYpcs=
kmodules.xyz/crd-schema-fuzz v0.29.1/go.mod h1:n708z9YQqLMP2KNLQVgBcRJw1QpSWLvpNCEi+KJDOYE=
kmodules.xyz/custom-resources v0.29.0 h1:RaDM2+wSVXiwIvLqmkTVYpwoH83AC8wruXe2p2rOZNY=
Expand Down
67 changes: 46 additions & 21 deletions pkg/apiserver/apiserver.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,10 @@ import (
servicemonitorcontroller "go.openviz.dev/grafana-tools/pkg/controllers/servicemonitor"
dashgroupstorage "go.openviz.dev/grafana-tools/pkg/registry/ui/dashboardgroup"

"github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring"
monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
core "k8s.io/api/core/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
Expand All @@ -46,9 +48,11 @@ import (
"k8s.io/klog/v2"
"k8s.io/klog/v2/klogr"
"kmodules.xyz/authorizer"
"kmodules.xyz/client-go/apiextensions"
clustermeta "kmodules.xyz/client-go/cluster"
appcatalogapi "kmodules.xyz/custom-resources/apis/appcatalog/v1alpha1"
mona "kmodules.xyz/monitoring-agent-api/api/v1"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/manager"
Expand All @@ -71,6 +75,7 @@ func init() {
utilruntime.Must(monitoringv1.AddToScheme(Scheme))
utilruntime.Must(appcatalogapi.AddToScheme(Scheme))
utilruntime.Must(chartsapi.AddToScheme(Scheme))
utilruntime.Must(apiextensionsv1.AddToScheme(Scheme))

// we need to add the options to empty v1
// TODO fix the server code to avoid this
Expand Down Expand Up @@ -141,7 +146,7 @@ func (c completedConfig) New(ctx context.Context) (*UIServer, error) {
// ctrl.SetLogger(...)
log.SetLogger(klogr.New()) // nolint:staticcheck

mgr, err := manager.New(c.ExtraConfig.ClientConfig, manager.Options{
mgr, err := ctrl.NewManager(c.ExtraConfig.ClientConfig, ctrl.Options{
Scheme: Scheme,
Metrics: metricsserver.Options{BindAddress: ""},
HealthProbeBindAddress: "",
Expand Down Expand Up @@ -173,30 +178,50 @@ func (c completedConfig) New(ctx context.Context) (*UIServer, error) {
}
}

if err = promtehsucontroller.NewReconciler(
mgr.GetClient(),
bc,
cid,
).SetupWithManager(mgr); err != nil {
klog.Error(err, "unable to create controller", "controller", "Prometheus")
if err := apiextensions.NewReconciler(ctx, mgr).SetupWithManager(mgr); err != nil {
klog.Error(err, "unable to create controller controller CustomResourceReconciler")
os.Exit(1)
}

if err = servicemonitorcontroller.NewFederationReconciler(
c.ExtraConfig.ClientConfig,
mgr.GetClient(),
).SetupWithManager(mgr); err != nil {
klog.Error(err, "unable to create controller", " federation controller", "ServiceMonitor")
os.Exit(1)
}
apiextensions.RegisterSetup(schema.GroupKind{
Group: monitoring.GroupName,
Kind: monitoringv1.PrometheusesKind,
}, func(ctx context.Context, mgr ctrl.Manager) {
if err = promtehsucontroller.NewReconciler(
mgr.GetClient(),
bc,
cid,
).SetupWithManager(mgr); err != nil {
klog.Error(err, "unable to create controller", "controller", "Prometheus")
os.Exit(1)
}
})

if err = servicemonitorcontroller.NewAutoReconciler(
c.ExtraConfig.ClientConfig,
mgr.GetClient(),
).SetupWithManager(mgr); err != nil {
klog.Error(err, "unable to create controller", "auto controller", "ServiceMonitor")
os.Exit(1)
}
apiextensions.RegisterSetup(schema.GroupKind{
Group: monitoring.GroupName,
Kind: monitoringv1.ServiceMonitorsKind,
}, func(ctx context.Context, mgr ctrl.Manager) {
if err = servicemonitorcontroller.NewFederationReconciler(
c.ExtraConfig.ClientConfig,
mgr.GetClient(),
).SetupWithManager(mgr); err != nil {
klog.Error(err, "unable to create controller", " federation controller", "ServiceMonitor")
os.Exit(1)
}
})

apiextensions.RegisterSetup(schema.GroupKind{
Group: monitoring.GroupName,
Kind: monitoringv1.ServiceMonitorsKind,
}, func(ctx context.Context, mgr ctrl.Manager) {
if err = servicemonitorcontroller.NewAutoReconciler(
c.ExtraConfig.ClientConfig,
mgr.GetClient(),
).SetupWithManager(mgr); err != nil {
klog.Error(err, "unable to create controller", "auto controller", "ServiceMonitor")
os.Exit(1)
}
})

if err := mgr.GetFieldIndexer().IndexField(context.Background(), &appcatalogapi.AppBinding{}, mona.DefaultGrafanaKey, func(rawObj client.Object) []string {
app := rawObj.(*appcatalogapi.AppBinding)
Expand Down
5 changes: 5 additions & 0 deletions pkg/registry/ui/dashboardgroup/storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ var (
_ rest.Scoper = &Storage{}
_ rest.Storage = &Storage{}
_ rest.Creater = &Storage{}
_ rest.SingularNameProvider = &Storage{}
)

func NewStorage(kc client.Client, a authorizer.Authorizer) *Storage {
Expand All @@ -70,6 +71,10 @@ func (r *Storage) GroupVersionKind(_ schema.GroupVersion) schema.GroupVersionKin
return uiapi.SchemeGroupVersion.WithKind(uiapi.ResourceKindDashboardGroup)
}

func (r *Storage) GetSingularName() string {
return strings.ToLower(uiapi.ResourceKindDashboardGroup)
}

func (r *Storage) NamespaceScoped() bool {
return false
}
Expand Down
87 changes: 87 additions & 0 deletions vendor/kmodules.xyz/client-go/apiextensions/controller.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
/*
Copyright AppsCode Inc. and Contributors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiextensions

import (
"context"
"sync"

apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
)

type SetupFn func(ctx context.Context, mgr ctrl.Manager)

var setupFns = map[schema.GroupKind]SetupFn{
// schema.GroupKind{"compute.gcp.kubedb.com", "Firewall"}: firewall.Setup,
}

var (
setupDone = map[schema.GroupKind]bool{}
mu sync.Mutex
)

type Reconciler struct {
ctx context.Context
mgr ctrl.Manager
}

func NewReconciler(ctx context.Context, mgr ctrl.Manager) *Reconciler {
return &Reconciler{ctx: ctx, mgr: mgr}
}

func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
log := log.FromContext(ctx)
var crd apiextensions.CustomResourceDefinition
if err := r.mgr.GetClient().Get(ctx, req.NamespacedName, &crd); err != nil {
log.Error(err, "unable to fetch CustomResourceDefinition")
return ctrl.Result{}, client.IgnoreNotFound(err)
}

gk := schema.GroupKind{
Group: crd.Spec.Group,
Kind: crd.Spec.Names.Kind,
}
mu.Lock()
defer mu.Unlock()
_, found := setupDone[gk]
if found {
return ctrl.Result{}, nil
}
setup, found := setupFns[gk]
if found {
setup(r.ctx, r.mgr)
setupDone[gk] = true
}
return ctrl.Result{}, nil
}

func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&apiextensions.CustomResourceDefinition{}).
Complete(r)
}

func RegisterSetup(gk schema.GroupKind, fn SetupFn) {
mu.Lock()
defer mu.Unlock()

setupFns[gk] = fn
}
20 changes: 20 additions & 0 deletions vendor/kmodules.xyz/client-go/apiextensions/kubernetes.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,3 +109,23 @@ func WaitForCRDReady(client crd_cs.Interface, crds []*CustomResourceDefinition)
})
return errors.Wrap(err, "timed out waiting for CRD")
}

func RemoveCRDs(client crd_cs.Interface, crds []*CustomResourceDefinition) error {
for _, crd := range crds {
// Use crd v1 for k8s >= 1.16, if available
// ref: https://github.com/kubernetes/kubernetes/issues/91395
if crd.V1 == nil {
gvr := schema.GroupVersionResource{
Group: crd.V1beta1.Spec.Group,
Version: crd.V1beta1.Spec.Versions[0].Name,
Resource: crd.V1beta1.Spec.Names.Plural,
}
return fmt.Errorf("missing V1 definition for %s", gvr)
}
err := client.ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.V1.Name, metav1.DeleteOptions{})
if err != nil && !kerr.IsNotFound(err) {
return err
}
}
return nil
}
4 changes: 2 additions & 2 deletions vendor/modules.txt
Original file line number Diff line number Diff line change
Expand Up @@ -408,7 +408,7 @@ go.opentelemetry.io/proto/otlp/collector/trace/v1
go.opentelemetry.io/proto/otlp/common/v1
go.opentelemetry.io/proto/otlp/resource/v1
go.opentelemetry.io/proto/otlp/trace/v1
# go.openviz.dev/apimachinery v0.0.7-0.20240105050246-85dfcd1152cb
# go.openviz.dev/apimachinery v0.0.7
## explicit; go 1.21.5
go.openviz.dev/apimachinery/apis/openviz
go.openviz.dev/apimachinery/apis/openviz/install
Expand Down Expand Up @@ -1325,7 +1325,7 @@ kmodules.xyz/authorizer/apiserver
kmodules.xyz/authorizer/rbac
kmodules.xyz/authorizer/rbac/helpers
kmodules.xyz/authorizer/rbac/validation
# kmodules.xyz/client-go v0.29.5
# kmodules.xyz/client-go v0.29.6
## explicit; go 1.21.5
kmodules.xyz/client-go
kmodules.xyz/client-go/api/v1
Expand Down

0 comments on commit 7e614bb

Please sign in to comment.