diff --git a/.github/workflows/agent-container-pr.yml b/.github/workflows/agent-container-pr.yml index e34509fb..a50e0acf 100644 --- a/.github/workflows/agent-container-pr.yml +++ b/.github/workflows/agent-container-pr.yml @@ -8,6 +8,10 @@ on: branches: - 'main' +permissions: + contents: read + packages: write + jobs: build: runs-on: ubuntu-latest diff --git a/.github/workflows/agent-git-pr.yml b/.github/workflows/agent-git-pr.yml index 7c4a5ede..813f570f 100644 --- a/.github/workflows/agent-git-pr.yml +++ b/.github/workflows/agent-git-pr.yml @@ -8,6 +8,10 @@ on: branches: - 'main' +permissions: + contents: read + packages: write + jobs: build: runs-on: ubuntu-latest diff --git a/.github/workflows/agent-kubviz-pr.yml b/.github/workflows/agent-kubviz-pr.yml index 3aca320e..c6df7e29 100644 --- a/.github/workflows/agent-kubviz-pr.yml +++ b/.github/workflows/agent-kubviz-pr.yml @@ -8,6 +8,10 @@ on: branches: - 'main' +permissions: + contents: read + packages: write + jobs: build: runs-on: ubuntu-latest diff --git a/.github/workflows/client-pr.yml b/.github/workflows/client-pr.yml index 87464778..43fdba59 100644 --- a/.github/workflows/client-pr.yml +++ b/.github/workflows/client-pr.yml @@ -8,6 +8,10 @@ on: branches: - 'main' +permissions: + contents: read + packages: write + jobs: build: runs-on: ubuntu-latest diff --git a/.github/workflows/migration-pr.yml b/.github/workflows/migration-pr.yml index 2bcd5997..2ad31086 100644 --- a/.github/workflows/migration-pr.yml +++ b/.github/workflows/migration-pr.yml @@ -8,6 +8,10 @@ on: branches: - 'main' +permissions: + contents: read + packages: write + jobs: build: runs-on: ubuntu-latest diff --git a/.github/workflows/otel-collector-image.yml b/.github/workflows/otel-collector-image.yml new file mode 100644 index 00000000..32e27afe --- /dev/null +++ b/.github/workflows/otel-collector-image.yml @@ -0,0 +1,99 @@ +name: OTEL Collector Docker Image CI + +on: + push: + paths-ignore: + - '**.md' + - 'charts/**' + branches: + - 'main' + +jobs: + + build: + + runs-on: ubuntu-latest + permissions: + packages: write + id-token: write + contents: read + actions: read + security-events: write + env: + REGISTRY: ghcr.io + GH_URL: https://github.com + steps: + - name: Checkout GitHub Action + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Docker metadata + id: metadata + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ github.repository }}/otel-collector + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value={{sha}},enable=${{ github.ref_type != 'tag' }} + flavor: | + latest=true + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - + name: Set up Go + uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed #5.1.0 + with: + go-version: "1.22" + cache: false + + - + name: Build OpenTelemetry Collector + working-directory: otel-collector + run: | + curl --proto '=https' --tlsv1.2 -fL -o ocb https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/cmd%2Fbuilder%2Fv0.112.0/ocb_0.112.0_linux_amd64 + chmod +x ocb + ./ocb --config builder-config.yaml + cp _build/kubviz-otel-collector . + + - name: Build image and push to GitHub Container Registry + uses: docker/build-push-action@v4 + with: + context: otel-collector + file: ./otel-collector/Dockerfile + tags: ${{ env.REGISTRY }}/${{ github.repository }}/otel-collector:${{ github.run_id }} + labels: ${{ steps.metadata.outputs.labels }} + + push: true + + - name: Install cosign + uses: sigstore/cosign-installer@main + + - name: Sign the images + run: | + cosign sign -y ${{ env.REGISTRY }}/${{ github.repository }}/otel-collector:${{ github.run_id }} + env: + COSIGN_EXPERIMENTAL: 1 + + - name: Verify the pushed tags + run: cosign verify ${{ env.REGISTRY }}/${{ github.repository }}/otel-collector:${{ github.run_id }} --certificate-identity ${{ env.GH_URL }}/${{ github.repository }}/.github/workflows/otel-collector-image.yml@refs/heads/main --certificate-oidc-issuer https://token.actions.githubusercontent.com + env: + COSIGN_EXPERIMENTAL: 1 + + - name: Run Trivy in GitHub SBOM mode and submit results to Dependency Graph + uses: aquasecurity/trivy-action@master + with: + scan-type: 'fs' + format: 'github' + output: 'dependency-results.sbom.json' + image-ref: '.' + github-pat: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/otel-collector-pr.yml b/.github/workflows/otel-collector-pr.yml new file mode 100644 index 00000000..f856fb21 --- /dev/null +++ b/.github/workflows/otel-collector-pr.yml @@ -0,0 +1,70 @@ +name: OTEL Collector Docker Image CI + +on: + pull_request: + paths-ignore: + - '**.md' + - 'charts/**' + branches: + - 'main' + +permissions: + contents: read + packages: write + +jobs: + build: + runs-on: ubuntu-latest + env: + REGISTRY: ghcr.io + GH_URL: https://github.com + steps: + - + name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - + name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - uses: docker/setup-buildx-action@v1 + name: Set up Docker Buildx + + - + name: Login to ghcr registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - + name: Set up Go + uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed #5.1.0 + with: + go-version: "1.22" + cache: false + + - + name: Build OpenTelemetry Collector + working-directory: otel-collector + run: | + curl --proto '=https' --tlsv1.2 -fL -o ocb https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/cmd%2Fbuilder%2Fv0.112.0/ocb_0.112.0_linux_amd64 + chmod +x ocb + ./ocb --config builder-config.yaml + cp _build/kubviz-otel-collector . + + - + name: Build and push on PR + uses: docker/build-push-action@v4 + if: github.event_name == 'pull_request' + with: + context: otel-collector + file: ./otel-collector/Dockerfile + push: true + tags: ${{ env.REGISTRY }}/${{ github.repository }}/otel-collector:pr-${{ github.event.pull_request.number }} + build-args: | + "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" + diff --git a/.github/workflows/otel-collector-release.yml b/.github/workflows/otel-collector-release.yml new file mode 100644 index 00000000..800b506c --- /dev/null +++ b/.github/workflows/otel-collector-release.yml @@ -0,0 +1,72 @@ +name: otel-collector-release +on: + push: + tags: + - "v*.*.*" +jobs: + push_to_registry: + name: Build and push Docker image github container registry. + runs-on: ubuntu-20.04 + permissions: + packages: write + id-token: write + contents: read + actions: read + security-events: write + env: + REGISTRY: ghcr.io + GH_URL: https://github.com + steps: + - name: Set environment variable + run: | + echo "RELEASE_VERSION=${GITHUB_REF:10}" >> $GITHUB_ENV + - name: Test environment variable + run: echo ${{ env.RELEASE_VERSION }} + - name: Check out GitHub repo + uses: actions/checkout@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - + name: Set up Go + uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed #5.1.0 + with: + go-version: "1.22" + cache: false + - + name: Build OpenTelemetry Collector + working-directory: otel-collector + run: | + curl --proto '=https' --tlsv1.2 -fL -o ocb https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/cmd%2Fbuilder%2Fv0.112.0/ocb_0.112.0_linux_amd64 + chmod +x ocb + ./ocb --config builder-config.yaml + cp _build/kubviz-otel-collector . + - name: Build image and push to GitHub Container Registry + uses: docker/build-push-action@v4 + with: + push: true + context: otel-collector + file: ./otel-collector/Dockerfile + tags: ${{ env.REGISTRY }}/${{ github.repository }}/otel-collector:${{ env.RELEASE_VERSION }} + - name: Install cosign + uses: sigstore/cosign-installer@main + - name: Sign the images + run: | + cosign sign -y ${{ env.REGISTRY }}/${{ github.repository }}/otel-collector:${{ env.RELEASE_VERSION }} + env: + COSIGN_EXPERIMENTAL: 1 + - name: Verify the pushed tags + run: cosign verify ${{ env.REGISTRY }}/${{ github.repository }}/otel-collector:${{ env.RELEASE_VERSION }} --certificate-identity ${{ env.GH_URL }}/${{ github.repository }}/.github/workflows/otel-collector-release.yml@refs/tags/${{ env.RELEASE_VERSION }} --certificate-oidc-issuer https://token.actions.githubusercontent.com + env: + COSIGN_EXPERIMENTAL: 1 + - name: Run Trivy in GitHub SBOM mode and submit results to Dependency Graph + uses: aquasecurity/trivy-action@master + with: + scan-type: 'fs' + format: 'github' + output: 'dependency-results.sbom.json' + image-ref: '.' + github-pat: ${{ secrets.GITHUB_TOKEN }} # or ${{ secrets.github_pat_name }} if you're using a PAT diff --git a/.gitignore b/.gitignore index 87460586..6b899b43 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ allocs.pprof cpu.pprof steps-to-test.txt Dockerfile-grphqlserver-build +vendor +kubviz_agent diff --git a/agent/config/config.go b/agent/config/config.go index 775fffdc..cb357915 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -11,6 +11,8 @@ type AgentConfigurations struct { SANamespace string `envconfig:"SA_NAMESPACE" default:"default"` SAName string `envconfig:"SA_NAME" default:"default"` OutdatedInterval string `envconfig:"OUTDATED_INTERVAL" default:"0"` + KubeAllResourcesInterval string `envconfig:"KUBE_ALL_RESOURCES_INTERVAL" default:"*/30 * * * *"` + KubeAllResourcesEnabled bool `envconfig:"KUBE_ALL_RESOURCES_ENABLED" default:"true"` GetAllInterval string `envconfig:"GETALL_INTERVAL" default:"*/30 * * * *"` KubeScoreInterval string `envconfig:"KUBESCORE_INTERVAL" default:"*/40 * * * *"` RakkessInterval string `envconfig:"RAKKESS_INTERVAL" default:"*/50 * * * *"` diff --git a/agent/kubviz/k8smetrics_agent.go b/agent/kubviz/k8smetrics_agent.go index d0c0c7f6..313214ec 100644 --- a/agent/kubviz/k8smetrics_agent.go +++ b/agent/kubviz/k8smetrics_agent.go @@ -17,21 +17,22 @@ import ( "github.com/intelops/kubviz/pkg/mtlsnats" "github.com/intelops/kubviz/pkg/opentelemetry" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "github.com/intelops/kubviz/agent/config" "github.com/intelops/kubviz/agent/kubviz/plugins/events" - "github.com/intelops/kubviz/agent/kubviz/plugins/ketall" + "github.com/intelops/kubviz/agent/kubviz/plugins/kubeallresources" "github.com/intelops/kubviz/agent/kubviz/plugins/kubepreupgrade" - - "github.com/intelops/kubviz/agent/kubviz/plugins/kuberhealthy" "github.com/intelops/kubviz/agent/kubviz/plugins/kubescore" "github.com/intelops/kubviz/agent/kubviz/plugins/outdated" "github.com/intelops/kubviz/agent/kubviz/plugins/rakkess" - "github.com/intelops/kubviz/agent/kubviz/plugins/trivy" + + "github.com/intelops/kubviz/agent/kubviz/plugins/kuberhealthy" + "github.com/intelops/kubviz/agent/kubviz/scheduler" _ "k8s.io/client-go/plugin/pkg/client/auth/azure" @@ -40,6 +41,9 @@ import ( "github.com/intelops/kubviz/agent/server" //_ "k8s.io/client-go/plugin/pkg/client/auth/openstack" + "k8s.io/client-go/discovery" + "k8s.io/client-go/discovery/cached/memory" + "k8s.io/client-go/restmapper" "k8s.io/client-go/tools/clientcmd" ) @@ -73,8 +77,9 @@ func main() { log.Fatal("Failed to retrieve agent configurations", err) } var ( - config *rest.Config - clientset *kubernetes.Clientset + config *rest.Config + clientset *kubernetes.Clientset + dynamicClient *dynamic.DynamicClient ) var mtlsConfig mtlsnats.MtlsConfig @@ -113,14 +118,27 @@ func main() { log.Fatal(err) } clientset = events.GetK8sClient(config) + dynamicClient, err = dynamic.NewForConfig(config) + if err != nil { + log.Fatal(err) + } } else { config, err = rest.InClusterConfig() if err != nil { log.Fatal(err) } clientset = events.GetK8sClient(config) + dynamicClient, err = dynamic.NewForConfig(config) + if err != nil { + log.Fatal(err) + } } + // Create a REST mapper + discoveryClient := discovery.NewDiscoveryClientForConfigOrDie(config) + cachedDiscoveryClient := memory.NewMemCacheClient(discoveryClient) + restMapper := restmapper.NewDeferredDiscoveryRESTMapper(cachedDiscoveryClient) + tp, err := opentelemetry.InitTracer() if err != nil { log.Fatal(err) @@ -131,7 +149,8 @@ func main() { } }() - go events.PublishMetrics(clientset, js, clusterMetricsChan) + go events.PublishMetrics(clientset, dynamicClient, restMapper, js, clusterMetricsChan) + if cfg.KuberHealthyEnable { go kuberhealthy.StartKuberHealthy(js) } @@ -141,6 +160,12 @@ func main() { events.LogErr(err) err = kubepreupgrade.KubePreUpgradeDetector(config, js) events.LogErr(err) + + if cfg.KubeAllResourcesEnabled { + err = kubeallresources.PublishAllResources(config) + events.LogErr(err) + } + err = ketall.GetAllResources(config, js) events.LogErr(err) err = rakkess.RakeesOutput(config, js) diff --git a/agent/kubviz/otel/log.go b/agent/kubviz/otel/log.go new file mode 100644 index 00000000..e7deb63f --- /dev/null +++ b/agent/kubviz/otel/log.go @@ -0,0 +1,57 @@ +package otel + +import ( + "context" + stdlog "log" + "time" + + "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc" + "go.opentelemetry.io/otel/log" + logsdk "go.opentelemetry.io/otel/sdk/log" +) + +const ( + logName = "github.com/intelops/kubviz/agent/kubviz/otel" + kubvizEventName = "kubviz.event.name" + logTimeout = 3 * time.Second +) + +var loggerProvider *logsdk.LoggerProvider + +func newLoggerProvider() (*logsdk.LoggerProvider, error) { + // Use env var to configure the endpoint: + // OTEL_EXPORTER_OTLP_ENDPOINT=grpc://localhost:4317 + exporter, err := otlploggrpc.New(context.Background()) + + if err != nil { + return nil, err + } + + loggerProvider := logsdk.NewLoggerProvider( + logsdk.WithProcessor(logsdk.NewBatchProcessor(exporter)), + ) + + return loggerProvider, nil +} + +func init() { + var err error + loggerProvider, err = newLoggerProvider() + if err != nil { + stdlog.Fatalln(err.Error()) + } +} + +func PublishEventLog(eventName string, body []byte) { + logRecord := log.Record{} + logRecord.SetBody(log.StringValue(string(body))) + logRecord.SetTimestamp(time.Now()) + logRecord.SetSeverity(log.SeverityTrace2) + logRecord.AddAttributes(log.KeyValue{Key: kubvizEventName, Value: log.StringValue(eventName)}) + + ctx, cancel := context.WithTimeout(context.Background(), logTimeout) + defer cancel() + + logger := loggerProvider.Logger(logName) + logger.Emit(ctx, logRecord) +} diff --git a/agent/kubviz/plugins/events/event_metrics_utils.go b/agent/kubviz/plugins/events/event_metrics_utils.go index 17ef114f..fd81ebbe 100644 --- a/agent/kubviz/plugins/events/event_metrics_utils.go +++ b/agent/kubviz/plugins/events/event_metrics_utils.go @@ -11,6 +11,7 @@ import ( "strings" "time" + kubevizotel "github.com/intelops/kubviz/agent/kubviz/otel" "github.com/intelops/kubviz/constants" "github.com/intelops/kubviz/model" "github.com/intelops/kubviz/pkg/opentelemetry" @@ -18,8 +19,12 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" @@ -29,7 +34,7 @@ var ClusterName string = os.Getenv("CLUSTER_NAME") // publishMetrics publishes stream of events // with subject "METRICS.created" -func PublishMetrics(clientset *kubernetes.Clientset, js nats.JetStreamContext, errCh chan error) { +func PublishMetrics(clientset *kubernetes.Clientset, dynamicClient *dynamic.DynamicClient, restMapper meta.RESTMapper, js nats.JetStreamContext, errCh chan error) { ctx := context.Background() tracer := otel.Tracer("kubviz-publish-metrics") @@ -37,7 +42,7 @@ func PublishMetrics(clientset *kubernetes.Clientset, js nats.JetStreamContext, e span.SetAttributes(attribute.String("kubviz-agent", "publish-metrics")) defer span.End() - watchK8sEvents(clientset, js) + watchK8sEvents(clientset, dynamicClient, restMapper, js) errCh <- nil } @@ -58,6 +63,7 @@ func publishK8sMetrics(id string, mtype string, mdata *v1.Event, js nats.JetStre } metricsJson, _ := json.Marshal(metrics) _, err := js.Publish(constants.EventSubject, metricsJson) + if err != nil { return true, err } @@ -164,7 +170,7 @@ func LogErr(err error) { log.Println(err) } } -func watchK8sEvents(clientset *kubernetes.Clientset, js nats.JetStreamContext) { +func watchK8sEvents(clientset *kubernetes.Clientset, dynamicClient *dynamic.DynamicClient, restMapper meta.RESTMapper, js nats.JetStreamContext) { ctx := context.Background() tracer := otel.Tracer("kubviz-watch-k8sevents") @@ -178,6 +184,7 @@ func watchK8sEvents(clientset *kubernetes.Clientset, js nats.JetStreamContext) { v1.NamespaceAll, fields.Everything(), ) + _, controller := cache.NewInformer( watchlist, &v1.Event{}, @@ -193,6 +200,22 @@ func watchK8sEvents(clientset *kubernetes.Clientset, js nats.JetStreamContext) { for _, image := range images { publishK8sMetrics(string(event.ObjectMeta.UID), "ADD", event, js, image) } + + // Publish to OTEL logs + unstructuredObj, err := getUnstructuredObject(dynamicClient, restMapper, event.InvolvedObject) + if err != nil { + log.Printf("Error getting unstructured object: %v\n", err) + return + } + + err = publishUnstructuredObject(unstructuredObj) + if err != nil { + log.Printf("Error publishing unstructured object as OTEL log: %v\n", err) + return + } + + // fmt.Println("watchK8sEvents AddFunc:") + // fmt.Println(unstructuredObj) }, DeleteFunc: func(obj interface{}) { event := obj.(*v1.Event) @@ -226,3 +249,26 @@ func watchK8sEvents(clientset *kubernetes.Clientset, js nats.JetStreamContext) { time.Sleep(time.Second) } } + +func getUnstructuredObject(dynamicClient dynamic.Interface, restMapper meta.RESTMapper, involvedObject v1.ObjectReference) (*unstructured.Unstructured, error) { + gvk := schema.FromAPIVersionAndKind(involvedObject.APIVersion, involvedObject.Kind) + + mapping, err := restMapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + return nil, fmt.Errorf("failed to get REST mapping: %v", err) + } + + gvr := mapping.Resource + + return dynamicClient.Resource(gvr).Namespace(involvedObject.Namespace).Get(context.TODO(), involvedObject.Name, metav1.GetOptions{}) +} + +func publishUnstructuredObject(obj *unstructured.Unstructured) error { + objJson, err := json.Marshal(obj) + if err != nil { + return err + } + + kubevizotel.PublishEventLog(constants.EventSubject_kubeallresources, objJson) + return nil +} diff --git a/agent/kubviz/plugins/kubeallresources/kubeallresources.go b/agent/kubviz/plugins/kubeallresources/kubeallresources.go new file mode 100644 index 00000000..705ff2b1 --- /dev/null +++ b/agent/kubviz/plugins/kubeallresources/kubeallresources.go @@ -0,0 +1,78 @@ +package kubeallresources + +import ( + "context" + "encoding/json" + "fmt" + "os" + + "github.com/intelops/kubviz/constants" + "github.com/intelops/kubviz/pkg/opentelemetry" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/discovery" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + + kubevizotel "github.com/intelops/kubviz/agent/kubviz/otel" +) + +var ClusterName string = os.Getenv("CLUSTER_NAME") + +func PublishAllResources(config *rest.Config) error { + ctx := context.Background() + tracer := otel.Tracer("kubeallresources") + _, span := tracer.Start(opentelemetry.BuildContext(ctx), "PublishAllResources") + span.SetAttributes(attribute.String("kubeallresources-plugin-agent", "kubeallresources-output")) + defer span.End() + + // Create a new discovery client to discover all resources in the cluster + dc := discovery.NewDiscoveryClientForConfigOrDie(config) + + // Create a new dynamic client to list resources in the cluster + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + return err + } + // Get a list of all available API groups and versions in the cluster + resourceLists, err := dc.ServerPreferredResources() + if err != nil { + return err + } + gvrs, err := discovery.GroupVersionResources(resourceLists) + if err != nil { + return err + } + // Iterate over all available API groups and versions and list all resources in each group + for gvr := range gvrs { + // List all resources in the group + list, err := dynamicClient.Resource(gvr).Namespace("").List(context.Background(), metav1.ListOptions{}) + if err != nil { + fmt.Printf("Error listing %s: %v\n", gvr.String(), err) + continue + } + + for _, item := range list.Items { + err := publishUnstructuredObject(&item) + if err != nil { + fmt.Println("error while publishing unstructured object", err) + continue + } + } + } + + return nil +} + +func publishUnstructuredObject(obj *unstructured.Unstructured) error { + objJson, err := json.Marshal(obj) + if err != nil { + return err + } + + kubevizotel.PublishEventLog(constants.EventSubject_kubeallresources, objJson) + return nil +} diff --git a/agent/kubviz/scheduler/scheduler.go b/agent/kubviz/scheduler/scheduler.go index ccd9564e..29e6cc59 100644 --- a/agent/kubviz/scheduler/scheduler.go +++ b/agent/kubviz/scheduler/scheduler.go @@ -104,6 +104,18 @@ func InitScheduler(config *rest.Config, js nats.JetStreamContext, cfg config.Age log.Fatal("failed to do job", err) } } + + if cfg.KubeAllResourcesEnabled && cfg.KubeAllResourcesInterval != "" && cfg.KubeAllResourcesInterval != "0" { + sj, err := NewKubeAllResourcesJob(config, cfg.KubeAllResourcesInterval) + if err != nil { + log.Fatal("no time interval", err) + } + err = s.AddJob("KubeAllResource", sj) + if err != nil { + log.Fatal("failed to do job", err) + } + } + if cfg.GetAllInterval != "" && cfg.GetAllInterval != "0" { sj, err := NewKetallJob(config, js, cfg.GetAllInterval) if err != nil { diff --git a/agent/kubviz/scheduler/scheduler_watch.go b/agent/kubviz/scheduler/scheduler_watch.go index f2e769af..6183038e 100644 --- a/agent/kubviz/scheduler/scheduler_watch.go +++ b/agent/kubviz/scheduler/scheduler_watch.go @@ -3,6 +3,7 @@ package scheduler import ( "github.com/intelops/kubviz/agent/kubviz/plugins/events" "github.com/intelops/kubviz/agent/kubviz/plugins/ketall" + "github.com/intelops/kubviz/agent/kubviz/plugins/kubeallresources" "github.com/intelops/kubviz/agent/kubviz/plugins/kubepreupgrade" "github.com/nats-io/nats.go" "k8s.io/client-go/kubernetes" @@ -20,6 +21,11 @@ type OutDatedImagesJob struct { frequency string } +type KubeAllResourcesJob struct { + config *rest.Config + frequency string +} + type KetallJob struct { config *rest.Config js nats.JetStreamContext @@ -121,6 +127,23 @@ func (j *OutDatedImagesJob) Run() { err := outdated.OutDatedImages(j.config, j.js) events.LogErr(err) } + +func NewKubeAllResourcesJob(config *rest.Config, frequency string) (*KubeAllResourcesJob, error) { + return &KubeAllResourcesJob{ + config: config, + frequency: frequency, + }, nil +} +func (v *KubeAllResourcesJob) CronSpec() string { + return v.frequency +} + +func (j *KubeAllResourcesJob) Run() { + // Call the Ketall function with the provided config and js + err := kubeallresources.PublishAllResources(j.config) + events.LogErr(err) +} + func NewKetallJob(config *rest.Config, js nats.JetStreamContext, frequency string) (*KetallJob, error) { return &KetallJob{ config: config, diff --git a/charts/agent/.gitignore b/charts/agent/.gitignore new file mode 100644 index 00000000..8d894615 --- /dev/null +++ b/charts/agent/.gitignore @@ -0,0 +1,2 @@ +charts +Chart.lock diff --git a/charts/agent/templates/deployment.yaml b/charts/agent/templates/deployment.yaml index 8de9de0d..c4ab147b 100644 --- a/charts/agent/templates/deployment.yaml +++ b/charts/agent/templates/deployment.yaml @@ -101,7 +101,12 @@ spec: value: {{ .Values.opentelemetry.url }} - name : APPLICATION_NAME value : {{ .Values.opentelemetry.appName }} - + {{- if .Values.transportV2.enabled }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.transportV2.exporterOtlpEndpoint }} + {{- end }} + - name: KUBE_ALL_RESOURCES_ENABLED + value: "{{ .Values.kubeAllResources.enabled }}" volumeMounts: {{- if .Values.mtls.enabled }} - name: mtls diff --git a/charts/agent/values.yaml b/charts/agent/values.yaml index 3436019e..b0ae54c6 100644 --- a/charts/agent/values.yaml +++ b/charts/agent/values.yaml @@ -267,6 +267,13 @@ opentelemetry: url: "otelcollector.local" appName: "kubviz" +transportV2: + enabled: true + exporterOtlpEndpoint: grpc://kubviz-client-otel-collector:4317 + +kubeAllResources: + enabled: true + clusterName: "kubviz" externalSecrets: diff --git a/charts/client/.gitignore b/charts/client/.gitignore new file mode 100644 index 00000000..8d894615 --- /dev/null +++ b/charts/client/.gitignore @@ -0,0 +1,2 @@ +charts +Chart.lock diff --git a/charts/client/templates/configmap-otel-collector-config.yaml b/charts/client/templates/configmap-otel-collector-config.yaml new file mode 100644 index 00000000..634b5305 --- /dev/null +++ b/charts/client/templates/configmap-otel-collector-config.yaml @@ -0,0 +1,34 @@ +{{- if .Values.otelCollector.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "client.fullname" . }}-otel-collector-config +data: + config.yaml: |- + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + max_recv_msg_size_mib: 50 + max_concurrent_streams: 100 + + processors: + batch: + + exporters: + debug: + verbosity: detailed + nats: + token: {{ .Values.nats.auth.token }} + url: nats://{{ include "client.fullname" . }}-nats:4222 + stream_name: METRICS + subject_name: METRICS.kubeallresources + + service: + pipelines: + logs: + receivers: [otlp] + processors: [batch] + exporters: [nats] +{{- end }} \ No newline at end of file diff --git a/charts/client/templates/deployment-otel-collector.yaml b/charts/client/templates/deployment-otel-collector.yaml new file mode 100644 index 00000000..562d382d --- /dev/null +++ b/charts/client/templates/deployment-otel-collector.yaml @@ -0,0 +1,37 @@ +{{- if .Values.otelCollector.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "client.fullname" . }}-otel-collector + labels: + app: {{ include "client.fullname" . }}-otel-collector +spec: + replicas: 1 + selector: + matchLabels: + app: {{ include "client.fullname" . }}-otel-collector + template: + metadata: + labels: + app: {{ include "client.fullname" . }}-otel-collector + spec: + containers: + - name: otel-collector + image: "{{ .Values.otelCollector.image.repository }}:{{ .Values.otelCollector.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.otelCollector.image.pullPolicy }} + volumeMounts: + - name: otel-collector-config + mountPath: /etc/kubviz-otelcol-config + ports: + - name: otlp-grpc + containerPort: 4137 + protocol: TCP + - containerPort: 55678 + protocol: TCP + - containerPort: 55679 + protocol: TCP + volumes: + - name: otel-collector-config + configMap: + name: {{ include "client.fullname" . }}-otel-collector-config +{{- end }} \ No newline at end of file diff --git a/charts/client/templates/deployment.yaml b/charts/client/templates/deployment.yaml index 7daeda56..cf52d06a 100644 --- a/charts/client/templates/deployment.yaml +++ b/charts/client/templates/deployment.yaml @@ -148,6 +148,10 @@ spec: name: {{ .Values.existingClickhouse.secret.name }} key: {{ .Values.existingClickhouse.secret.passwordkey }} {{- end }} + {{- end }} + {{- if .Values.dgraph.enabled }} + - name: DGRAPH_GRPC_ADDRESS + value: {{ .Values.dgraph.grpcAddress }} {{- end }} - name: DB_PORT value: "9000" @@ -161,6 +165,8 @@ spec: value: {{ .Values.opentelemetry.url }} - name : APPLICATION_NAME value : {{ .Values.opentelemetry.appName }} + - name : KUBEALLRESOURCES_EVENTS_CONSUMER + value : {{ .Values.consumer.kubeallresourcesconsumer }} - name : KETALL_EVENTS_CONSUMER value : {{ .Values.consumer.ketallconsumer }} - name : RAKEES_METRICS_CONSUMER diff --git a/charts/client/templates/service-otel-collector.yaml b/charts/client/templates/service-otel-collector.yaml new file mode 100644 index 00000000..8c2490ee --- /dev/null +++ b/charts/client/templates/service-otel-collector.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: {{ include "client.fullname" . }}-otel-collector + name: {{ include "client.fullname" . }}-otel-collector +spec: + ports: + - port: 4317 + protocol: TCP + targetPort: 4317 + selector: + app: {{ include "client.fullname" . }}-otel-collector diff --git a/charts/client/values.yaml b/charts/client/values.yaml index 0f66da94..a9fd5706 100644 --- a/charts/client/values.yaml +++ b/charts/client/values.yaml @@ -83,6 +83,14 @@ affinity: {} externalSecrets: create: false +otelCollector: + enabled: true + image: + repository: localhost:5000/kubviz-otel-collector + pullPolicy: Always + # Overrides the image tag whose default is the chart appVersion. + tag: "v1.1.9" + nats: enabled: true #Authentication setup @@ -124,6 +132,10 @@ existingClickhouse: # usernamekey: "" # passwordkey: "" +dgraph: + enabled: true + grpcAddress: dgraph-public:9080 + grafana: enabled: false plugins: @@ -187,6 +199,7 @@ opentelemetry: appName: "kubviz" consumer: + kubeallresourcesconsumer: "KUBEALLRESOURCES_EVENTS_CONSUMER" ketallconsumer: "KETALL_EVENTS_CONSUMER" rakeesconsumer: "RAKEES_METRICS_CONSUMER" outdatedconsumer: "OUTDATED_EVENTS_CONSUMER" diff --git a/client/pkg/application/application.go b/client/pkg/application/application.go index 6d9c3c99..76e40cff 100644 --- a/client/pkg/application/application.go +++ b/client/pkg/application/application.go @@ -8,21 +8,26 @@ import ( "os/signal" "syscall" + "github.com/dgraph-io/dgo/v240" + "github.com/dgraph-io/dgo/v240/protos/api" "github.com/intelops/kubviz/client/pkg/clickhouse" "github.com/intelops/kubviz/client/pkg/clients" "github.com/intelops/kubviz/client/pkg/config" + "github.com/intelops/kubviz/client/pkg/dgraph" "github.com/intelops/kubviz/client/pkg/storage" "github.com/intelops/kubviz/pkg/opentelemetry" "github.com/kelseyhightower/envconfig" "github.com/robfig/cron/v3" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" + "google.golang.org/grpc" ) type Application struct { - Config *config.Config - conn *clients.NATSContext - dbClient clickhouse.DBInterface + Config *config.Config + conn *clients.NATSContext + dbClient clickhouse.DBInterface + dgraphGrpc *grpc.ClientConn } const ( @@ -66,8 +71,28 @@ func Start() *Application { if err != nil { log.Fatal(err) } + + // K8s Dgraph + var dgraphClient *dgo.Dgraph + var grpcConn *grpc.ClientConn + + if cfg.DgraphGrpcAddress != "" { + // Initialize Dgraph client + grpcConn, err = grpc.Dial(cfg.DgraphGrpcAddress, grpc.WithInsecure()) + if err != nil { + log.Fatalf("Error connecting to Dgraph: %v", err) + } + dgraphClient = dgo.NewDgraphClient(api.NewDgraphClient(grpcConn)) + + // Setup schema + err = dgraph.SetupDgraphSchema(ctx, dgraphClient) + if err != nil { + log.Fatalf("Failed to set up Dgraph schema: %v", err) + } + } + // Connect to NATS - natsContext, err := clients.NewNATSContext(cfg, dbClient) + natsContext, err := clients.NewNATSContext(cfg, dbClient, dgraphClient) if err != nil { log.Fatal("Error establishing connection to NATS:", err) } @@ -119,17 +144,29 @@ func Start() *Application { } return &Application{ - Config: cfg, - conn: natsContext, - dbClient: dbClient, + Config: cfg, + conn: natsContext, + dbClient: dbClient, + dgraphGrpc: grpcConn, } } func (app *Application) Close() { log.Printf("Closing the service gracefully") - app.conn.Close() - app.dbClient.Close() + + if app.conn != nil { + app.conn.Close() + } + + if app.dbClient != nil { + app.dbClient.Close() + } + + if app.dgraphGrpc != nil { + app.dgraphGrpc.Close() + } } + func exportDataForTables(db *sql.DB) error { //pvcMountPath := "/mnt/client/kbz" tables := []string{ diff --git a/client/pkg/clients/clients.go b/client/pkg/clients/clients.go index bee3fa2e..c4da2d12 100644 --- a/client/pkg/clients/clients.go +++ b/client/pkg/clients/clients.go @@ -5,6 +5,7 @@ import ( "log" "time" + "github.com/dgraph-io/dgo/v240" "github.com/intelops/kubviz/client/pkg/clickhouse" "github.com/intelops/kubviz/client/pkg/config" "github.com/intelops/kubviz/pkg/mtlsnats" @@ -18,7 +19,7 @@ type NATSContext struct { dbClient clickhouse.DBInterface } -func NewNATSContext(conf *config.Config, dbClient clickhouse.DBInterface) (*NATSContext, error) { +func NewNATSContext(conf *config.Config, dbClient clickhouse.DBInterface, dgraphClient *dgo.Dgraph) (*NATSContext, error) { log.Println("Waiting before connecting to NATS at:", conf.NatsAddress) time.Sleep(1 * time.Second) @@ -81,7 +82,7 @@ func NewNATSContext(conf *config.Config, dbClient clickhouse.DBInterface) (*NATS if err != nil { return nil, fmt.Errorf("kubeviz metrics stream not found %w", err) } - ctx.SubscribeAllKubvizNats(dbClient) + ctx.SubscribeAllKubvizNats(dbClient, dgraphClient) return ctx, nil } diff --git a/client/pkg/clients/kubviz_client.go b/client/pkg/clients/kubviz_client.go index 2e83a8b5..be0e0c93 100644 --- a/client/pkg/clients/kubviz_client.go +++ b/client/pkg/clients/kubviz_client.go @@ -5,17 +5,21 @@ import ( "encoding/json" "log" + "github.com/dgraph-io/dgo/v240" + "github.com/intelops/kubviz/client/pkg/clickhouse" + "github.com/intelops/kubviz/client/pkg/config" + "github.com/intelops/kubviz/client/pkg/dgraph" + "github.com/intelops/kubviz/client/pkg/kubernetes" "github.com/intelops/kubviz/constants" + "github.com/intelops/kubviz/model" "github.com/intelops/kubviz/pkg/opentelemetry" "github.com/kelseyhightower/envconfig" "github.com/kuberhealthy/kuberhealthy/v2/pkg/health" "github.com/nats-io/nats.go" + "go.opentelemetry.io/collector/pdata/plog" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - - "github.com/intelops/kubviz/client/pkg/clickhouse" - "github.com/intelops/kubviz/client/pkg/config" - "github.com/intelops/kubviz/model" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) type SubscriptionInfo struct { @@ -24,7 +28,7 @@ type SubscriptionInfo struct { Handler func(msg *nats.Msg) } -func (n *NATSContext) SubscribeAllKubvizNats(conn clickhouse.DBInterface) { +func (n *NATSContext) SubscribeAllKubvizNats(conn clickhouse.DBInterface, dgraphClient *dgo.Dgraph) { ctx := context.Background() tracer := otel.Tracer("kubviz-client") @@ -35,9 +39,6 @@ func (n *NATSContext) SubscribeAllKubvizNats(conn clickhouse.DBInterface) { if err := envconfig.Process("", cfg); err != nil { log.Fatalf("Could not parse env Config: %v", err) } - subscribe := func(sub SubscriptionInfo) { - n.stream.Subscribe(sub.Subject, sub.Handler, nats.Durable(sub.Consumer), nats.ManualAck()) - } subscriptions := []SubscriptionInfo{ { @@ -209,8 +210,95 @@ func (n *NATSContext) SubscribeAllKubvizNats(conn clickhouse.DBInterface) { }, } + if dgraphClient != nil { + plogUnmarshaller := &plog.JSONUnmarshaler{} + + // Create a new repository instance + repo := dgraph.NewDgraphKubernetesResourceRepository(dgraphClient) + relationshipRepo := dgraph.NewDgraphRelationshipRepository(dgraphClient) + + subscriptions = append(subscriptions, SubscriptionInfo{ + Subject: constants.KubeAllResourcesSubject, + Consumer: cfg.KubeAllResourcesConsumer, + Handler: func(msg *nats.Msg) { + msg.Ack() + + logs, err := plogUnmarshaller.UnmarshalLogs(msg.Data) + if err != nil { + // logger.WithError(err).Error("error while unmarshalling logs") + log.Println("error while unmarshalling logs") + return + } + + rls := logs.ResourceLogs() + for i := 0; i < rls.Len(); i++ { + rl := rls.At(i) + sls := rl.ScopeLogs() + for j := 0; j < sls.Len(); j++ { + sl := sls.At(j) + logRecords := sl.LogRecords() + for k := 0; k < logRecords.Len(); k++ { + ctx := context.Background() + + logRecord := logRecords.At(k) + + // Print log body + logRecordBody := logRecord.Body().AsString() + + // Print attributes + // attrs := logRecord.Attributes() + // attrs.Range(func(k string, v pcommon.Value) bool { + // fmt.Printf("Attribute - %s: %v\n", k, v.AsString()) + // return true + // }) + var k8sUnstructured unstructured.Unstructured + err := json.Unmarshal([]byte(logRecordBody), &k8sUnstructured) + if err != nil { + log.Println(err) + continue + } + + // log.Printf("NATSContext.KubeAllResourcesConsumer: Received k8s unstructured: %#v,", logRecordBody) + // log.Printf("NATSContext.KubeAllResourcesConsumer: Received k8s object: kind:%s, namespace:%s, name:%s\n\n", k8sUnstructured.GetKind(), k8sUnstructured.GetNamespace(), k8sUnstructured.GetName()) + // log.Println("-----") + + dgraphResource, err := kubernetes.ConvertUnstructuredToDgraphResource(&k8sUnstructured) + if err != nil { + log.Println(err) + continue + } + + // Save to Dgraph + err = repo.Save(ctx, dgraphResource) + if err != nil { + log.Println(err) + continue + } + + // Debug + // dgraphResourceJSON, _ := json.Marshal(dgraphResource) + // log.Printf("Saved resource to Dgraph: %s\n", string(dgraphResourceJSON)) + // log.Println("-----") + + relationships := kubernetes.BuildRelationships(&k8sUnstructured) + err = relationshipRepo.Save(ctx, dgraphResource.MetadataUID, relationships) + if err != nil { + log.Println("Failed to update relationships", err) + continue + } + + // Debug + // relationshipsJSON, _ := json.Marshal(relationships) + // log.Printf("Saved relationships to Dgraph: %s\n", string(relationshipsJSON)) + } + } + } + }, + }) + } + for _, sub := range subscriptions { log.Printf("Creating nats consumer %s with subject: %s \n", sub.Consumer, sub.Subject) - subscribe(sub) + n.stream.Subscribe(sub.Subject, sub.Handler, nats.Durable(sub.Consumer), nats.ManualAck()) } } diff --git a/client/pkg/config/config.go b/client/pkg/config/config.go index 94045005..f3f9592c 100644 --- a/client/pkg/config/config.go +++ b/client/pkg/config/config.go @@ -1,29 +1,31 @@ package config type Config struct { - NatsAddress string `envconfig:"NATS_ADDRESS"` - NatsToken string `envconfig:"NATS_TOKEN"` - DbPort int `envconfig:"DB_PORT"` - DBAddress string `envconfig:"DB_ADDRESS"` - ClickHouseUsername string `envconfig:"CLICKHOUSE_USERNAME"` - ClickHousePassword string `envconfig:"CLICKHOUSE_PASSWORD"` - KetallConsumer string `envconfig:"KETALL_EVENTS_CONSUMER" required:"true"` - RakeesConsumer string `envconfig:"RAKEES_METRICS_CONSUMER" required:"true"` - OutdatedConsumer string `envconfig:"OUTDATED_EVENTS_CONSUMER" required:"true"` - DeprecatedConsumer string `envconfig:"DEPRECATED_API_CONSUMER" required:"true"` - DeletedConsumer string `envconfig:"DELETED_API_CONSUMER" required:"true"` - KubvizConsumer string `envconfig:"KUBVIZ_EVENTS_CONSUMER" required:"true"` - KubscoreConsumer string `envconfig:"KUBSCORE_CONSUMER" required:"true"` - TrivyConsumer string `envconfig:"TRIVY_CONSUMER" required:"true"` - TrivyImageConsumer string `envconfig:"TRIVY_IMAGE_CONSUMER" required:"true"` - TrivySbomConsumer string `envconfig:"TRIVY_SBOM_CONSUMER" required:"true"` - KuberhealthyConsumer string `envconfig:"KUBERHEALTHY_CONSUMER" required:"true"` - AwsEnable bool `envconfig:"AWS_ENABLE" default:"false"` - AWSRegion string `envconfig:"AWS_REGION"` - AWSAccessKey string `envconfig:"AWS_ACCESS_KEY"` - AWSSecretKey string `envconfig:"AWS_SECRET_KEY"` - S3BucketName string `envconfig:"S3_BUCKET_NAME"` - S3ObjectKey string `envconfig:"S3_OBJECT_KEY"` + NatsAddress string `envconfig:"NATS_ADDRESS"` + NatsToken string `envconfig:"NATS_TOKEN"` + DbPort int `envconfig:"DB_PORT"` + DBAddress string `envconfig:"DB_ADDRESS"` + ClickHouseUsername string `envconfig:"CLICKHOUSE_USERNAME"` + ClickHousePassword string `envconfig:"CLICKHOUSE_PASSWORD"` + DgraphGrpcAddress string `envconfig:"DGRAPH_GRPC_ADDRESS"` + KubeAllResourcesConsumer string `envconfig:"KUBEALLRESOURCES_EVENTS_CONSUMER" required:"true"` + KetallConsumer string `envconfig:"KETALL_EVENTS_CONSUMER" required:"true"` + RakeesConsumer string `envconfig:"RAKEES_METRICS_CONSUMER" required:"true"` + OutdatedConsumer string `envconfig:"OUTDATED_EVENTS_CONSUMER" required:"true"` + DeprecatedConsumer string `envconfig:"DEPRECATED_API_CONSUMER" required:"true"` + DeletedConsumer string `envconfig:"DELETED_API_CONSUMER" required:"true"` + KubvizConsumer string `envconfig:"KUBVIZ_EVENTS_CONSUMER" required:"true"` + KubscoreConsumer string `envconfig:"KUBSCORE_CONSUMER" required:"true"` + TrivyConsumer string `envconfig:"TRIVY_CONSUMER" required:"true"` + TrivyImageConsumer string `envconfig:"TRIVY_IMAGE_CONSUMER" required:"true"` + TrivySbomConsumer string `envconfig:"TRIVY_SBOM_CONSUMER" required:"true"` + KuberhealthyConsumer string `envconfig:"KUBERHEALTHY_CONSUMER" required:"true"` + AwsEnable bool `envconfig:"AWS_ENABLE" default:"false"` + AWSRegion string `envconfig:"AWS_REGION"` + AWSAccessKey string `envconfig:"AWS_ACCESS_KEY"` + AWSSecretKey string `envconfig:"AWS_SECRET_KEY"` + S3BucketName string `envconfig:"S3_BUCKET_NAME"` + S3ObjectKey string `envconfig:"S3_OBJECT_KEY"` } type GraphQlConfig struct { diff --git a/client/pkg/dgraph/dgraph.go b/client/pkg/dgraph/dgraph.go new file mode 100644 index 00000000..d0d3ca07 --- /dev/null +++ b/client/pkg/dgraph/dgraph.go @@ -0,0 +1,72 @@ +package dgraph + +import ( + "context" + + "github.com/dgraph-io/dgo/v240" + "github.com/dgraph-io/dgo/v240/protos/api" +) + +// Dgraph schema for Kubernetes resources +const dgraphSchema = ` +group: string @index(exact) . +apiVersion: string @index(exact) . +kind: string @index(exact) . +metadata_name: string @index(exact) . +metadata_namespace: string @index(exact) . +metadata_uid: string @index(exact) . +metadata_labels: [uid] . +status_phase: string @index(exact) . +spec_nodeName: string @index(exact) . +isCurrent: bool @index(bool) . +lastUpdated: datetime . + +key: string @index(exact) . +value: string . + +relationships: [uid] @reverse . +targetResource: string @index(exact) . +relationshipType: string @index(exact) . +targetUID: string @index(exact) . +target: uid . + +type KubernetesResource { + dgraph.type: string + group: string + apiVersion: string + kind: string + + metadata_name: string + metadata_namespace: string + metadata_uid: string! + metadata_labels: [Label] + + isCurrent: bool + lastUpdated: DateTime + + status_phase: string + spec_nodeName: string + relationships: [Relationship] +} + +type Label { + key: string + value: string +} + +type Relationship { + dgraph.type: string + targetResource: string + relationshipType: string + targetUID: string + target: KubernetesResource +} +` + +// SetupDgraphSchema sets up the Dgraph schema for Kubernetes resources +func SetupDgraphSchema(ctx context.Context, dgraphClient *dgo.Dgraph) error { + op := &api.Operation{ + Schema: dgraphSchema, + } + return dgraphClient.Alter(ctx, op) +} diff --git a/client/pkg/dgraph/dgraph_kubernetes_resource.go b/client/pkg/dgraph/dgraph_kubernetes_resource.go new file mode 100644 index 00000000..95b96dde --- /dev/null +++ b/client/pkg/dgraph/dgraph_kubernetes_resource.go @@ -0,0 +1,250 @@ +package dgraph + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + + dgraphModel "github.com/intelops/kubviz/model/dgraph" + + "github.com/dgraph-io/dgo/v240" + "github.com/dgraph-io/dgo/v240/protos/api" +) + +var ErrResourceNotFound = errors.New("resource not found") + +// DgraphKubernetesResourceRepository handles Dgraph operations for KubernetesResource +type DgraphKubernetesResourceRepository struct { + client *dgo.Dgraph +} + +// NewDgraphKubernetesResourceRepository creates a new DgraphKubernetesResourceRepository instance +func NewDgraphKubernetesResourceRepository(client *dgo.Dgraph) *DgraphKubernetesResourceRepository { + return &DgraphKubernetesResourceRepository{client: client} +} + +func (repo *DgraphKubernetesResourceRepository) Save(ctx context.Context, resource *dgraphModel.KubernetesResource) error { + txn := repo.client.NewTxn() + defer txn.Discard(ctx) + + existingResource, err := repo.FindByKubernetesUID(ctx, resource.MetadataUID) + if err != nil && !errors.Is(err, ErrResourceNotFound) { + return fmt.Errorf("error checking existing resource: %v", err) + } + + var mutations []*api.Mutation + + if existingResource != nil { + resource.UID = existingResource.UID + + // Handle upsert labels + + // Compute label diffs + labelsToAdd, labelsToRemove, _ := diffLabels(existingResource.MetadataLabels, resource.MetadataLabels) + + // Prepare label removal mutation if needed + if len(labelsToRemove) > 0 { + deleteLabelsNquads := generateLabelDeleteNquads(resource.UID, labelsToRemove) + mutations = append(mutations, &api.Mutation{ + DelNquads: []byte(strings.Join(deleteLabelsNquads, "\n")), + }) + } + + // Prepare label addition mutations if needed + if len(labelsToAdd) > 0 { + addLabelsNquads := generateLabelAddNquads(resource.UID, labelsToAdd) + mutations = append(mutations, &api.Mutation{ + SetNquads: []byte(strings.Join(addLabelsNquads, "\n")), + }) + } + } + + resourceCopy := *resource + resourceCopy.DgraphType = []string{"KubernetesResource"} + + if existingResource != nil { + // Unsetting labels here to be handled in a separate mutation + resourceCopy.MetadataLabels = nil + } + + resourceJSON, err := json.Marshal(resourceCopy) + if err != nil { + return fmt.Errorf("error json marshaling resource: %v", err) + } + + mutations = append(mutations, &api.Mutation{ + SetJson: resourceJSON, + }) + + // Perform all mutations in a single request + request := &api.Request{ + Mutations: mutations, + CommitNow: true, + } + + _, err = txn.Do(ctx, request) + if err != nil { + return fmt.Errorf("error performing mutations: %v", err) + } + + return nil +} + +func diffLabels(oldLabels, newLabels []dgraphModel.Label) (toAdd, toRemove, toUpdate []dgraphModel.Label) { + oldMap := make(map[string]string) + for _, label := range oldLabels { + oldMap[label.Key] = label.Value + } + + newMap := make(map[string]string) + for _, label := range newLabels { + newMap[label.Key] = label.Value + } + + toAdd = []dgraphModel.Label{} + toRemove = []dgraphModel.Label{} + toUpdate = []dgraphModel.Label{} + + for _, newLabel := range newLabels { + if oldValue, exists := oldMap[newLabel.Key]; !exists { + toAdd = append(toAdd, newLabel) + } else if oldValue != newLabel.Value { + toUpdate = append(toUpdate, newLabel) + } + } + + for _, oldLabel := range oldLabels { + if _, exists := newMap[oldLabel.Key]; !exists { + toRemove = append(toRemove, oldLabel) + } + } + + return toAdd, toRemove, toUpdate +} + +func generateLabelDeleteNquads(uid string, labels []dgraphModel.Label) []string { + var nquads []string + for _, label := range labels { + nquad := fmt.Sprintf("<%s> * (key = %q) .", uid, label.Key) + nquads = append(nquads, nquad) + } + return nquads +} + +func generateLabelAddNquads(uid string, labelsToAdd []dgraphModel.Label) []string { + var nquads []string + for _, label := range labelsToAdd { + labelUID := fmt.Sprintf("_:label_%s", label.Key) + nquads = append(nquads, fmt.Sprintf("<%s> %s .", uid, labelUID)) + nquads = append(nquads, fmt.Sprintf("%s \"Label\" .", labelUID)) + nquads = append(nquads, fmt.Sprintf("%s %q .", labelUID, label.Key)) + nquads = append(nquads, fmt.Sprintf("%s %q .", labelUID, label.Value)) + } + return nquads +} + +// getResourceQueryFields returns the common query fields for KubernetesResource +func (repo *DgraphKubernetesResourceRepository) getResourceQueryFields() string { + return ` + uid + dgraph.type + group + apiVersion + kind + metadata_name + metadata_namespace + metadata_uid + metadata_labels { + key + value + } + status_phase + spec_nodeName + isCurrent + lastUpdated + additionalFields + relationships { + uid + dgraph.type + targetResource + relationshipType + targetUID + target { + uid + metadata_name + metadata_namespace + metadata_uid + } + } + ` +} + +// FindByDgraphUID retrieves a KubernetesResource by its Dgraph UID +func (repo *DgraphKubernetesResourceRepository) FindByDgraphUID(ctx context.Context, dgraphUID string) (*dgraphModel.KubernetesResource, error) { + txn := repo.client.NewTxn() + defer txn.Discard(ctx) + + q := fmt.Sprintf(` + query findResource($dgraphUID: string) { + resources(func: uid($dgraphUID)) @filter(eq(dgraph.type, "KubernetesResource")) { + %s + } + } + `, repo.getResourceQueryFields()) + + vars := map[string]string{"$dgraphUID": dgraphUID} + resp, err := txn.QueryWithVars(ctx, q, vars) + if err != nil { + return nil, fmt.Errorf("error querying Dgraph: %v", err) + } + + var result struct { + Resources []dgraphModel.KubernetesResource `json:"resources"` + } + + if err := json.Unmarshal(resp.Json, &result); err != nil { + return nil, fmt.Errorf("error unmarshaling response: %v", err) + } + + if len(result.Resources) == 0 { + return nil, ErrResourceNotFound + } + + return &result.Resources[0], nil +} + +// FindByKubernetesUID retrieves a KubernetesResource by its Kubernetes UID +func (repo *DgraphKubernetesResourceRepository) FindByKubernetesUID(ctx context.Context, k8sUID string) (*dgraphModel.KubernetesResource, error) { + txn := repo.client.NewTxn() + defer txn.Discard(ctx) + + q := fmt.Sprintf(` + query findResource($k8sUID: string) { + resources(func: eq(metadata_uid, $k8sUID)) @filter(eq(dgraph.type, "KubernetesResource")) { + %s + } + } + `, repo.getResourceQueryFields()) + + vars := map[string]string{"$k8sUID": k8sUID} + resp, err := txn.QueryWithVars(ctx, q, vars) + if err != nil { + return nil, fmt.Errorf("error querying Dgraph: %v", err) + } + + var result struct { + Resources []dgraphModel.KubernetesResource `json:"resources"` + } + + if err := json.Unmarshal(resp.Json, &result); err != nil { + return nil, fmt.Errorf("error unmarshaling response: %v", err) + } + + if len(result.Resources) == 0 { + return nil, ErrResourceNotFound + } + + return &result.Resources[0], nil +} diff --git a/client/pkg/dgraph/dgraph_kubernetes_resource_test.go b/client/pkg/dgraph/dgraph_kubernetes_resource_test.go new file mode 100644 index 00000000..73588ecf --- /dev/null +++ b/client/pkg/dgraph/dgraph_kubernetes_resource_test.go @@ -0,0 +1,97 @@ +package dgraph + +import ( + "testing" + + dgraphModel "github.com/intelops/kubviz/model/dgraph" + + "github.com/stretchr/testify/assert" +) + +func TestDiffLabels(t *testing.T) { + tests := []struct { + name string + oldLabels []dgraphModel.Label + newLabels []dgraphModel.Label + expectAdd []dgraphModel.Label + expectRemove []dgraphModel.Label + expectUpdate []dgraphModel.Label + }{ + { + name: "Add new label", + oldLabels: []dgraphModel.Label{{Key: "app", Value: "test"}}, + newLabels: []dgraphModel.Label{{Key: "app", Value: "test"}, {Key: "env", Value: "prod"}}, + expectAdd: []dgraphModel.Label{{Key: "env", Value: "prod"}}, + expectRemove: []dgraphModel.Label{}, + expectUpdate: []dgraphModel.Label{}, + }, + { + name: "Remove label", + oldLabels: []dgraphModel.Label{{Key: "app", Value: "test"}, {Key: "env", Value: "prod"}}, + newLabels: []dgraphModel.Label{{Key: "app", Value: "test"}}, + expectAdd: []dgraphModel.Label{}, + expectRemove: []dgraphModel.Label{{Key: "env", Value: "prod"}}, + expectUpdate: []dgraphModel.Label{}, + }, + { + name: "Update label", + oldLabels: []dgraphModel.Label{{Key: "app", Value: "test"}}, + newLabels: []dgraphModel.Label{{Key: "app", Value: "newtest"}}, + expectAdd: []dgraphModel.Label{}, + expectRemove: []dgraphModel.Label{}, + expectUpdate: []dgraphModel.Label{{Key: "app", Value: "newtest"}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + add, remove, update := diffLabels(tt.oldLabels, tt.newLabels) + assert.Equal(t, tt.expectAdd, add, "Added labels mismatch") + assert.Equal(t, tt.expectRemove, remove, "Removed labels mismatch") + assert.Equal(t, tt.expectUpdate, update, "Updated labels mismatch") + + // Additional checks to ensure slices are never nil + assert.NotNil(t, add, "Added labels should not be nil") + assert.NotNil(t, remove, "Removed labels should not be nil") + assert.NotNil(t, update, "Updated labels should not be nil") + }) + } +} + +func TestGenerateLabelDeleteNquads(t *testing.T) { + uid := "0x1234" + labels := []dgraphModel.Label{ + {Key: "app", Value: "test"}, + {Key: "env", Value: "prod"}, + } + + expected := []string{ + `<0x1234> * (key = "app") .`, + `<0x1234> * (key = "env") .`, + } + + result := generateLabelDeleteNquads(uid, labels) + assert.Equal(t, expected, result) +} + +func TestGenerateLabelAddNquads(t *testing.T) { + uid := "0x1234" + labels := []dgraphModel.Label{ + {Key: "app", Value: "test"}, + {Key: "env", Value: "prod"}, + } + + expected := []string{ + `<0x1234> _:label_app .`, + `_:label_app "Label" .`, + `_:label_app "app" .`, + `_:label_app "test" .`, + `<0x1234> _:label_env .`, + `_:label_env "Label" .`, + `_:label_env "env" .`, + `_:label_env "prod" .`, + } + + result := generateLabelAddNquads(uid, labels) + assert.Equal(t, expected, result) +} diff --git a/client/pkg/dgraph/dgraph_relationship.go b/client/pkg/dgraph/dgraph_relationship.go new file mode 100644 index 00000000..a0db2972 --- /dev/null +++ b/client/pkg/dgraph/dgraph_relationship.go @@ -0,0 +1,213 @@ +package dgraph + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/davecgh/go-spew/spew" + "github.com/dgraph-io/dgo/v240" + "github.com/dgraph-io/dgo/v240/protos/api" + "github.com/intelops/kubviz/client/pkg/kubernetes" +) + +type DgraphRelationshipRepository struct { + client *dgo.Dgraph +} + +func NewDgraphRelationshipRepository(client *dgo.Dgraph) *DgraphRelationshipRepository { + return &DgraphRelationshipRepository{client: client} +} + +func (repo *DgraphRelationshipRepository) Save(ctx context.Context, sourceUID string, relationships []kubernetes.RelationshipInfo) error { + txn := repo.client.NewTxn() + defer txn.Discard(ctx) + + // Find the source Dgraph UID + sourceDgraphUID, err := FindDgraphUIDByKubernetesUID(ctx, repo.client, sourceUID) + if err != nil { + return fmt.Errorf("failed to find source Dgraph UID: %v", err) + } + + // Prepare a mutation to delete existing relationships + deleteNquads := []string{ + fmt.Sprintf("<%s> * .", sourceDgraphUID), + } + + var setNquadsAll []string + + i := 0 + for _, info := range relationships { + var setNquads []string + spew.Dump(info) + var targetDgraphUID string + if info.TargetK8sUID == "" { + // Resolve by name + targetDgraphUID, err = getDgraphResourceUID(ctx, repo.client, info.TargetName, info.TargetNamespace, info.TargetResource) + if err != nil { + // TODO: logs + // fmt.Printf("error while finding Dgraph UID for target: %s\n", err.Error()) + continue + } + } else { + // Resolve by UID + targetDgraphUID, err = FindDgraphUIDByKubernetesUID(ctx, repo.client, info.TargetK8sUID) + if err != nil { + // TODO: logs + // fmt.Printf("error while finding Dgraph UID for target: %s\n", err.Error()) + continue + } + } + + // Create a new relationship node + relationshipUID := fmt.Sprintf("_:rel%d", i) // Create a blank node + setNquads = append(setNquads, fmt.Sprintf("%s \"Relationship\" .", relationshipUID)) + setNquads = append(setNquads, fmt.Sprintf("%s %q .", relationshipUID, info.TargetResource)) + setNquads = append(setNquads, fmt.Sprintf("%s %q .", relationshipUID, info.RelationshipType)) + + if info.TargetK8sUID != "" { + setNquads = append(setNquads, fmt.Sprintf("%s %q .", relationshipUID, info.TargetK8sUID)) + } + + // Link the new relationship to the target object: + // Relationship -> Target + setNquads = append(setNquads, fmt.Sprintf("%s <%s> .", relationshipUID, targetDgraphUID)) + + // Link the new relationship to the source object: + // Source -> Relationship -> Target + setNquads = append(setNquads, fmt.Sprintf("<%s> %s .", sourceDgraphUID, relationshipUID)) + + setNquadsAll = append(setNquadsAll, setNquads...) + + i++ + } + + // Combine delete and set mutations + mutation := &api.Mutation{ + DelNquads: []byte(strings.Join(deleteNquads, "\n")), + SetNquads: []byte(strings.Join(setNquadsAll, "\n")), + } + + // Perform the mutation + _, err = txn.Mutate(ctx, mutation) + if err != nil { + return fmt.Errorf("failed to update relationships: %v", err) + } + + // Commit the transaction + return txn.Commit(ctx) +} + +// FindDgraphUIDByKubernetesUID retrieves the Dgraph UID for a given Kubernetes UID +func FindDgraphUIDByKubernetesUID(ctx context.Context, client *dgo.Dgraph, k8sUID string) (string, error) { + txn := client.NewTxn() + defer txn.Discard(ctx) + + query := ` + query findDgraphUID($k8sUID: string) { + resource(func: eq(metadata_uid, $k8sUID)) { + uid + } + } + ` + vars := map[string]string{"$k8sUID": k8sUID} + resp, err := txn.QueryWithVars(ctx, query, vars) + if err != nil { + return "", fmt.Errorf("failed to query Dgraph: %v", err) + } + + var result struct { + Resource []struct { + UID string `json:"uid"` + } `json:"resource"` + } + + if err := json.Unmarshal(resp.Json, &result); err != nil { + return "", fmt.Errorf("failed to unmarshal Dgraph response: %v", err) + } + + if len(result.Resource) == 0 { + return "", fmt.Errorf("resource not found for UID: %s", k8sUID) + } + + return result.Resource[0].UID, nil +} + +func getDgraphResourceUID(ctx context.Context, dgraphClient *dgo.Dgraph, name, namespace, targetResource string) (string, error) { + parts := strings.SplitN(targetResource, "/", 3) + + group := "" + kind := "" + + if len(parts) == 1 { + kind = parts[0] + } else if len(parts) == 2 { + kind = parts[1] + } else { + group = parts[0] + kind = parts[2] + } + + var query string + var groupFilter string + + if group == "" { + groupFilter = "(not(has(group)) OR eq(group, \"\"))" + } else { + groupFilter = "eq(group, $group)" + } + + if namespace != "" { + // Query for namespaced resources + query = fmt.Sprintf(`query resource($name: string, $namespace: string, $group: string, $kind: string) { + resource(func: eq(dgraph.type, "KubernetesResource")) + @filter(eq(metadata_name, $name) AND + eq(metadata_namespace, $namespace) AND + %s AND + eq(kind, $kind)) { + uid + } + }`, groupFilter) + } else { + // Query for cluster-scoped resources + query = fmt.Sprintf(`query resource($name: string, $group: string, $kind: string) { + resource(func: eq(dgraph.type, "KubernetesResource")) + @filter(eq(metadata_name, $name) AND + %s AND + eq(kind, $kind)) { + uid + } + }`, groupFilter) + } + + vars := map[string]string{ + "$name": name, + "$group": group, + "$kind": kind, + } + if namespace != "" { + vars["$namespace"] = namespace + } + + resp, err := dgraphClient.NewTxn().QueryWithVars(ctx, query, vars) + if err != nil { + return "", fmt.Errorf("failed to query Dgraph: %v", err) + } + + var result struct { + Resource []struct { + UID string `json:"uid"` + } `json:"resource"` + } + + if err := json.Unmarshal(resp.Json, &result); err != nil { + return "", fmt.Errorf("failed to unmarshal Dgraph response: %v", err) + } + + if len(result.Resource) == 0 { + return "", fmt.Errorf("resource not found: ns: %s , name: %s (group: %s, kind: %s)", namespace, name, group, kind) + } + + return result.Resource[0].UID, nil +} diff --git a/client/pkg/kubernetes/kubernetes_relationship.go b/client/pkg/kubernetes/kubernetes_relationship.go new file mode 100644 index 00000000..3d09a7a0 --- /dev/null +++ b/client/pkg/kubernetes/kubernetes_relationship.go @@ -0,0 +1,178 @@ +package kubernetes + +import ( + "fmt" + "strings" + "time" + + dgraphModel "github.com/intelops/kubviz/model/dgraph" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/util/jsonpath" +) + +func ConvertKubernetesObjectToUnstructured(obj runtime.Object) (*unstructured.Unstructured, error) { + objUnst := &unstructured.Unstructured{} + + objMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + if err != nil { + return nil, err + } + + objUnst.SetUnstructuredContent(objMap) + objUnst.SetGroupVersionKind(obj.GetObjectKind().GroupVersionKind()) + + return objUnst, nil +} + +func ConvertUnstructuredToDgraphResource(obj *unstructured.Unstructured) (*dgraphModel.KubernetesResource, error) { + resource := &dgraphModel.KubernetesResource{ + DgraphType: []string{"KubernetesResource"}, + Group: obj.GroupVersionKind().Group, + APIVersion: obj.GroupVersionKind().Version, + Kind: obj.GroupVersionKind().Kind, + } + + resource.MetadataName = obj.GetName() + resource.MetadataNamespace = obj.GetNamespace() + resource.MetadataUID = string(obj.GetUID()) + + // Extract labels + for k, v := range obj.GetLabels() { + resource.MetadataLabels = append(resource.MetadataLabels, dgraphModel.Label{ + Key: k, + Value: v, + }) + } + + resource.IsCurrent = true + resource.LastUpdated = time.Now() + + // Example for specific fields + if obj.GetKind() == "Pod" { + resource.StatusPhase = getNestedString(obj.Object, "status", "phase") + resource.SpecNodeName = getNestedString(obj.Object, "spec", "nodeName") + } + + // Extract additional fields + resource.AdditionalFields = make(map[string]string) + if spec, ok := obj.Object["spec"].(map[string]interface{}); ok { + for k, v := range spec { + resource.AdditionalFields[fmt.Sprintf("spec_%s", k)] = fmt.Sprintf("%v", v) + } + } + + return resource, nil +} + +func getNestedString(obj map[string]interface{}, fields ...string) string { + val, found, err := unstructured.NestedString(obj, fields...) + if !found || err != nil { + return "" + } + return val +} + +func BuildRelationships(obj *unstructured.Unstructured) []RelationshipInfo { + relationships := []RelationshipInfo{} + + // Always check default paths + for jsonPath, relDef := range defaultRelationships { + relationships = append(relationships, buildRelationshipsFromPath(obj, jsonPath, relDef)...) + } + + // Check type-specific relationships + group := obj.GetObjectKind().GroupVersionKind().Group + version := obj.GetObjectKind().GroupVersionKind().Version + kind := obj.GetObjectKind().GroupVersionKind().Kind + + if groupRelationships, ok := kubernetesRelationships[group]; ok { + if versionRelationships, ok := groupRelationships[version]; ok { + if kindRelationships, ok := versionRelationships[kind]; ok { + for jsonPath, relDef := range kindRelationships { + relationships = append(relationships, buildRelationshipsFromPath(obj, jsonPath, relDef)...) + } + } + } + } + + return relationships +} + +func buildRelationshipsFromPath(obj *unstructured.Unstructured, jsonPath string, relDef RelationshipDef) []RelationshipInfo { + relationships := []RelationshipInfo{} + + values, err := getValuesFromJSONPath(obj.Object, "{"+jsonPath+"}") + if err != nil { + // TODO: log + // fmt.Printf("Error getting values from JSONPath %s: %v\n", jsonPath, err) + return relationships + } + + lastPart := getJSONPathLastPart(jsonPath) + + for _, value := range values { + if value == "" { + continue + } + + valueStr := value.(string) + targetNamespace := obj.GetNamespace() + if relDef.TargetResource == "v1/Namespace" { + targetNamespace = "" + } + + relInfo := RelationshipInfo{ + TargetResource: relDef.TargetResource, + TargetNamespace: targetNamespace, + RelationshipType: relDef.RelationshipType, + } + + if lastPart == "uid" { + relInfo.TargetK8sUID = valueStr + } else { + relInfo.TargetName = valueStr + } + + relationships = append(relationships, relInfo) + } + + return relationships +} + +func getValuesFromJSONPath(obj interface{}, jsonPathStr string) ([]interface{}, error) { + j := jsonpath.New("jsonpath") + j.AllowMissingKeys(true) + + err := j.Parse(jsonPathStr) + if err != nil { + return nil, fmt.Errorf("error parsing jsonpath %s: %v", jsonPathStr, err) + } + + results, err := j.FindResults(obj) + if err != nil { + return nil, fmt.Errorf("error finding results for jsonpath %s: %v", jsonPathStr, err) + } + + var values []interface{} + for _, result := range results { + for _, r := range result { + values = append(values, r.Interface()) + } + } + + if values == nil { + values = []interface{}{} + } + + return values, nil +} + +func getJSONPathLastPart(jsonPath string) string { + parts := strings.Split(jsonPath, ".") + if len(parts) == 0 { + return "" + } + return parts[len(parts)-1] +} diff --git a/client/pkg/kubernetes/kubernetes_relationship_map.go b/client/pkg/kubernetes/kubernetes_relationship_map.go new file mode 100644 index 00000000..7e50eeea --- /dev/null +++ b/client/pkg/kubernetes/kubernetes_relationship_map.go @@ -0,0 +1,69 @@ +package kubernetes + +type RelationshipDef struct { + TargetResource string + RelationshipType string +} + +type RelationshipInfo struct { + TargetK8sUID string + TargetResource string + TargetName string + TargetNamespace string + RelationshipType string +} + +var defaultRelationships = map[string]RelationshipDef{ + ".metadata.ownerReferences[*].uid": {"", "owned-by"}, + ".metadata.namespace": {"v1/Namespace", "belongs-to"}, +} + +var kubernetesRelationships = map[string]map[string]map[string]map[string]RelationshipDef{ + "": { // Core API group + "v1": { + "Pod": { + ".spec.volumes[*].configMap.name": {"v1/ConfigMap", "mount-volume"}, + ".spec.volumes[*].secret.secretName": {"v1/Secret", "mount-volume"}, + ".spec.containers[*].envFrom[*].configMapRef.name": {"v1/ConfigMap", "env-config"}, + ".spec.containers[*].envFrom[*].secretRef.name": {"v1/Secret", "env-config"}, + ".spec.serviceAccountName": {"v1/ServiceAccount", "use-account"}, + }, + }, + }, + "apps": { + "v1": { + "Deployment": { + ".spec.template.spec.volumes[*].configMap.name": {"v1/ConfigMap", "mount-volume"}, + ".spec.template.spec.volumes[*].secret.secretName": {"v1/Secret", "mount-volume"}, + ".spec.template.spec.containers[*].envFrom[*].configMapRef.name": {"v1/ConfigMap", "env-config"}, + ".spec.template.spec.containers[*].envFrom[*].secretRef.name": {"v1/Secret", "env-config"}, + ".spec.template.spec.serviceAccountName": {"v1/ServiceAccount", "use-account"}, + }, + "Statefulset": { + ".spec.template.spec.volumes[*].configMap.name": {"v1/ConfigMap", "mount-volume"}, + ".spec.template.spec.volumes[*].secret.secretName": {"v1/Secret", "mount-volume"}, + ".spec.template.spec.containers[*].envFrom[*].configMapRef.name": {"v1/ConfigMap", "env-config"}, + ".spec.template.spec.containers[*].envFrom[*].secretRef.name": {"v1/Secret", "env-config"}, + ".spec.template.spec.serviceAccountName": {"v1/ServiceAccount", "use-account"}, + }, + }, + }, + "networking.k8s.io": { + "v1": { + "Ingress": { + ".spec.rules[*].http.paths[*].backend.service.name": {"v1/Service", "route-traffic"}, + }, + }, + }, + "rbac.authorization.k8s.io": { + "v1": { + "Role": { + ".rules[*].resourceNames[*]": {"", "grant-access"}, + }, + "RoleBinding": { + ".roleRef.name": {"rbac.authorization.k8s.io/v1/Role", "bind-role"}, + ".subjects[*].name": {"", "assign-to"}, + }, + }, + }, +} diff --git a/client/pkg/kubernetes/kubernetes_relationship_test.go b/client/pkg/kubernetes/kubernetes_relationship_test.go new file mode 100644 index 00000000..272b048a --- /dev/null +++ b/client/pkg/kubernetes/kubernetes_relationship_test.go @@ -0,0 +1,177 @@ +package kubernetes + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func TestConvertUnstructuredToDgraphResource(t *testing.T) { + obj := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": map[string]interface{}{ + "name": "test-pod", + "namespace": "default", + "uid": "12345", + "labels": map[string]interface{}{ + "app": "test", + }, + }, + "status": map[string]interface{}{ + "phase": "Running", + }, + "spec": map[string]interface{}{ + "nodeName": "node1", + }, + }, + } + + result, err := ConvertUnstructuredToDgraphResource(obj) + + assert.NoError(t, err) + assert.Equal(t, "v1", result.APIVersion) + assert.Equal(t, "Pod", result.Kind) + assert.Equal(t, "test-pod", result.MetadataName) + assert.Equal(t, "default", result.MetadataNamespace) + assert.Equal(t, "12345", result.MetadataUID) + assert.Equal(t, "Running", result.StatusPhase) + assert.Equal(t, "node1", result.SpecNodeName) + assert.Len(t, result.MetadataLabels, 1) + assert.Equal(t, "app", result.MetadataLabels[0].Key) + assert.Equal(t, "test", result.MetadataLabels[0].Value) +} + +func TestGetNestedString(t *testing.T) { + obj := map[string]interface{}{ + "a": map[string]interface{}{ + "b": map[string]interface{}{ + "c": "value", + }, + }, + } + + tests := []struct { + name string + obj map[string]interface{} + fields []string + expected string + }{ + {"Existing nested field", obj, []string{"a", "b", "c"}, "value"}, + {"Non-existent field", obj, []string{"x", "y", "z"}, ""}, + {"Partially existing field", obj, []string{"a", "b", "d"}, ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := getNestedString(tt.obj, tt.fields...) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestBuildRelationships(t *testing.T) { + obj := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": map[string]interface{}{ + "name": "test-pod", + "namespace": "default", + "ownerReferences": []interface{}{ + map[string]interface{}{ + "kind": "Deployment", + "name": "test-deployment", + "uid": "67890", + }, + }, + }, + "spec": map[string]interface{}{ + "volumes": []interface{}{ + map[string]interface{}{ + "name": "config", + "configMap": map[string]interface{}{ + "name": "test-config", + }, + }, + }, + }, + }, + } + + relationships := BuildRelationships(obj) + + assert.Len(t, relationships, 3) + + // Check owner relationship + ownerRel := relationships[0] + assert.Equal(t, "owned-by", ownerRel.RelationshipType) + assert.Equal(t, "67890", ownerRel.TargetK8sUID) + + // Check namespace relationship + namespaceRel := relationships[1] + assert.Equal(t, "v1/Namespace", namespaceRel.TargetResource) + assert.Equal(t, "belongs-to", namespaceRel.RelationshipType) + assert.Equal(t, "default", namespaceRel.TargetName) + + // Check ConfigMap relationship + configMapRel := relationships[2] + assert.Equal(t, "v1/ConfigMap", configMapRel.TargetResource) + assert.Equal(t, "mount-volume", configMapRel.RelationshipType) + assert.Equal(t, "test-config", configMapRel.TargetName) +} + +func TestGetValuesFromJSONPath(t *testing.T) { + obj := map[string]interface{}{ + "items": []interface{}{ + map[string]interface{}{"name": "item1"}, + map[string]interface{}{"name": "item2"}, + }, + } + + tests := []struct { + name string + obj interface{} + jsonPath string + expectedVals []interface{} + expectError bool + }{ + {"Valid path", obj, "{.items[*].name}", []interface{}{"item1", "item2"}, false}, + {"Invalid path", obj, "{.nonexistent}", []interface{}{}, false}, + {"Syntax error", obj, "{.items[*}", nil, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + values, err := getValuesFromJSONPath(tt.obj, tt.jsonPath) + if tt.expectError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expectedVals, values) + } + }) + } +} + +func TestGetJSONPathLastPart(t *testing.T) { + tests := []struct { + name string + jsonPath string + expected string + }{ + {"Simple path", "metadata.name", "name"}, + {"Complex path", "spec.template.spec.containers[*].name", "name"}, + {"Empty path", "", ""}, + {"Single part", "uid", "uid"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := getJSONPathLastPart(tt.jsonPath) + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/constants/constants.go b/constants/constants.go index 6dbffaaa..4b010f03 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -1,33 +1,35 @@ package constants const ( - KUBERHEALTHY_SUBJECT = "METRICS.kuberhealthy" - KUBESCORE_SUBJECT = "METRICS.kubescore" - TRIVY_K8S_SUBJECT = "METRICS.trivyk8s" - StreamSubjects = "METRICS.*" - EventSubject = "METRICS.kubvizevent" - StreamName = "METRICS" - EventSubject_getall_resource = "METRICS.ketall" - EventSubject_outdated_images = "METRICS.outdated" - EventSubject_rakees = "METRICS.rakees" - EventSubject_deleted = "METRICS.deletedAPI" - EventSubject_depricated = "METRICS.deprecatedAPI" - KetallSubject = "METRICS.ketall" - KetallConsumer = "KETALL_EVENTS_CONSUMER" - RakeesSubject = "METRICS.rakees" - RakeesConsumer = "RAKEES_METRICS_CONSUMER" - OutdatedSubject = "METRICS.outdated" - OutdatedConsumer = "OUTDATED_EVENTS_CONSUMER" - DeprecatedSubject = "METRICS.deprecatedAPI" - DeprecatedConsumer = "DEPRECATED_API_CONSUMER" - DeletedSubject = "METRICS.deletedAPI" - DeletedConsumer = "DELETED_API_CONSUMER" - KubvizSubject = "METRICS.kubvizevent" - KubvizConsumer = "KUBVIZ_EVENTS_CONSUMER" - KubscoreConsumer = "KUBSCORE_CONSUMER" - TrivyConsumer = "TRIVY_CONSUMER" - TRIVY_IMAGE_SUBJECT = "METRICS.trivyimage" - Trivy_Image_Consumer = "TRIVY_IMAGE_CONSUMER" - TRIVY_SBOM_SUBJECT = "METRICS.trivysbom" - Trivy_Sbom_Consumer = "TRIVY_SBOM_CONSUMER" + KUBERHEALTHY_SUBJECT = "METRICS.kuberhealthy" + KUBESCORE_SUBJECT = "METRICS.kubescore" + TRIVY_K8S_SUBJECT = "METRICS.trivyk8s" + StreamSubjects = "METRICS.*" + EventSubject = "METRICS.kubvizevent" + StreamName = "METRICS" + EventSubject_kubeallresources = "METRICS.kubeallresources" + EventSubject_getall_resource = "METRICS.ketall" + EventSubject_outdated_images = "METRICS.outdated" + EventSubject_rakees = "METRICS.rakees" + EventSubject_deleted = "METRICS.deletedAPI" + EventSubject_depricated = "METRICS.deprecatedAPI" + KubeAllResourcesSubject = "METRICS.kubeallresources" + KetallSubject = "METRICS.ketall" + KetallConsumer = "KETALL_EVENTS_CONSUMER" + RakeesSubject = "METRICS.rakees" + RakeesConsumer = "RAKEES_METRICS_CONSUMER" + OutdatedSubject = "METRICS.outdated" + OutdatedConsumer = "OUTDATED_EVENTS_CONSUMER" + DeprecatedSubject = "METRICS.deprecatedAPI" + DeprecatedConsumer = "DEPRECATED_API_CONSUMER" + DeletedSubject = "METRICS.deletedAPI" + DeletedConsumer = "DELETED_API_CONSUMER" + KubvizSubject = "METRICS.kubvizevent" + KubvizConsumer = "KUBVIZ_EVENTS_CONSUMER" + KubscoreConsumer = "KUBSCORE_CONSUMER" + TrivyConsumer = "TRIVY_CONSUMER" + TRIVY_IMAGE_SUBJECT = "METRICS.trivyimage" + Trivy_Image_Consumer = "TRIVY_IMAGE_CONSUMER" + TRIVY_SBOM_SUBJECT = "METRICS.trivysbom" + Trivy_Sbom_Consumer = "TRIVY_SBOM_CONSUMER" ) diff --git a/dockerfiles/agent/kubviz/Dockerfile b/dockerfiles/agent/kubviz/Dockerfile index 8bdbf652..db89bb38 100644 --- a/dockerfiles/agent/kubviz/Dockerfile +++ b/dockerfiles/agent/kubviz/Dockerfile @@ -2,10 +2,16 @@ FROM golang:1.22 as builder WORKDIR /workspace + # Copy the Go Modules manifests -COPY ./ ./ +COPY ./go.mod ./ +COPY ./go.sum ./ RUN go mod download + +COPY ./ ./ + RUN gofmt -w -r '"github.com/googleapis/gnostic/OpenAPIv2" -> "github.com/googleapis/gnostic/openapiv2"' /go/pkg/mod/sigs.k8s.io/kustomize/ + # Build RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o kubviz_agent agent/kubviz/*.go diff --git a/dockerfiles/client/Dockerfile b/dockerfiles/client/Dockerfile index 19436734..7a70ec4d 100644 --- a/dockerfiles/client/Dockerfile +++ b/dockerfiles/client/Dockerfile @@ -2,10 +2,14 @@ FROM cgr.dev/chainguard/go:latest AS builder WORKDIR /workspace + # Copy the Go Modules manifests -COPY ./ ./ +COPY ./go.mod ./ +COPY ./go.sum ./ RUN go mod download +COPY ./ ./ + # Build RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o k8smetrics_client client/main.go diff --git a/go.mod b/go.mod index 61568e10..15dfce41 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/intelops/kubviz -go 1.22 +go 1.22.6 + +toolchain go1.23.0 require ( bou.ke/monkey v1.0.2 @@ -11,6 +13,7 @@ require ( github.com/blang/semver v3.5.1+incompatible github.com/corneliusweig/tabwriter v0.0.0-20190512204542-5f8a091e83b5 github.com/davecgh/go-spew v1.1.1 + github.com/dgraph-io/dgo/v240 v240.0.0 github.com/docker/distribution v2.8.2+incompatible github.com/docker/docker v24.0.4+incompatible github.com/genuinetools/reg v0.16.1 @@ -21,7 +24,7 @@ require ( github.com/golang-migrate/migrate/v4 v4.16.2 github.com/golang/mock v1.5.0 github.com/google/gnostic v0.5.7-v3refs - github.com/google/uuid v1.3.1 + github.com/google/uuid v1.6.0 github.com/hashicorp/go-version v1.6.0 github.com/intelops/go-common v1.0.19 github.com/kelseyhightower/envconfig v1.4.0 @@ -31,14 +34,19 @@ require ( github.com/robfig/cron/v3 v3.0.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.7.0 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 github.com/zegl/kube-score v1.17.0 + go.opentelemetry.io/collector/pdata v1.17.0 go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.46.1 - go.opentelemetry.io/otel v1.21.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 - go.opentelemetry.io/otel/sdk v1.21.0 - golang.org/x/term v0.14.0 + go.opentelemetry.io/otel v1.29.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.5.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.29.0 + go.opentelemetry.io/otel/log v0.5.0 + go.opentelemetry.io/otel/sdk v1.29.0 + go.opentelemetry.io/otel/sdk/log v0.5.0 + golang.org/x/term v0.23.0 + google.golang.org/grpc v1.67.1 k8s.io/api v0.27.3 k8s.io/apimachinery v0.27.3 k8s.io/cli-runtime v0.27.3 @@ -60,7 +68,7 @@ require ( github.com/aquasecurity/trivy-db v0.0.0-20230703082116-dc52e83376ce // indirect github.com/aquasecurity/trivy-kubernetes v0.5.7-0.20230628140707-dae3bdb6ee81 // indirect github.com/bytedance/sonic v1.9.1 // indirect - github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect @@ -73,7 +81,7 @@ require ( github.com/go-errors/errors v1.4.2 // indirect github.com/go-faster/city v1.0.1 // indirect github.com/go-faster/errors v0.6.1 // indirect - github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.1 // indirect @@ -83,7 +91,7 @@ require ( github.com/go-playground/validator/v10 v10.14.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.2 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-containerregistry v0.15.2 // indirect @@ -91,7 +99,7 @@ require ( github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/googleapis/gax-go/v2 v2.11.0 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/imdario/mergo v0.3.15 // indirect @@ -136,34 +144,32 @@ require ( github.com/showa-93/go-mask v0.6.0 // indirect github.com/spdx/tools-golang v0.5.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/objx v0.5.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect github.com/xlab/treeprint v1.1.0 // indirect go.etcd.io/bbolt v1.3.7 // indirect - go.opentelemetry.io/otel/metric v1.21.0 // indirect - go.opentelemetry.io/otel/trace v1.21.0 // indirect - go.opentelemetry.io/proto/otlp v1.0.0 // indirect + go.opentelemetry.io/otel/metric v1.29.0 // indirect + go.opentelemetry.io/otel/trace v1.29.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect go.uber.org/atomic v1.10.0 // indirect - go.uber.org/multierr v1.9.0 // indirect + go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.24.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.15.0 // indirect + golang.org/x/crypto v0.26.0 // indirect golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect - golang.org/x/net v0.18.0 // indirect - golang.org/x/oauth2 v0.11.0 // indirect - golang.org/x/sys v0.14.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/net v0.28.0 // indirect + golang.org/x/oauth2 v0.22.0 // indirect + golang.org/x/sys v0.24.0 // indirect + golang.org/x/text v0.17.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.9.3 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/api v0.126.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230913181813-007df8e322eb // indirect - google.golang.org/grpc v1.59.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240822170219-fc7c04adadcd // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 071cec4f..fbeff764 100644 --- a/go.sum +++ b/go.sum @@ -29,9 +29,8 @@ cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUM cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= -cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY= +cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= @@ -157,11 +156,11 @@ github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1 github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= @@ -197,6 +196,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/denverdino/aliyungo v0.0.0-20191023002520-dba750c0c223/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= +github.com/dgraph-io/dgo/v240 v240.0.0 h1:LgpaQoQuM8YD3/oHjjqzCfORPlw34OnIcA/jMiKuDJc= +github.com/dgraph-io/dgo/v240 v240.0.0/go.mod h1:YrKW6k5cJpG6qP+MtNlXBogNMTupDmnnmiF6heC0Uao= github.com/dhui/dktest v0.3.16 h1:i6gq2YQEtcrjKbeJpBkWjE8MmLZPYllcjOFbTZuPDnw= github.com/dhui/dktest v0.3.16/go.mod h1:gYaA3LRmM8Z4vJl2MA0THIigJoZrwOansEOsp+kqxp0= github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E= @@ -284,8 +285,8 @@ github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7 github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= @@ -329,8 +330,6 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-migrate/migrate/v4 v4.16.2 h1:8coYbMKUyInrFk1lfGfRovTLAW7PhWp8qQDT2iKfuoA= github.com/golang-migrate/migrate/v4 v4.16.2/go.mod h1:pfcJX4nPHaVdc5nmdCikFBWtm+UBpiZjRNNsyBbp0/o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= -github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -362,8 +361,8 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -417,8 +416,8 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaU github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -438,8 +437,8 @@ github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/grpc-gateway v1.11.1/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -687,8 +686,8 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rubenv/sql-migrate v1.3.1 h1:Vx+n4Du8X8VTYuXbhNxdEUoh6wiJERA0GlWocR5FrbA= github.com/rubenv/sql-migrate v1.3.1/go.mod h1:YzG/Vh82CwyhTFXy+Mf5ahAiiEOpAlHurg+23VEzcsk= github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww= @@ -730,8 +729,9 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -743,8 +743,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tchap/go-patricia/v2 v2.3.1 h1:6rQp39lgIYZ+MHmdEq4xzuk1t7OdC35z/xm0BGhTkes= github.com/tchap/go-patricia/v2 v2.3.1/go.mod h1:VZRHKAb53DLaG+nA9EaYYiaEx6YztwDlLElMsnSHD4k= github.com/terminalstatic/go-xsd-validate v0.1.5 h1:RqpJnf6HGE2CB/lZB1A8BYguk8uRtcvYAPLCF15qguo= @@ -798,24 +798,32 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/collector/pdata v1.17.0 h1:z8cjjT2FThAehWu5fbF48OnZyK5q8xd1UhC4XszDo0w= +go.opentelemetry.io/collector/pdata v1.17.0/go.mod h1:yZaQ9KZAm/qie96LTygRKxOXMq0/54h8OW7330ycuvQ= go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.46.1 h1:mMv2jG58h6ZI5t5S9QCVGdzCmAsTakMa3oxVgpSD44g= go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.46.1/go.mod h1:oqRuNKG0upTaDPbLVCG8AD0G2ETrfDtmh7jViy7ox6M= go.opentelemetry.io/contrib/propagators/b3 v1.21.1 h1:WPYiUgmw3+b7b3sQ1bFBFAf0q+Di9dvNc3AtYfnT4RQ= go.opentelemetry.io/contrib/propagators/b3 v1.21.1/go.mod h1:EmzokPoSqsYMBVK4nRnhsfm5mbn8J1eDuz/U1UaQaWg= -go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= -go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 h1:tIqheXEFWAZ7O8A7m+J0aPTmpJN3YQ7qetUAdkkkKpk= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0/go.mod h1:nUeKExfxAQVbiVFn32YXpXZZHZ61Cc3s3Rn1pDBGAb0= -go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= -go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= -go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= -go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= -go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= -go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= -go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= -go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= +go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.5.0 h1:iWyFL+atC9S1e6MFDLNUZieyKTmsrvsDzuozUDbFg8E= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.5.0/go.mod h1:0Ur7rPCJmkHksYcBywsFXnKBG3pqGl4TGltZ+T3qhSA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0 h1:dIIDULZJpgdiHz5tXrTgKIMLkus6jEFa7x5SOKcyR7E= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0/go.mod h1:jlRVBe7+Z1wyxFSUs48L6OBQZ5JwH2Hg/Vbl+t9rAgI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.29.0 h1:nSiV3s7wiCam610XcLbYOmMfJxB9gO4uK3Xgv5gmTgg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.29.0/go.mod h1:hKn/e/Nmd19/x1gvIHwtOwVWM+VhuITSWip3JUDghj0= +go.opentelemetry.io/otel/log v0.5.0 h1:x1Pr6Y3gnXgl1iFBwtGy1W/mnzENoK0w0ZoaeOI3i30= +go.opentelemetry.io/otel/log v0.5.0/go.mod h1:NU/ozXeGuOR5/mjCRXYbTC00NFJ3NYuraV/7O78F0rE= +go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= +go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= +go.opentelemetry.io/otel/sdk v1.29.0 h1:vkqKjk7gwhS8VaWb0POZKmIEDimRCMsopNYnriHyryo= +go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok= +go.opentelemetry.io/otel/sdk/log v0.5.0 h1:A+9lSjlZGxkQOr7QSBJcuyyYBw79CufQ69saiJLey7o= +go.opentelemetry.io/otel/sdk/log v0.5.0/go.mod h1:zjxIW7sw1IHolZL2KlSAtrUi8JHttoeiQy43Yl3WuVQ= +go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= +go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -823,8 +831,8 @@ go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= -go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= @@ -842,8 +850,8 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= -golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -882,8 +890,8 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= -golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -928,8 +936,8 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -942,8 +950,8 @@ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= -golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= +golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= +golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -956,8 +964,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1018,15 +1026,15 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= -golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= +golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1037,8 +1045,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1099,8 +1107,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= -golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1182,10 +1190,10 @@ google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaE google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= -google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= -google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230913181813-007df8e322eb h1:Isk1sSH7bovx8Rti2wZK0UZF6oraBDK74uoyLEEVFN0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230913181813-007df8e322eb/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/genproto/googleapis/api v0.0.0-20240822170219-fc7c04adadcd h1:BBOTEWLuuEGQy9n1y9MhVJ9Qt0BDu21X8qZs71/uPZo= +google.golang.org/genproto/googleapis/api v0.0.0-20240822170219-fc7c04adadcd/go.mod h1:fO8wJzT2zbQbAjbIoos1285VfEIYKDDY+Dt+WpTkh6g= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed h1:J6izYgfBXAI3xTKLgxzTmUltdYaLsuBxFCgDHWJ/eXg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1206,8 +1214,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1221,8 +1229,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/model/dgraph/types.go b/model/dgraph/types.go new file mode 100644 index 00000000..d8beec8b --- /dev/null +++ b/model/dgraph/types.go @@ -0,0 +1,38 @@ +package dgraph + +import "time" + +// KubernetesResource represents a Kubernetes resource in Dgraph +type KubernetesResource struct { + UID string `json:"uid,omitempty"` + DgraphType []string `json:"dgraph.type,omitempty"` + Group string `json:"group,omitempty"` + APIVersion string `json:"apiVersion,omitempty"` + Kind string `json:"kind,omitempty"` + MetadataName string `json:"metadata_name,omitempty"` + MetadataNamespace string `json:"metadata_namespace,omitempty"` + MetadataUID string `json:"metadata_uid,omitempty"` + MetadataLabels []Label `json:"metadata_labels,omitempty"` + StatusPhase string `json:"status_phase,omitempty"` + SpecNodeName string `json:"spec_nodeName,omitempty"` + IsCurrent bool `json:"isCurrent,omitempty"` + LastUpdated time.Time `json:"lastUpdated,omitempty"` + AdditionalFields map[string]string `json:"additionalFields,omitempty"` + Relationships []Relationship `json:"relationships,omitempty"` +} + +// Label represents a Kubernetes label in Dgraph +type Label struct { + Key string `json:"key,omitempty"` + Value string `json:"value,omitempty"` +} + +// DgraphRelationship represents a relationship between Kubernetes resources in Dgraph +type Relationship struct { + UID string `json:"uid,omitempty"` + DgraphType []string `json:"dgraph.type,omitempty"` + TargetResource string `json:"targetResource,omitempty"` + RelationshipType string `json:"relationshipType,omitempty"` + TargetUID string `json:"targetUID,omitempty"` + Target *KubernetesResource `json:"target,omitempty"` +} diff --git a/otel-collector/.gitignore b/otel-collector/.gitignore new file mode 100644 index 00000000..ae4a00cd --- /dev/null +++ b/otel-collector/.gitignore @@ -0,0 +1,2 @@ +_build +kubviz-otel-collector diff --git a/otel-collector/Dockerfile b/otel-collector/Dockerfile new file mode 100644 index 00000000..f5adc2b5 --- /dev/null +++ b/otel-collector/Dockerfile @@ -0,0 +1,11 @@ +FROM ubuntu + +ARG USER_UID=10001 +USER ${USER_UID} + +COPY --chmod=755 kubviz-otel-collector /kubviz-otel-collector +COPY config.yaml /etc/kubviz-otelcol-config/config.yaml +ENTRYPOINT ["/kubviz-otel-collector"] +CMD ["--config", "/etc/kubviz-otelcol-config/config.yaml"] +EXPOSE 4317 55678 55679 + diff --git a/otel-collector/builder-config.yaml b/otel-collector/builder-config.yaml new file mode 100644 index 00000000..09d10b18 --- /dev/null +++ b/otel-collector/builder-config.yaml @@ -0,0 +1,24 @@ +dist: + name: kubviz-otel-collector + description: KubViz OTel Collector distribution + output_path: _build + otelcol_version: 0.108.1 + +exporters: + - gomod: + go.opentelemetry.io/collector/exporter/debugexporter v0.108.1 + - gomod: + go.opentelemetry.io/collector/exporter/otlpexporter v0.108.1 + - gomod: + github.com/intelops/kubviz/pkg/opentelemetry/exporters/natsexporter v0.0.0-local + +processors: + - gomod: + go.opentelemetry.io/collector/processor/batchprocessor v0.108.1 + +receivers: + - gomod: + go.opentelemetry.io/collector/receiver/otlpreceiver v0.108.1 + +replaces: + - github.com/intelops/kubviz/pkg/opentelemetry/exporters/natsexporter => ../../pkg/opentelemetry/exporters/natsexporter diff --git a/otel-collector/config.yaml b/otel-collector/config.yaml new file mode 100644 index 00000000..e77f8cf5 --- /dev/null +++ b/otel-collector/config.yaml @@ -0,0 +1,26 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + max_recv_msg_size_mib: 50 + max_concurrent_streams: 100 + +processors: + batch: + +exporters: + debug: + verbosity: detailed + nats: + token: + url: nats://kubviz-client-nats:4222 + stream_name: kubviz-otel + subject_name: kubviz-otel + +service: + pipelines: + logs: + receivers: [otlp] + processors: [batch] + exporters: [nats] diff --git a/pkg/opentelemetry/exporters/natsexporter/config.go b/pkg/opentelemetry/exporters/natsexporter/config.go new file mode 100644 index 00000000..30631899 --- /dev/null +++ b/pkg/opentelemetry/exporters/natsexporter/config.go @@ -0,0 +1,9 @@ +package natsexporter + +// Config defines the configuration for NATS exporter. +type Config struct { + Token string `mapstructure:"token"` + URL string `mapstructure:"url"` + StreamName string `mapstructure:"stream_name"` + SubjectName string `mapstructure:"subject_name"` +} diff --git a/pkg/opentelemetry/exporters/natsexporter/doc.go b/pkg/opentelemetry/exporters/natsexporter/doc.go new file mode 100644 index 00000000..2917ab2c --- /dev/null +++ b/pkg/opentelemetry/exporters/natsexporter/doc.go @@ -0,0 +1,4 @@ +//go:generate mdatagen metadata.yaml + +// Package natsexporter exports data to NATS. +package natsexporter diff --git a/pkg/opentelemetry/exporters/natsexporter/factory.go b/pkg/opentelemetry/exporters/natsexporter/factory.go new file mode 100644 index 00000000..59418dc6 --- /dev/null +++ b/pkg/opentelemetry/exporters/natsexporter/factory.go @@ -0,0 +1,92 @@ +package natsexporter + +import ( + "context" + + "github.com/intelops/kubviz/pkg/opentelemetry/exporters/natsexporter/internal/metadata" + "github.com/nats-io/nats.go" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/exporter" + "go.opentelemetry.io/collector/exporter/exporterhelper" + "go.opentelemetry.io/collector/pdata/plog" + "go.uber.org/zap" +) + +// NewFactory creates a factory for OTLP exporter. +func NewFactory() exporter.Factory { + return exporter.NewFactory( + metadata.Type, + createDefaultConfig, + exporter.WithLogs(createLogsExporter, metadata.LogsStability)) +} + +type NatsExporter interface { + component.Component + consumeLogs(_ context.Context, ld plog.Logs) error +} + +func createDefaultConfig() component.Config { + return &Config{} +} + +func createLogsExporter( + ctx context.Context, + set exporter.Settings, + cfg component.Config, +) (exporter.Logs, error) { + natsExporter := getOrCreateNatsExporter(cfg, set.Logger) + return exporterhelper.NewLogsExporter( + ctx, + set, + cfg, + natsExporter.consumeLogs, + exporterhelper.WithStart(natsExporter.Start), + exporterhelper.WithShutdown(natsExporter.Shutdown), + exporterhelper.WithCapabilities(consumer.Capabilities{MutatesData: false}), + ) +} + +func getOrCreateNatsExporter(cfg component.Config, logger *zap.Logger) NatsExporter { + conf := cfg.(*Config) + return newNatsExporter(conf, logger) +} + +func newNatsExporter(conf *Config, logger *zap.Logger) NatsExporter { + logsMarshaler := createLogMarshaler() + + streamName := conf.StreamName + natsOptions := []nats.Option{} + if conf.Token != "" { + natsOptions = append(natsOptions, nats.Token(conf.Token)) + } + + js := NewJetstream(logger, conf.URL, natsOptions, streamName) + + err := js.Connect() + if err != nil { + logger.Error("failed to connect to NATS server", zap.Error(err)) + } + + streamConfig := nats.StreamConfig{ + Name: streamName, + Subjects: []string{conf.SubjectName}, + Retention: nats.LimitsPolicy, + Discard: nats.DiscardOld, + Storage: nats.FileStorage, + } + + err = js.Init(streamConfig) + if err != nil { + logger.Error("failed to init NATS Jetstream", zap.Error(err)) + } + + return &natsExporter{ + conf: conf, + logger: logger, + logsMarshaller: logsMarshaler, + jetStream: js, + subjectName: conf.SubjectName, + } +} diff --git a/pkg/opentelemetry/exporters/natsexporter/go.mod b/pkg/opentelemetry/exporters/natsexporter/go.mod new file mode 100644 index 00000000..e71b4cb8 --- /dev/null +++ b/pkg/opentelemetry/exporters/natsexporter/go.mod @@ -0,0 +1,48 @@ +module github.com/intelops/kubviz/pkg/opentelemetry/exporters/natsexporter + +go 1.22.0 + +require ( + github.com/nats-io/nats.go v1.37.0 + go.opentelemetry.io/collector/component v0.108.1 + go.opentelemetry.io/collector/consumer v0.108.1 + go.opentelemetry.io/collector/exporter v0.108.1 + go.opentelemetry.io/collector/pdata v1.14.1 + go.uber.org/zap v1.27.0 +) + +require ( + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/go-viper/mapstructure/v2 v2.1.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/knadh/koanf/maps v0.1.1 // indirect + github.com/knadh/koanf/providers/confmap v0.1.0 // indirect + github.com/knadh/koanf/v2 v2.1.1 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/nats-io/nkeys v0.4.7 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + go.opentelemetry.io/collector v0.108.1 // indirect + go.opentelemetry.io/collector/config/configretry v1.14.1 // indirect + go.opentelemetry.io/collector/config/configtelemetry v0.108.1 // indirect + go.opentelemetry.io/collector/confmap v1.14.1 // indirect + go.opentelemetry.io/collector/consumer/consumerprofiles v0.108.1 // indirect + go.opentelemetry.io/collector/extension v0.108.1 // indirect + go.opentelemetry.io/collector/pdata/pprofile v0.108.1 // indirect + go.opentelemetry.io/otel v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.28.0 // indirect + go.opentelemetry.io/otel/trace v1.28.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/sys v0.24.0 // indirect + golang.org/x/text v0.16.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect + google.golang.org/grpc v1.65.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/pkg/opentelemetry/exporters/natsexporter/go.sum b/pkg/opentelemetry/exporters/natsexporter/go.sum new file mode 100644 index 00000000..b93f93d1 --- /dev/null +++ b/pkg/opentelemetry/exporters/natsexporter/go.sum @@ -0,0 +1,165 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-viper/mapstructure/v2 v2.1.0 h1:gHnMa2Y/pIxElCH2GlZZ1lZSsn6XMtufpGyP1XxdC/w= +github.com/go-viper/mapstructure/v2 v2.1.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= +github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU= +github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU= +github.com/knadh/koanf/v2 v2.1.1 h1:/R8eXqasSTsmDCsAyYj+81Wteg8AqrV9CP6gvsTsOmM= +github.com/knadh/koanf/v2 v2.1.1/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/nats-io/nats.go v1.37.0 h1:07rauXbVnnJvv1gfIyghFEo6lUcYRY0WXc3x7x0vUxE= +github.com/nats-io/nats.go v1.37.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= +github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= +github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.20.1 h1:IMJXHOD6eARkQpxo8KkhgEVFlBNm+nkrFUyGlIu7Na8= +github.com/prometheus/client_golang v1.20.1/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/collector v0.108.1 h1:c3JZU5g5KezDXaMgL7GDFB7ihuLNzXo6eBuaJOmBiDA= +go.opentelemetry.io/collector v0.108.1/go.mod h1:7GL32WiQkZzJjxHstHme9igzYumDsw1hFPep3v1guHQ= +go.opentelemetry.io/collector/component v0.108.1 h1:X+Afj07brX0NC36t6PvAq+ehaeUO/Q9eJNOUwhInpeY= +go.opentelemetry.io/collector/component v0.108.1/go.mod h1:qrP2TiJT5qw4sSiAKne+gb+hb1qTixLXuQRZjMCnLjY= +go.opentelemetry.io/collector/component/componentstatus v0.108.1 h1:63B7/TsZ1wIT+JFopOAAvyM8GaWDyJQncQ/G12HR334= +go.opentelemetry.io/collector/component/componentstatus v0.108.1/go.mod h1:PAJmG8ip5gvEtJ9uPpLtVmwnxtjA8t4MmEbzg6nQxVs= +go.opentelemetry.io/collector/config/configretry v1.14.1 h1:kJBpb/1GGKktEa6taf4TgfFY34JD4WpCFVq9by+VWyU= +go.opentelemetry.io/collector/config/configretry v1.14.1/go.mod h1:KvQF5cfphq1rQm1dKR4eLDNQYw6iI2fY72NMZVa+0N0= +go.opentelemetry.io/collector/config/configtelemetry v0.108.1 h1:jjdH1R6FMz4l1nSvY7F3H0F09xW9LKTke7k3ZAPcmSU= +go.opentelemetry.io/collector/config/configtelemetry v0.108.1/go.mod h1:R0MBUxjSMVMIhljuDHWIygzzJWQyZHXXWIgQNxcFwhc= +go.opentelemetry.io/collector/confmap v1.14.1 h1:GPMa+q5ThiBFQaYKJ7xeomiw9tIokkTA1AiF1zwKJck= +go.opentelemetry.io/collector/confmap v1.14.1/go.mod h1:GrIZ12P/9DPOuTpe2PIS51a0P/ZM6iKtByVee1Uf3+k= +go.opentelemetry.io/collector/consumer v0.108.1 h1:75zHUpIDfgZMp3t9fYdpXXE6/wsBs9DwTZdfwS3+NDI= +go.opentelemetry.io/collector/consumer v0.108.1/go.mod h1:xu2pOTqK+uCFXZWd9RcU8s6sCRwK5GyuP64YuHLMzzA= +go.opentelemetry.io/collector/consumer/consumerprofiles v0.108.1 h1:inFvEN7nB9ECFUjJzv+TsylCM8r1ocqOUrnNVdkMkZo= +go.opentelemetry.io/collector/consumer/consumerprofiles v0.108.1/go.mod h1:F6Shxg3TqoDZe2+p2PkVRUsnJMqATQxbs4c1nfaJrAc= +go.opentelemetry.io/collector/consumer/consumertest v0.108.1 h1:WLvi27Vu5nkltjhmdN+pDO9EuRphYpOQxoqZ9A3RkKU= +go.opentelemetry.io/collector/consumer/consumertest v0.108.1/go.mod h1:tRqOtUukG76iBlPTAUwFSU87dDO+x33Gyn2x9mecfko= +go.opentelemetry.io/collector/exporter v0.108.1 h1:VjtbIwwR2B1GMf69FxVvwcKYIpbkC7v2wBxLt5vjYKI= +go.opentelemetry.io/collector/exporter v0.108.1/go.mod h1:nD32bomG/yYyqwTZQJFIhRhP++bbk+3oqlx+/RZZ6XY= +go.opentelemetry.io/collector/extension v0.108.1 h1:XNQ9bOegD38gktyLJXlGN2Wb6AbiBi2nAuiOIdg+zA0= +go.opentelemetry.io/collector/extension v0.108.1/go.mod h1:fZE5j4jL9XVbm4H53gj9y/VDhP/5assmZ4yU0EwSjBc= +go.opentelemetry.io/collector/pdata v1.14.1 h1:wXZjtQA7Vy5HFqco+yA95ENyMQU5heBB1IxMHQf6mUk= +go.opentelemetry.io/collector/pdata v1.14.1/go.mod h1:z1dTjwwtcoXxZx2/nkHysjxMeaxe9pEmYTEr4SMNIx8= +go.opentelemetry.io/collector/pdata/pprofile v0.108.1 h1:/XbunfZ+/jt1+d1p4zM4vZ/AgeaIJsayjYdlN1fV+tk= +go.opentelemetry.io/collector/pdata/pprofile v0.108.1/go.mod h1:/GvG2WcN9Dajlw4QaIOjgz7N32wSfPL3qxJ0BKOcVPo= +go.opentelemetry.io/collector/pdata/testdata v0.108.1 h1:TpBDoBMBYvC/Ibswe3Ec2eof8XrRrEec6+tfnTeTSGk= +go.opentelemetry.io/collector/pdata/testdata v0.108.1/go.mod h1:PdUmBA4yDRD4Wf0fpCyrpdZexz9EDoHBw5Ot4iIUPRs= +go.opentelemetry.io/collector/receiver v0.108.1 h1:YQgDv69v3fgd6uoiGZ+vUdUPdNzoodbLzjB7XfdQvxs= +go.opentelemetry.io/collector/receiver v0.108.1/go.mod h1:eKe/VJgdvHr8JsBDma/PF3DlaheTRC2X6AmCUByJCNU= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/exporters/prometheus v0.50.0 h1:2Ewsda6hejmbhGFyUvWZjUThC98Cf8Zy6g0zkIimOng= +go.opentelemetry.io/otel/exporters/prometheus v0.50.0/go.mod h1:pMm5PkUo5YwbLiuEf7t2xg4wbP0/eSJrMxIMxKosynY= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= +go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= +go.opentelemetry.io/otel/sdk/metric v1.28.0 h1:OkuaKgKrgAbYrrY0t92c+cC+2F6hsFNnCQArXCKlg08= +go.opentelemetry.io/otel/sdk/metric v1.28.0/go.mod h1:cWPjykihLAPvXKi4iZc1dpER3Jdq2Z0YLse3moQUCpg= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/opentelemetry/exporters/natsexporter/internal/metadata/generated_status.go b/pkg/opentelemetry/exporters/natsexporter/internal/metadata/generated_status.go new file mode 100644 index 00000000..a7c4209b --- /dev/null +++ b/pkg/opentelemetry/exporters/natsexporter/internal/metadata/generated_status.go @@ -0,0 +1,18 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "go.opentelemetry.io/collector/component" +) + +var ( + Type = component.MustNewType("nats") + ScopeName = "natsexporter" +) + +const ( + TracesStability = component.StabilityLevelAlpha + MetricsStability = component.StabilityLevelAlpha + LogsStability = component.StabilityLevelAlpha +) diff --git a/pkg/opentelemetry/exporters/natsexporter/marshalers.go b/pkg/opentelemetry/exporters/natsexporter/marshalers.go new file mode 100644 index 00000000..d027c61a --- /dev/null +++ b/pkg/opentelemetry/exporters/natsexporter/marshalers.go @@ -0,0 +1,29 @@ +package natsexporter + +import ( + "go.opentelemetry.io/collector/pdata/plog" +) + +type LogsMarshaler interface { + Marshal(logs plog.Logs) ([]byte, error) +} + +type pdataLogsMarshaler struct { + marshaler plog.Marshaler +} + +func createLogMarshaler() LogsMarshaler { + return newPdataLogsMarshaler(&plog.JSONMarshaler{}) +} + +func newPdataLogsMarshaler(marshaler plog.Marshaler) LogsMarshaler { + return pdataLogsMarshaler{ + marshaler: marshaler, + } +} + +func (p pdataLogsMarshaler) Marshal(ld plog.Logs) ([]byte, error) { + bts, err := p.marshaler.MarshalLogs(ld) + + return bts, err +} diff --git a/pkg/opentelemetry/exporters/natsexporter/metadata.yaml b/pkg/opentelemetry/exporters/natsexporter/metadata.yaml new file mode 100644 index 00000000..c2655d2e --- /dev/null +++ b/pkg/opentelemetry/exporters/natsexporter/metadata.yaml @@ -0,0 +1,9 @@ +type: nats + +status: + class: exporter + stability: + alpha: [logs] + distributions: [] + codeowners: + active: [andylibrian] diff --git a/pkg/opentelemetry/exporters/natsexporter/nats_client.go b/pkg/opentelemetry/exporters/natsexporter/nats_client.go new file mode 100644 index 00000000..2423bbed --- /dev/null +++ b/pkg/opentelemetry/exporters/natsexporter/nats_client.go @@ -0,0 +1,189 @@ +package natsexporter + +import ( + "errors" + "fmt" + "time" + + "github.com/nats-io/nats.go" + "go.uber.org/zap" +) + +// JetStreamConnection represents a connection to NATS JetStream. +type JetStreamConnection struct { + NATSConn *nats.Conn // The NATS connection. + JSContext nats.JetStreamContext // The JetStream context. +} + +// JetStream represents a message queue using NATS JetStream. +type JetStream struct { + URL string // The NATS server URL. + Options []nats.Option // NATS options. + StreamName string // The name of the JetStream stream. + Conn JetStreamConnection // The JetStream connection. + Subscription *nats.Subscription // NATS subscription for consuming messages. + logger *zap.Logger // A logger for logging messages and errors. +} + +// NewJetstream creates and returns a new JetStream instance. +func NewJetstream(logger *zap.Logger, url string, options []nats.Option, streamName string) *JetStream { + return &JetStream{ + URL: url, + Options: options, + StreamName: streamName, + logger: logger, + } +} + +// Connect establishes a connection to NATS JetStream. +func (j *JetStream) Connect() error { + nc, err := nats.Connect(j.URL, j.Options...) + if err != nil { + return fmt.Errorf("nats: jetstream connect: failed to connect to NATS server: %w", err) + } + + j.logger.Info("successfully connected to NATS server") + + jetStreamContext, err := nc.JetStream() + if err != nil { + return fmt.Errorf("nats: jetstream connect: failed to get JetStream context: %w", err) + } + + j.logger.Info("successfully got JetStream context") + j.Conn = JetStreamConnection{NATSConn: nc, JSContext: jetStreamContext} + + return nil +} + +// Init initializes the JetStream queue and consumer. +// +// Parameters: +// - streamConfig: Configuration for creating the JetStream stream. +// +// Returns: +// - error: An error if there is any issue initializing the JetStream queue and consumer. +func (j *JetStream) Init(streamConfig nats.StreamConfig) error { + if err := j.CreateStreamIfNotExist(streamConfig); err != nil { + return fmt.Errorf("nats: jetstream init: failed to create stream: %w", err) + } + + if _, err := j.CreateConsumer(); err != nil { + return fmt.Errorf("nats: jetstream init: failed to create consumer: %w", err) + } + + return j.CreateSubscription() +} + +// CreateStreamIfNotExist creates a JetStream stream if it doesn't already exist. +// +// Parameters: +// - streamConfig: Configuration for creating the JetStream stream. +// +// Returns: +// - error: An error if there is any issue creating the stream or if the stream already exists. +func (j *JetStream) CreateStreamIfNotExist(streamConfig nats.StreamConfig) error { + if j.Conn.JSContext == nil { + err := errors.New("cannot create stream due to nil connection") + return fmt.Errorf("nats: jetstream CreateStreamIfNotExist: %w", err) + } + + var err error + + streamInfo, err := j.Conn.JSContext.StreamInfo(j.StreamName) + if streamInfo != nil && err == nil { + j.logger.Info("stream already exists, skipping creation", zap.String("stream", j.StreamName)) + return nil + } + + if err != nil && err != nats.ErrStreamNotFound { + j.logger.Warn("error calling JetStream StreamInfo", zap.String("stream", j.StreamName), zap.Error(err)) + } + + j.logger.Info("jetstream: creating stream", zap.String("stream", j.StreamName), zap.Any("config", streamConfig)) + + _, err = j.Conn.JSContext.AddStream(&streamConfig) + if err != nil { + return fmt.Errorf("jetstream CreateStreamIfNotExist: error while creating stream: %s. %w", j.StreamName, err) + } + + j.logger.Info("jetstream: stream created", zap.String("stream", j.StreamName)) + + return nil +} + +// CreateConsumer creates a JetStream consumer for the stream. +// +// Returns: +// - *nats.ConsumerInfo: Information about the created JetStream consumer. +// - error: An error if there is any issue creating the consumer. +func (j *JetStream) CreateConsumer() (*nats.ConsumerInfo, error) { + return j.Conn.JSContext.AddConsumer(j.StreamName, &nats.ConsumerConfig{ + Durable: j.StreamName + "-TODO", + DeliverSubject: j.StreamName + "-DeliverSubject", + DeliverGroup: j.StreamName + "-TODO", + AckPolicy: nats.AckExplicitPolicy, + }) +} + +// CreateSubscription creates a NATS subscription for consuming messages from JetStream. +// +// Returns: +// - error: An error if there is any issue creating the subscription. +func (j *JetStream) CreateSubscription() error { + subscription, err := j.Conn.NATSConn.QueueSubscribeSync(j.StreamName+"-DeliverSubject", j.StreamName+"-TODO") + if err != nil { + return fmt.Errorf("nats: jetstream CreateSubscription: failed to create subscription: %w", err) + } + j.Subscription = subscription + return nil +} + +// Publish publishes a protobuf message to the JetStream stream. +// +// Returns: +// - error: An error if there is any issue publishing the message. +func (j *JetStream) Publish(subject string, data []byte) error { + err := j.publishWithRetry(subject, data) + if err != nil { + return fmt.Errorf("jetstream: failed to publish: %w", err) + } + + return nil +} + +func (j *JetStream) publishWithRetry(subject string, data []byte) error { + maxRetries := 5 + RetryInterval := 5 * time.Second + var err error + for i := 0; i < maxRetries; i++ { + _, err = j.Conn.JSContext.Publish(subject, data) + if err == nil { + return nil + } + + j.logger.Warn("jetstream: publish attempt failed", zap.Int("attempt", i+1), zap.Error(err)) + + time.Sleep(RetryInterval) + } + + return err +} + +// NextMessage retrieves the next message from the JetStream queue and unmarshals it into the provided protobuf message. +func (j *JetStream) NextMessage() ([]byte, error) { + msg, err := j.Subscription.NextMsg(1 * time.Hour) + if errors.Is(err, nats.ErrTimeout) { + return nil, err + } + + if err != nil { + return nil, fmt.Errorf("jetstream: error while getting the next message: %w", err) + } + + err = msg.Ack() + if err != nil { + return nil, fmt.Errorf("jetstream: failed to ack message: %w", err) + } + + return msg.Data, nil +} diff --git a/pkg/opentelemetry/exporters/natsexporter/nats_exporter.go b/pkg/opentelemetry/exporters/natsexporter/nats_exporter.go new file mode 100644 index 00000000..f9cb9705 --- /dev/null +++ b/pkg/opentelemetry/exporters/natsexporter/nats_exporter.go @@ -0,0 +1,45 @@ +package natsexporter + +import ( + "context" + "fmt" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/pdata/plog" + "go.uber.org/zap" +) + +type natsExporter struct { + conf *Config + subjectName string + logger *zap.Logger + logsMarshaller LogsMarshaler + jetStream *JetStream +} + +func (n *natsExporter) consumeLogs(ctx context.Context, logs plog.Logs) error { + marshalled, err := n.logsMarshaller.Marshal(logs) + if err != nil { + return err + } + + err = n.jetStream.Publish(n.subjectName, marshalled) + if err != nil { + return err + } + + // TODO: remove + fmt.Printf("natsExporter.consumeLogs() Log Marshalled: %s\n", string(marshalled)) + + return nil +} + +func (n *natsExporter) Start(_ context.Context, host component.Host) error { + n.logger.Info("natsexporter: started") + return nil +} + +func (n *natsExporter) Shutdown(context.Context) error { + n.logger.Info("natsexporter: shutting down") + return nil +}