Merge branch 'intelops:main' into main

.github/workflows/migration-image.yml

@@ -0,0 +1,79 @@
+name: Migration Docker Image CI
+
+on:
+  push:
+    paths-ignore:
+      - '**.md'
+    branches:
+      - 'main'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      id-token: write
+      contents: read
+      actions: read
+      security-events: write
+    env:
+      REGISTRY: ghcr.io
+      GH_URL: https://github.com
+    steps:
+      - name: Checkout GitHub Action
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Docker metadata
+        id: metadata
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ github.repository }}/migration
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=raw,value={{sha}},enable=${{ github.ref_type != 'tag' }}
+          flavor: |
+            latest=true
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build image and push to GitHub Container Registry
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./dockerfiles/migration/Dockerfile
+          tags: ${{ env.REGISTRY }}/${{ github.repository }}/migration:${{ github.run_id }}
+          labels: ${{ steps.metadata.outputs.labels }}
+          push: true
+
+      - name: Install cosign
+        uses: sigstore/cosign-installer@main
+
+      - name: Sign the images
+        run: |
+          cosign sign -y ${{ env.REGISTRY }}/${{ github.repository }}/migration:${{ github.run_id }}
+        env:
+          COSIGN_EXPERIMENTAL: 1
+
+      - name: Verify the pushed tags
+        run: cosign verify ${{ env.REGISTRY }}/${{ github.repository }}/migration:${{ github.run_id }} --certificate-identity ${{ env.GH_URL }}/${{ github.repository }}/.github/workflows/migration-image.yml@refs/heads/main  --certificate-oidc-issuer https://token.actions.githubusercontent.com
+        env:
+          COSIGN_EXPERIMENTAL: 1
+
+      - name: Run Trivy in GitHub SBOM mode and submit results to Dependency Graph
+        uses: aquasecurity/trivy-action@master
+        with:
+          scan-type: 'fs'
+          format: 'github'
+          output: 'dependency-results.sbom.json'
+          image-ref: '.'
+          github-pat: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/migration-pr.yml

@@ -0,0 +1,47 @@
+name: Migration Docker Image CI
+
+on:
+  pull_request:
+    branches:
+      - 'main'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      REGISTRY: ghcr.io
+      GH_URL: https://github.com
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+        with:
+            fetch-depth: 0
+
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      -
+        name: Login to ghcr registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      -
+        name: Build and push on PR
+        uses: docker/build-push-action@v4
+        if: github.event_name == 'pull_request'
+        with:
+          context: .
+          file: ./dockerfiles/migration/Dockerfile
+          push: true
+          tags: ${{ env.REGISTRY }}/${{ github.repository }}/migration:pr-${{ github.event.pull_request.number }}
+          build-args: |
+            "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}"

.github/workflows/migration-release.yml

@@ -0,0 +1,60 @@
+name: migration-release
+
+on:
+  push:
+    tags:
+      - "v*.*.*"
+
+jobs:
+  push_to_registry:
+    name: Build and push Docker image to GitHub container registry.
+    runs-on: ubuntu-20.04
+    permissions:
+      packages: write
+      id-token: write
+      contents: read
+      actions: read
+      security-events: write
+    env:
+      REGISTRY: ghcr.io
+      GH_URL: https://github.com
+    steps:
+      - name: Set environment variable
+        run: |
+          echo "RELEASE_VERSION=${GITHUB_REF:10}" >> $GITHUB_ENV
+      - name: Test environment variable
+        run: echo ${{ env.RELEASE_VERSION }}
+      - name: Check out GitHub repo
+        uses: actions/checkout@v3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build image and push to GitHub Container Registry
+        uses: docker/build-push-action@v4
+        with:
+          push: true
+          context: ./
+          file: ./dockerfiles/migration/Dockerfile
+          tags: ${{ env.REGISTRY }}/${{ github.repository }}/migration:${{ env.RELEASE_VERSION }}
+      - name: Install cosign
+        uses: sigstore/cosign-installer@main
+      - name: Sign the images
+        run: |
+          cosign sign -y ${{ env.REGISTRY }}/${{ github.repository }}/migration:${{ env.RELEASE_VERSION }}
+        env:
+          COSIGN_EXPERIMENTAL: 1
+      - name: Verify the pushed tags
+        run: cosign verify ${{ env.REGISTRY }}/${{ github.repository }}/migration:${{ env.RELEASE_VERSION }} --certificate-identity ${{ env.GH_URL }}/${{ github.repository }}/.github/workflows/migration-release.yml@refs/tags/${{ env.RELEASE_VERSION }}  --certificate-oidc-issuer https://token.actions.githubusercontent.com
+        env:
+          COSIGN_EXPERIMENTAL: 1
+      - name: Run Trivy in GitHub SBOM mode and submit results to Dependency Graph
+        uses: aquasecurity/trivy-action@master
+        with:
+          scan-type: 'fs'
+          format: 'github'
+          output: 'dependency-results.sbom.json'
+          image-ref: '.'
+          github-pat: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -1,2 +1,5 @@
 civo
-migration/
+alloc.svg
+allocs.pprof
+cpu.pprof
+steps-to-test.txt

agent/config/config.go

@@ -6,8 +6,15 @@ import (
 )
 
 type AgentConfigurations struct {
-	SANamespace string `envconfig:"SA_NAMESPACE" default:"default"`
-	SAName      string `envconfig:"SA_NAME" default:"default"`
+	SANamespace            string `envconfig:"SA_NAMESPACE" default:"default"`
+	SAName                 string `envconfig:"SA_NAME" default:"default"`
+	OutdatedInterval       string `envconfig:"OUTDATED_INTERVAL" default:"*/20 * * * *"`
+	GetAllInterval         string `envconfig:"GETALL_INTERVAL" default:"*/30 * * * *"`
+	KubeScoreInterval      string `envconfig:"KUBESCORE_INTERVAL" default:"*/40 * * * *"`
+	RakkessInterval        string `envconfig:"RAKKESS_INTERVAL" default:"*/50 * * * *"`
+	KubePreUpgradeInterval string `envconfig:"KUBEPREUPGRADE_INTERVAL" default:"*/60 * * * *"`
+	TrivyInterval          string `envconfig:"TRIVY_INTERVAL" default:"*/10 * * * *"`
+	SchedulerEnable        bool   `envconfig:"SCHEDULER_ENABLE" default:"false"`
 }
 
 func GetAgentConfigurations() (serviceConf *AgentConfigurations, err error) {

agent/kubviz/k8smetrics_agent.go

@@ -4,10 +4,14 @@ import (
 	"encoding/json"
 	"log"
 	"os"
+	"os/signal"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
+	"github.com/intelops/go-common/logging"
+
 	"github.com/go-co-op/gocron"
 	"github.com/nats-io/nats.go"
 
@@ -22,6 +26,7 @@ import (
 
 	"fmt"
 
+	"github.com/intelops/kubviz/agent/config"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/azure"
@@ -77,6 +82,10 @@ func main() {
 	log.SetFlags(log.LstdFlags | log.Lshortfile)
 	env := Production
 	clusterMetricsChan := make(chan error, 1)
+	cfg, err := config.GetAgentConfigurations()
+	if err != nil {
+		log.Fatal("Failed to retrieve agent configurations", err)
+	}
 	var (
 		config    *rest.Config
 		clientset *kubernetes.Clientset
@@ -126,15 +135,30 @@ func main() {
 	if schedulingIntervalStr == "" {
 		schedulingIntervalStr = "20m"
 	}
-	schedulingInterval, err := time.ParseDuration(schedulingIntervalStr)
-	if err != nil {
-		log.Fatalf("Failed to parse SCHEDULING_INTERVAL: %v", err)
+	if cfg.SchedulerEnable { // Assuming "cfg.Schedule" is a boolean indicating whether to schedule or not.
+		scheduler := initScheduler(config, js, *cfg, clientset)
+
+		// Start the scheduler
+		scheduler.Start()
+		signals := make(chan os.Signal, 1)
+		signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
+		<-signals
+
+		scheduler.Stop()
+	} else {
+		if schedulingIntervalStr == "" {
+			schedulingIntervalStr = "20m"
+		}
+		schedulingInterval, err := time.ParseDuration(schedulingIntervalStr)
+		if err != nil {
+			log.Fatalf("Failed to parse SCHEDULING_INTERVAL: %v", err)
+		}
+		s := gocron.NewScheduler(time.UTC)
+		s.Every(schedulingInterval).Do(func() {
+			collectAndPublishMetrics()
+		})
+		s.StartBlocking()
 	}
-	s := gocron.NewScheduler(time.UTC)
-	s.Every(schedulingInterval).Do(func() {
-		collectAndPublishMetrics()
-	})
-	s.StartBlocking()
 }
 
 // publishMetrics publishes stream of events
@@ -272,3 +296,68 @@ func watchK8sEvents(clientset *kubernetes.Clientset, js nats.JetStreamContext) {
 		time.Sleep(time.Second)
 	}
 }
+func initScheduler(config *rest.Config, js nats.JetStreamContext, cfg config.AgentConfigurations, clientset *kubernetes.Clientset) (s *Scheduler) {
+	log := logging.NewLogger()
+	s = NewScheduler(log)
+	if cfg.OutdatedInterval != "" {
+		sj, err := NewOutDatedImagesJob(config, js, cfg.OutdatedInterval)
+		if err != nil {
+			log.Fatal("no time interval", err)
+		}
+		err = s.AddJob("Outdated", sj)
+		if err != nil {
+			log.Fatal("failed to do job", err)
+		}
+	}
+	if cfg.GetAllInterval != "" {
+		sj, err := NewKetallJob(config, js, cfg.GetAllInterval)
+		if err != nil {
+			log.Fatal("no time interval", err)
+		}
+		err = s.AddJob("GetALL", sj)
+		if err != nil {
+			log.Fatal("failed to do job", err)
+		}
+	}
+	if cfg.KubeScoreInterval != "" {
+		sj, err := NewKubescoreJob(clientset, js, cfg.KubeScoreInterval)
+		if err != nil {
+			log.Fatal("no time interval", err)
+		}
+		err = s.AddJob("KubeScore", sj)
+		if err != nil {
+			log.Fatal("failed to do job", err)
+		}
+	}
+	if cfg.RakkessInterval != "" {
+		sj, err := NewRakkessJob(config, js, cfg.RakkessInterval)
+		if err != nil {
+			log.Fatal("no time interval", err)
+		}
+		err = s.AddJob("Rakkess", sj)
+		if err != nil {
+			log.Fatal("failed to do job", err)
+		}
+	}
+	if cfg.KubePreUpgradeInterval != "" {
+		sj, err := NewKubePreUpgradeJob(config, js, cfg.KubePreUpgradeInterval)
+		if err != nil {
+			log.Fatal("no time interval", err)
+		}
+		err = s.AddJob("KubePreUpgrade", sj)
+		if err != nil {
+			log.Fatal("failed to do job", err)
+		}
+	}
+	if cfg.TrivyInterval != "" {
+		sj, err := NewTrivyJob(config, js, cfg.TrivyInterval)
+		if err != nil {
+			log.Fatal("no time interval", err)
+		}
+		err = s.AddJob("Trivy", sj)
+		if err != nil {
+			log.Fatal("failed to do job", err)
+		}
+	}
+	return
+}