OTEL refactor

.github/workflows/agent-container-pr.yml

@@ -8,6 +8,10 @@ on:
     branches:
       - 'main'
 
+permissions:
+  contents: read
+  packages: write
+
 jobs:
   build:
     runs-on: ubuntu-latest

.github/workflows/agent-git-pr.yml

@@ -8,6 +8,10 @@ on:
     branches:
       - 'main'
 
+permissions:
+  contents: read
+  packages: write
+
 jobs:
   build:
     runs-on: ubuntu-latest

.github/workflows/agent-kubviz-pr.yml

@@ -8,6 +8,10 @@ on:
     branches:
       - 'main'
 
+permissions:
+  contents: read
+  packages: write
+
 jobs:
   build:
     runs-on: ubuntu-latest

.github/workflows/client-pr.yml

@@ -8,6 +8,10 @@ on:
     branches:
       - 'main'
 
+permissions:
+  contents: read
+  packages: write
+
 jobs:
   build:
     runs-on: ubuntu-latest

.github/workflows/migration-pr.yml

@@ -8,6 +8,10 @@ on:
     branches:
       - 'main'
 
+permissions:
+  contents: read
+  packages: write
+
 jobs:
   build:
     runs-on: ubuntu-latest

.github/workflows/otel-collector-image.yml

@@ -0,0 +1,99 @@
+name: OTEL Collector Docker Image CI
+
+on:
+  push:
+    paths-ignore:
+      - '**.md'
+      - 'charts/**'
+    branches:
+      - 'main'
+
+jobs:
+
+  build:
+
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      id-token: write
+      contents: read
+      actions: read
+      security-events: write
+    env:
+      REGISTRY: ghcr.io
+      GH_URL: https://github.com
+    steps:
+      - name: Checkout GitHub Action
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Docker metadata
+        id: metadata
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ github.repository }}/otel-collector
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=raw,value={{sha}},enable=${{ github.ref_type != 'tag' }}
+          flavor: |
+            latest=true
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      -
+        name: Set up Go
+        uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed #5.1.0
+        with:
+          go-version: "1.22"
+          cache: false
+
+      -
+        name: Build OpenTelemetry Collector
+        working-directory: otel-collector
+        run: |
+          curl --proto '=https' --tlsv1.2 -fL -o ocb https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/cmd%2Fbuilder%2Fv0.112.0/ocb_0.112.0_linux_amd64
+          chmod +x ocb
+          ./ocb --config builder-config.yaml
+          cp _build/kubviz-otel-collector .
+
+      - name: Build image and push to GitHub Container Registry
+        uses: docker/build-push-action@v4
+        with:
+          context: otel-collector
+          file: ./otel-collector/Dockerfile
+          tags: ${{ env.REGISTRY }}/${{ github.repository }}/otel-collector:${{ github.run_id }}
+          labels: ${{ steps.metadata.outputs.labels }}
+
+          push: true
+
+      - name: Install cosign
+        uses: sigstore/cosign-installer@main
+
+      - name: Sign the images
+        run: |
+          cosign sign -y ${{ env.REGISTRY }}/${{ github.repository }}/otel-collector:${{ github.run_id }}
+        env:
+          COSIGN_EXPERIMENTAL: 1
+
+      - name: Verify the pushed tags
+        run: cosign verify ${{ env.REGISTRY }}/${{ github.repository }}/otel-collector:${{ github.run_id }} --certificate-identity ${{ env.GH_URL }}/${{ github.repository }}/.github/workflows/otel-collector-image.yml@refs/heads/main  --certificate-oidc-issuer https://token.actions.githubusercontent.com
+        env:
+          COSIGN_EXPERIMENTAL: 1
+
+      - name: Run Trivy in GitHub SBOM mode and submit results to Dependency Graph
+        uses: aquasecurity/trivy-action@master
+        with:
+          scan-type: 'fs'
+          format: 'github'
+          output: 'dependency-results.sbom.json'
+          image-ref: '.'
+          github-pat: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/otel-collector-pr.yml

@@ -0,0 +1,70 @@
+name: OTEL Collector Docker Image CI
+
+on:
+  pull_request:
+    paths-ignore:
+      - '**.md'
+      - 'charts/**'
+    branches:
+      - 'main'
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      REGISTRY: ghcr.io
+      GH_URL: https://github.com
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+        with:
+            fetch-depth: 0
+
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - uses: docker/setup-buildx-action@v1
+        name: Set up Docker Buildx
+
+      -
+        name: Login to ghcr registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      -
+        name: Set up Go
+        uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed #5.1.0
+        with:
+          go-version: "1.22"
+          cache: false
+
+      -
+        name: Build OpenTelemetry Collector
+        working-directory: otel-collector
+        run: |
+          curl --proto '=https' --tlsv1.2 -fL -o ocb https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/cmd%2Fbuilder%2Fv0.112.0/ocb_0.112.0_linux_amd64
+          chmod +x ocb
+          ./ocb --config builder-config.yaml
+          cp _build/kubviz-otel-collector .
+
+      -
+        name: Build and push on PR
+        uses: docker/build-push-action@v4
+        if: github.event_name == 'pull_request'
+        with:
+          context: otel-collector
+          file: ./otel-collector/Dockerfile
+          push: true
+          tags: ${{ env.REGISTRY }}/${{ github.repository }}/otel-collector:pr-${{ github.event.pull_request.number }}
+          build-args: |
+            "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}"
+

.github/workflows/otel-collector-release.yml

@@ -0,0 +1,72 @@
+name: otel-collector-release
+on:
+  push:
+    tags:
+      - "v*.*.*"
+jobs:
+  push_to_registry:
+    name: Build and push Docker image github container registry.
+    runs-on: ubuntu-20.04
+    permissions:
+      packages: write
+      id-token: write
+      contents: read
+      actions: read
+      security-events: write
+    env:
+      REGISTRY: ghcr.io
+      GH_URL: https://github.com
+    steps:
+      - name: Set environment variable
+        run: |
+          echo "RELEASE_VERSION=${GITHUB_REF:10}" >> $GITHUB_ENV
+      - name: Test environment variable
+        run: echo ${{ env.RELEASE_VERSION }}
+      - name: Check out GitHub repo
+        uses: actions/checkout@v3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Set up Go
+        uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed #5.1.0
+        with:
+          go-version: "1.22"
+          cache: false
+      -
+        name: Build OpenTelemetry Collector
+        working-directory: otel-collector
+        run: |
+          curl --proto '=https' --tlsv1.2 -fL -o ocb https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/cmd%2Fbuilder%2Fv0.112.0/ocb_0.112.0_linux_amd64
+          chmod +x ocb
+          ./ocb --config builder-config.yaml
+          cp _build/kubviz-otel-collector .
+      - name: Build image and push to GitHub Container Registry
+        uses: docker/build-push-action@v4
+        with:
+          push: true
+          context: otel-collector
+          file: ./otel-collector/Dockerfile
+          tags: ${{ env.REGISTRY }}/${{ github.repository }}/otel-collector:${{ env.RELEASE_VERSION }}
+      - name: Install cosign
+        uses: sigstore/cosign-installer@main
+      - name: Sign the images
+        run: |
+          cosign sign -y ${{ env.REGISTRY }}/${{ github.repository }}/otel-collector:${{ env.RELEASE_VERSION }}
+        env:
+          COSIGN_EXPERIMENTAL: 1
+      - name: Verify the pushed tags
+        run: cosign verify ${{ env.REGISTRY }}/${{ github.repository }}/otel-collector:${{ env.RELEASE_VERSION }} --certificate-identity ${{ env.GH_URL }}/${{ github.repository }}/.github/workflows/otel-collector-release.yml@refs/tags/${{ env.RELEASE_VERSION }}  --certificate-oidc-issuer https://token.actions.githubusercontent.com
+        env:
+          COSIGN_EXPERIMENTAL: 1
+      - name: Run Trivy in GitHub SBOM mode and submit results to Dependency Graph
+        uses: aquasecurity/trivy-action@master
+        with:
+          scan-type: 'fs'
+          format: 'github'
+          output: 'dependency-results.sbom.json'
+          image-ref: '.'
+          github-pat: ${{ secrets.GITHUB_TOKEN }} # or ${{ secrets.github_pat_name }} if you're using a PAT

.gitignore

@@ -4,3 +4,5 @@ allocs.pprof
 cpu.pprof
 steps-to-test.txt
 Dockerfile-grphqlserver-build
+vendor
+kubviz_agent

agent/config/config.go

@@ -11,6 +11,8 @@ type AgentConfigurations struct {
 	SANamespace              string `envconfig:"SA_NAMESPACE" default:"default"`
 	SAName                   string `envconfig:"SA_NAME" default:"default"`
 	OutdatedInterval         string `envconfig:"OUTDATED_INTERVAL" default:"0"`
+	KubeAllResourcesInterval string `envconfig:"KUBE_ALL_RESOURCES_INTERVAL" default:"*/30 * * * *"`
+	KubeAllResourcesEnabled  bool   `envconfig:"KUBE_ALL_RESOURCES_ENABLED" default:"true"`
 	GetAllInterval           string `envconfig:"GETALL_INTERVAL" default:"*/30 * * * *"`
 	KubeScoreInterval        string `envconfig:"KUBESCORE_INTERVAL" default:"*/40 * * * *"`
 	RakkessInterval          string `envconfig:"RAKKESS_INTERVAL" default:"*/50 * * * *"`