⭐️ add integration tests to cnspec

Makefile

@@ -115,11 +115,10 @@ test: test/go test/lint
 test/go: cnspec/generate test/go/plain
 
 test/go/plain:
-	# TODO /motor/docker/docker_engine cannot be executed inside of docker
-	go test -cover $(shell go list ./... | grep -v '/motor/discovery/docker_engine')
+	go test -cover $(shell go list ./...)
 
 test/go/plain-ci: prep/tools
-	gotestsum --junitfile report.xml --format pkgname -- -cover $(shell go list ./... | grep -v '/vendor/' | grep -v '/motor/discovery/docker_engine')
+	gotestsum --junitfile report.xml --format pkgname -- -cover $(shell go list ./... | grep -v '/vendor/')
 
 .PHONY: test/lint/staticcheck
 test/lint/staticcheck:

test/bundles.go

@@ -0,0 +1,70 @@
+// Copyright (c) Mondoo, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package test
+
+import (
+	"context"
+	"errors"
+	"go.mondoo.com/cnquery/v9/logger"
+	"go.mondoo.com/cnquery/v9/providers"
+	"go.mondoo.com/cnquery/v9/providers-sdk/v1/inventory"
+	"go.mondoo.com/cnspec/v9/policy"
+	"go.mondoo.com/cnspec/v9/policy/scan"
+	"os"
+	"testing"
+)
+
+func init() {
+	logger.Set("info")
+}
+
+func TestMain(m *testing.M) {
+	// There seems to be a small timing issue when provider installation is close to schema update.
+	// The provider is registered in the init() function to make sure it is loaded early
+	providers.EnsureProvider(providers.ProviderLookup{ID: "go.mondoo.com/cnquery/v9/providers/terraform"}, true, nil)
+	providers.EnsureProvider(providers.ProviderLookup{ID: "go.mondoo.com/cnquery/v9/providers/k8s"}, true, nil)
+
+	// Run tests
+	exitVal := m.Run()
+	os.Exit(exitVal)
+}
+
+func runBundle(policyBundlePath string, policyMrn string, asset *inventory.Asset) (*policy.Report, error) {
+	ctx := context.Background()
+	policyBundle, err := policy.BundleFromPaths(policyBundlePath)
+	if err != nil {
+		return nil, err
+	}
+
+	policyBundle.OwnerMrn = "//policy.api.mondoo.app"
+
+	policyFilters := []string{}
+	if policyMrn != "" {
+		policyFilters = append(policyFilters, policyMrn)
+	}
+
+	scanner := scan.NewLocalScanner(scan.WithRecording(providers.NullRecording{})) // TODO: fix recording
+	result, err := scanner.RunIncognito(ctx, &scan.Job{
+		Inventory: &inventory.Inventory{
+			Spec: &inventory.InventorySpec{
+				Assets: []*inventory.Asset{asset},
+			},
+		},
+		Bundle:        policyBundle,
+		PolicyFilters: policyFilters,
+		ReportType:    scan.ReportType_FULL,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	reports := result.GetFull().Reports
+	if len(reports) > 0 {
+		for _, report := range reports {
+			return report, nil
+		}
+	}
+
+	return nil, errors.New("no report found")
+}

test/k8s_test.go

@@ -0,0 +1,57 @@
+// Copyright (c) Mondoo, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.mondoo.com/cnquery/v9/providers-sdk/v1/inventory"
+)
+
+func TestKubernetesBundles(t *testing.T) {
+	type TestCase struct {
+		bundleFile string
+		testDir    string
+		policyMrn  string
+		score      uint32
+	}
+
+	tests := []TestCase{
+		{
+			bundleFile: "./testdata/mondoo-kubernetes-security.mql.yaml",
+			testDir:    "./testdata/k8s/pass/pod.yaml",
+			policyMrn:  "//policy.api.mondoo.app/policies/mondoo-kubernetes-security",
+			score:      100,
+		},
+		{
+			bundleFile: "./testdata/mondoo-kubernetes-security.mql.yaml",
+			testDir:    "./testdata/k8s/fail/pod-nonroot.yaml",
+			policyMrn:  "//policy.api.mondoo.app/policies/mondoo-kubernetes-security",
+			score:      0x0,
+		},
+	}
+
+	for i := range tests {
+		test := tests[i]
+		t.Run(test.testDir, func(t *testing.T) {
+			report, err := runBundle(test.bundleFile, test.policyMrn, &inventory.Asset{
+				Connections: []*inventory.Config{{
+					Type: "k8s",
+					Options: map[string]string{
+						"path": test.testDir,
+					},
+					Discover: &inventory.Discovery{
+						Targets: []string{"pods"}, // ignore the manifest which does not return anything
+					},
+				}},
+			})
+			require.NoError(t, err)
+
+			score := report.Scores[test.policyMrn]
+			assert.Equal(t, test.score, score.Value)
+		})
+	}
+}

test/terraform_test.go

@@ -0,0 +1,78 @@
+// Copyright (c) Mondoo, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.mondoo.com/cnquery/v9/providers-sdk/v1/inventory"
+)
+
+func TestTerraformBundles(t *testing.T) {
+	type TestCase struct {
+		bundleFile string
+		testDir    string
+		policyMrn  string
+		score      uint32
+	}
+
+	tests := []TestCase{
+		{
+			bundleFile: "./testdata/mondoo-terraform-aws-security.mql.yaml",
+			testDir:    "./testdata/terraform/aws-3.xx/pass",
+			policyMrn:  "//policy.api.mondoo.app/policies/mondoo-terraform-aws-security",
+			score:      100,
+		}, {
+			bundleFile: "./testdata/mondoo-terraform-aws-security.mql.yaml",
+			testDir:    "./testdata/terraform/aws-3.xx/fail",
+			policyMrn:  "//policy.api.mondoo.app/policies/mondoo-terraform-aws-security",
+			// NOTE: terraform-aws-security-s3-bucket-level-public-access-prohibited is not correctly implemented but needs pay the piper.
+			// 3/28/2022 - Tests are passing now but not for the right reasons. We still need to revisit this query since it involves testing
+			//             whether configuration was applied to a specific bucket.
+			score: 0,
+		}, {
+			bundleFile: "./testdata/mondoo-terraform-aws-security.mql.yaml",
+			testDir:    "./testdata/terraform/aws-4.xx/pass",
+			policyMrn:  "//policy.api.mondoo.app/policies/mondoo-terraform-aws-security",
+			score:      100,
+		}, {
+			bundleFile: "./testdata/mondoo-terraform-aws-security.mql.yaml",
+			testDir:    "./testdata/terraform/aws-4.xx/fail",
+			policyMrn:  "//policy.api.mondoo.app/policies/mondoo-terraform-aws-security",
+			score:      0,
+		}, {
+			bundleFile: "./testdata/mondoo-terraform-gcp-security.mql.yaml",
+			testDir:    "./testdata/terraform/gcp/pass",
+			policyMrn:  "//policy.api.mondoo.app/policies/mondoo-terraform-gcp-security",
+			score:      100,
+		}, {
+			bundleFile: "./testdata/mondoo-terraform-gcp-security.mql.yaml",
+			testDir:    "./testdata/terraform/gcp/fail",
+			policyMrn:  "//policy.api.mondoo.app/policies/mondoo-terraform-gcp-security",
+			score:      0,
+		},
+	}
+
+	for i := range tests {
+		test := tests[i]
+		t.Run(test.testDir, func(t *testing.T) {
+			report, err := runBundle(test.bundleFile, test.policyMrn, &inventory.Asset{
+				Connections: []*inventory.Config{
+					{
+						Type: "terraform-hcl",
+						Options: map[string]string{
+							"path": test.testDir,
+						},
+					},
+				},
+			})
+			require.NoError(t, err)
+
+			score := report.Scores[test.policyMrn]
+			assert.Equal(t, test.score, score.Value)
+		})
+	}
+}

test/testdata/k8s/fail/daemonset.yaml

@@ -0,0 +1,65 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: fluentd-elasticsearch
+  namespace: kube-system
+  labels:
+    k8s-app: fluentd-logging
+spec:
+  selector:
+    matchLabels:
+      name: fluentd-elasticsearch
+  template:
+    metadata:
+      labels:
+        name: fluentd-elasticsearch
+    spec:
+      serviceAccount: default
+      serviceAccountName: default
+      servicautomountServiceAccountTokeneAccount: true
+      hostNetwork: true
+      hostPID: true
+      hostIPC: true
+      hostAliases:
+        - ip: 127.0.0.1
+          hostnames:
+            - myadded.examplehostname
+      tolerations:
+        # these tolerations are to have the daemonset runnable on control plane nodes
+        # remove them if your control plane nodes should not run pods
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule
+      containers:
+        - name: fluentd-elasticsearch
+          image: quay.io/fluentd_elasticsearch/fluentd:v2.5.2
+          imagePullPolicy: Always
+          resources:
+            limits:
+              memory: 200Mi
+            requests:
+              cpu: 100m
+              memory: 200Mi
+          securityContext:
+            privileged: true
+            readOnlyRootFilesystem: false
+            runAsNonRoot: false
+            runAsUser: 0
+            allowPrivilegeEscalation: true
+          volumeMounts:
+            - name: varlog
+              mountPath: /var/log
+            - name: varlibdockercontainers
+              mountPath: /var/lib/docker/containers
+              readOnly: true
+      terminationGracePeriodSeconds: 30
+      volumes:
+        - name: varlog
+          hostPath:
+            path: /var/log
+        - name: varlibdockercontainers
+          hostPath:
+            path: /var/lib/docker/containers

test/testdata/k8s/fail/pod-nonroot.yaml

@@ -0,0 +1,37 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: hello-pod
+  namespace: default
+spec:
+  serviceAccountName: mondoo-test
+  serviceAccount: mondoo-test
+  securityContext:
+    runAsNonRoot: true
+  containers:
+    - name: nginx
+      image: nginx:1.21.6
+      ports:
+        - containerPort: 8080
+      imagePullPolicy: Always
+      securityContext:
+        runAsNonRoot: false
+        privileged: false
+        readOnlyRootFilesystem: true
+      resources:
+        limits:
+          cpu: 100m
+          memory: 200Mi
+        requests:
+          cpu: 100m
+          memory: 200Mi
+      readinessProbe:
+        tcpSocket:
+          port: 8080
+        initialDelaySeconds: 5
+        periodSeconds: 10
+      livenessProbe:
+        tcpSocket:
+          port: 8080
+        initialDelaySeconds: 15
+        periodSeconds: 20

test/testdata/k8s/pass/batch.yaml

@@ -0,0 +1,49 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: hello-batch
+  namespace: sample
+spec:
+  template:
+    spec:
+      serviceAccountName: mondoo-test
+      serviceAccount: mondoo-test
+      containers:
+        - name: hello
+          image: busybox:v3
+          imagePullPolicy: Always
+          command: [ 'sh', '-c', 'echo "Hello, Kubernetes!" && sleep 3600' ]
+          livenessProbe:
+            exec:
+              command:
+              - /bin/sh
+              - -c
+              - exit 0
+            failureThreshold: 3
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+          readinessProbe:
+            exec:
+              command:
+              - /bin/sh
+              - -c
+              - exit 0
+            failureThreshold: 3
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+          resources:
+            limits:
+              cpu: 100m
+              memory: 100Mi
+            requests:
+              cpu: 100m
+              memory: 100Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            privileged: false
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 1000
+      restartPolicy: OnFailure