From c2d59a214edf98dbb9a6da903da3469c7f08d150 Mon Sep 17 00:00:00 2001 From: Karan Kumar Date: Wed, 7 Feb 2024 22:40:51 +0530 Subject: [PATCH] Add e2e tests to verify vulnerability scanning --- test/e2e/v1alpha1/common_cbs_test.go | 85 ++++++++++++++ test/e2e/v1alpha1/common_suite_test.go | 5 + .../e2e_vulnerability_scanning_test.go | 107 ++++++++++++++++++ test/e2e/v1alpha1/validators_test.go | 7 +- test/e2e/v1beta1/common_cbs_test.go | 85 ++++++++++++++ test/e2e/v1beta1/common_suite_test.go | 5 + .../e2e_vulnerability_scanning_test.go | 105 +++++++++++++++++ test/e2e/v1beta1/validators_test.go | 7 +- 8 files changed, 402 insertions(+), 4 deletions(-) create mode 100644 test/e2e/v1alpha1/common_cbs_test.go create mode 100644 test/e2e/v1alpha1/e2e_vulnerability_scanning_test.go create mode 100644 test/e2e/v1beta1/common_cbs_test.go create mode 100644 test/e2e/v1beta1/e2e_vulnerability_scanning_test.go diff --git a/test/e2e/v1alpha1/common_cbs_test.go b/test/e2e/v1alpha1/common_cbs_test.go new file mode 100644 index 0000000000..bb95e0c79c --- /dev/null +++ b/test/e2e/v1alpha1/common_cbs_test.go @@ -0,0 +1,85 @@ +// Copyright The Shipwright Contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package e2e_test + +import ( + "context" + + . "github.com/onsi/gomega" + + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + + buildv1alpha1 "github.com/shipwright-io/build/pkg/apis/build/v1alpha1" +) + +type clusterBuildStrategyPrototype struct { + clusterBuildStrategy buildv1alpha1.ClusterBuildStrategy +} + +func NewClusterBuildStrategyPrototype() *clusterBuildStrategyPrototype { + return &clusterBuildStrategyPrototype{ + clusterBuildStrategy: buildv1alpha1.ClusterBuildStrategy{}, + } +} + +func (c *clusterBuildStrategyPrototype) Name(name string) *clusterBuildStrategyPrototype { + c.clusterBuildStrategy.ObjectMeta.Name = name + return c +} + +func (c *clusterBuildStrategyPrototype) BuildStep(buildStep buildv1alpha1.BuildStep) *clusterBuildStrategyPrototype { + c.clusterBuildStrategy.Spec.BuildSteps = append(c.clusterBuildStrategy.Spec.BuildSteps, buildStep) + return c +} + +func (c *clusterBuildStrategyPrototype) Parameter(parameter buildv1alpha1.Parameter) *clusterBuildStrategyPrototype { + c.clusterBuildStrategy.Spec.Parameters = append(c.clusterBuildStrategy.Spec.Parameters, parameter) + return c +} + +func (c *clusterBuildStrategyPrototype) Volume(volume buildv1alpha1.BuildStrategyVolume) *clusterBuildStrategyPrototype { + c.clusterBuildStrategy.Spec.Volumes = append(c.clusterBuildStrategy.Spec.Volumes, volume) + return c +} + +func (c *clusterBuildStrategyPrototype) Create() (cbs *buildv1alpha1.ClusterBuildStrategy, err error) { + ctx := context.Background() + + _, err = testBuild. + BuildClientSet. + ShipwrightV1alpha1(). + ClusterBuildStrategies(). + Create(ctx, &c.clusterBuildStrategy, meta.CreateOptions{}) + + if err != nil { + return nil, err + } + + err = wait.PollImmediate(pollCreateInterval, pollCreateTimeout, func() (done bool, err error) { + cbs, err = testBuild.BuildClientSet.ShipwrightV1alpha1().ClusterBuildStrategies().Get(ctx, c.clusterBuildStrategy.Name, meta.GetOptions{}) + if err != nil { + return false, err + } + + return true, nil + }) + + return +} + +func (c *clusterBuildStrategyPrototype) TestMe(f func(clusterBuildStrategy *buildv1alpha1.ClusterBuildStrategy)) { + cbs, err := c.Create() + Expect(err).ToNot(HaveOccurred()) + + f(cbs) + + Expect(testBuild. + BuildClientSet. + ShipwrightV1alpha1(). + ClusterBuildStrategies(). + Delete(context.Background(), cbs.Name, meta.DeleteOptions{}), + ).To(Succeed()) +} diff --git a/test/e2e/v1alpha1/common_suite_test.go b/test/e2e/v1alpha1/common_suite_test.go index 14a166fc49..58a546910c 100644 --- a/test/e2e/v1alpha1/common_suite_test.go +++ b/test/e2e/v1alpha1/common_suite_test.go @@ -109,6 +109,11 @@ func (b *buildPrototype) OutputImage(image string) *buildPrototype { return b } +func (b *buildPrototype) OutputVulnerabilitySettings(settings buildv1alpha1.VulnerabilityScanOptions) *buildPrototype { + b.build.Spec.Output.VulnerabilityScan = &settings + return b +} + func (b *buildPrototype) determineParameterIndex(name string) int { index := -1 for i, paramValue := range b.build.Spec.ParamValues { diff --git a/test/e2e/v1alpha1/e2e_vulnerability_scanning_test.go b/test/e2e/v1alpha1/e2e_vulnerability_scanning_test.go new file mode 100644 index 0000000000..f4f3e6a056 --- /dev/null +++ b/test/e2e/v1alpha1/e2e_vulnerability_scanning_test.go @@ -0,0 +1,107 @@ +// Copyright The Shipwright Contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package e2e_test + +import ( + "fmt" + "os" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" + + buildv1alpha1 "github.com/shipwright-io/build/pkg/apis/build/v1alpha1" +) + +var _ = Describe("Vulnerability Scanning", func() { + var testID string + var err error + var buildRun *buildv1alpha1.BuildRun + var cbs *clusterBuildStrategyPrototype + var insecure bool + + AfterEach(func() { + testBuild.DeleteBR(buildRun.Name) + }) + + Context("Scanning for vulnerabilities in container images", func() { + var outputImage string + BeforeEach(func() { + testID = generateTestID("vuln") + outputImage = fmt.Sprintf("%s/%s:%s", + os.Getenv(EnvVarImageRepo), + testID, + "latest", + ) + + // Assume we use secure registry unless it is a cluster + // internal registry used as part of the test setup + insecure = strings.Contains(outputImage, "cluster.local") + cbs = NewClusterBuildStrategyPrototype(). + Name("crane-pull-" + testID). + BuildStep(buildv1alpha1.BuildStep{ + Container: corev1.Container{ + Name: "crane-pull", + Image: "gcr.io/go-containerregistry/crane:latest", + WorkingDir: "$(params.shp-source-root)", + SecurityContext: &corev1.SecurityContext{ + RunAsUser: pointer.Int64(1000), + RunAsGroup: pointer.Int64(1000), + }, + Env: []corev1.EnvVar{ + {Name: "DOCKER_CONFIG", Value: "/tekton/home/.docker"}, + {Name: "HOME", Value: "/tekton/home"}, + }, + Command: []string{"crane"}, + Args: []string{ + "pull", + "--format=tarball", + "python:3.4-alpine", + "$(params.shp-output-directory)/image.tar", + }, + }, + }) + }) + + It("should find vulnerabilities in an image", func() { + cbs.TestMe(func(cbs *buildv1alpha1.ClusterBuildStrategy) { + buildRun, err = NewBuildRunPrototype(). + Namespace(testBuild.Namespace). + Name(testID). + WithBuildSpec(NewBuildPrototype(). + ClusterBuildStrategy(cbs.Name). + Namespace(testBuild.Namespace). + Name(testID). + OutputImage(outputImage). + OutputImageCredentials(os.Getenv(EnvVarImageRepoSecret)). + OutputImageInsecure(insecure). + OutputVulnerabilitySettings(buildv1alpha1.VulnerabilityScanOptions{ + Enabled: true, + FailOnFinding: true, + IgnoreOptions: &buildv1alpha1.VulnerabilityIgnoreOptions{ + Issues: []string{"CVE-2018-20843", "CVE-2019-15903"}, + Severity: "high", + Unfixed: true, + }, + }). + BuildSpec()). + Create() + Expect(err).ToNot(HaveOccurred()) + validateBuildRunToFail(testBuild, buildRun) + buildRun, err = testBuild.GetBR(buildRun.Name) + Expect(err).To(BeNil()) + Expect(buildRun).ToNot(BeNil()) + Expect(buildRun.Status).ToNot(BeNil()) + Expect(buildRun.Status.Output).ToNot(BeNil()) + Expect(buildRun.Status.Output.Vulnerabilities).ToNot(BeNil()) + Expect(len(buildRun.Status.Output.Vulnerabilities)).ToNot(BeZero()) + + }) + }) + }) +}) diff --git a/test/e2e/v1alpha1/validators_test.go b/test/e2e/v1alpha1/validators_test.go index d85acc33d7..ffee2e46e2 100644 --- a/test/e2e/v1alpha1/validators_test.go +++ b/test/e2e/v1alpha1/validators_test.go @@ -214,10 +214,13 @@ func validateBuildRunResultsFromBundleSource(testBuildRun *buildv1alpha1.BuildRu func validateBuildRunToFail(testBuild *utils.TestBuild, testBuildRun *buildv1alpha1.BuildRun) { trueCondition := corev1.ConditionTrue falseCondition := corev1.ConditionFalse + var err error // Ensure the BuildRun has been created - err := testBuild.CreateBR(testBuildRun) - Expect(err).ToNot(HaveOccurred(), "Failed to create BuildRun") + if _, err := testBuild.GetBR(testBuildRun.Name); err != nil { + Expect(testBuild.CreateBR(testBuildRun)). + ToNot(HaveOccurred(), "Failed to create BuildRun") + } // Ensure a BuildRun eventually moves to a succeeded FALSE status nextStatusLog := time.Now().Add(60 * time.Second) diff --git a/test/e2e/v1beta1/common_cbs_test.go b/test/e2e/v1beta1/common_cbs_test.go new file mode 100644 index 0000000000..b287a5748b --- /dev/null +++ b/test/e2e/v1beta1/common_cbs_test.go @@ -0,0 +1,85 @@ +// Copyright The Shipwright Contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package e2e_test + +import ( + "context" + + . "github.com/onsi/gomega" + + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + + buildv1beta1 "github.com/shipwright-io/build/pkg/apis/build/v1beta1" +) + +type clusterBuildStrategyPrototype struct { + clusterBuildStrategy buildv1beta1.ClusterBuildStrategy +} + +func NewClusterBuildStrategyPrototype() *clusterBuildStrategyPrototype { + return &clusterBuildStrategyPrototype{ + clusterBuildStrategy: buildv1beta1.ClusterBuildStrategy{}, + } +} + +func (c *clusterBuildStrategyPrototype) Name(name string) *clusterBuildStrategyPrototype { + c.clusterBuildStrategy.ObjectMeta.Name = name + return c +} + +func (c *clusterBuildStrategyPrototype) BuildStep(buildStep buildv1beta1.Step) *clusterBuildStrategyPrototype { + c.clusterBuildStrategy.Spec.Steps = append(c.clusterBuildStrategy.Spec.Steps, buildStep) + return c +} + +func (c *clusterBuildStrategyPrototype) Parameter(parameter buildv1beta1.Parameter) *clusterBuildStrategyPrototype { + c.clusterBuildStrategy.Spec.Parameters = append(c.clusterBuildStrategy.Spec.Parameters, parameter) + return c +} + +func (c *clusterBuildStrategyPrototype) Volume(volume buildv1beta1.BuildStrategyVolume) *clusterBuildStrategyPrototype { + c.clusterBuildStrategy.Spec.Volumes = append(c.clusterBuildStrategy.Spec.Volumes, volume) + return c +} + +func (c *clusterBuildStrategyPrototype) Create() (cbs *buildv1beta1.ClusterBuildStrategy, err error) { + ctx := context.Background() + + _, err = testBuild. + BuildClientSet. + ShipwrightV1beta1(). + ClusterBuildStrategies(). + Create(ctx, &c.clusterBuildStrategy, meta.CreateOptions{}) + + if err != nil { + return nil, err + } + + err = wait.PollImmediate(pollCreateInterval, pollCreateTimeout, func() (done bool, err error) { + cbs, err = testBuild.BuildClientSet.ShipwrightV1beta1().ClusterBuildStrategies().Get(ctx, c.clusterBuildStrategy.Name, meta.GetOptions{}) + if err != nil { + return false, err + } + + return true, nil + }) + + return +} + +func (c *clusterBuildStrategyPrototype) TestMe(f func(clusterBuildStrategy *buildv1beta1.ClusterBuildStrategy)) { + cbs, err := c.Create() + Expect(err).ToNot(HaveOccurred()) + + f(cbs) + + Expect(testBuild. + BuildClientSet. + ShipwrightV1beta1(). + ClusterBuildStrategies(). + Delete(context.Background(), cbs.Name, meta.DeleteOptions{}), + ).To(Succeed()) +} diff --git a/test/e2e/v1beta1/common_suite_test.go b/test/e2e/v1beta1/common_suite_test.go index ac71a8c8b9..b3038e13b5 100644 --- a/test/e2e/v1beta1/common_suite_test.go +++ b/test/e2e/v1beta1/common_suite_test.go @@ -124,6 +124,11 @@ func (b *buildPrototype) OutputImage(image string) *buildPrototype { return b } +func (b *buildPrototype) OutputVulnerabilitySettings(settings buildv1beta1.VulnerabilityScanOptions) *buildPrototype { + b.build.Spec.Output.VulnerabilityScan = &settings + return b +} + func (b *buildPrototype) determineParameterIndex(name string) int { index := -1 for i, paramValue := range b.build.Spec.ParamValues { diff --git a/test/e2e/v1beta1/e2e_vulnerability_scanning_test.go b/test/e2e/v1beta1/e2e_vulnerability_scanning_test.go new file mode 100644 index 0000000000..e6127eea63 --- /dev/null +++ b/test/e2e/v1beta1/e2e_vulnerability_scanning_test.go @@ -0,0 +1,105 @@ +// Copyright The Shipwright Contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package e2e_test + +import ( + "fmt" + "os" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" + + buildv1beta1 "github.com/shipwright-io/build/pkg/apis/build/v1beta1" +) + +var _ = Describe("Vulnerability Scanning", func() { + var testID string + var err error + var buildRun *buildv1beta1.BuildRun + var cbs *clusterBuildStrategyPrototype + var insecure bool + + AfterEach(func() { + testBuild.DeleteBR(buildRun.Name) + }) + + Context("Scanning for vulnerabilities in container images", func() { + var outputImage string + BeforeEach(func() { + testID = generateTestID("vuln") + outputImage = fmt.Sprintf("%s/%s:%s", + os.Getenv(EnvVarImageRepo), + testID, + "latest", + ) + + // Assume we use secure registry unless it is a cluster + // internal registry used as part of the test setup + insecure = strings.Contains(outputImage, "cluster.local") + cbs = NewClusterBuildStrategyPrototype(). + Name("crane-pull-" + testID). + BuildStep(buildv1beta1.Step{ + Name: "crane-pull", + Image: "gcr.io/go-containerregistry/crane:latest", + WorkingDir: "$(params.shp-source-root)", + SecurityContext: &corev1.SecurityContext{ + RunAsUser: pointer.Int64(1000), + RunAsGroup: pointer.Int64(1000), + }, + Env: []corev1.EnvVar{ + {Name: "DOCKER_CONFIG", Value: "/tekton/home/.docker"}, + {Name: "HOME", Value: "/tekton/home"}, + }, + Command: []string{"crane"}, + Args: []string{ + "pull", + "--format=tarball", + "python:3.4-alpine", + "$(params.shp-output-directory)/image.tar", + }, + }) + }) + + It("should find vulnerabilities in an image", func() { + cbs.TestMe(func(cbs *buildv1beta1.ClusterBuildStrategy) { + buildRun, err = NewBuildRunPrototype(). + Namespace(testBuild.Namespace). + Name(testID). + WithBuildSpec(NewBuildPrototype(). + ClusterBuildStrategy(cbs.Name). + Namespace(testBuild.Namespace). + Name(testID). + OutputImage(outputImage). + OutputImageCredentials(os.Getenv(EnvVarImageRepoSecret)). + OutputImageInsecure(insecure). + OutputVulnerabilitySettings(buildv1beta1.VulnerabilityScanOptions{ + Enabled: true, + FailOnFinding: true, + IgnoreOptions: &buildv1beta1.VulnerabilityIgnoreOptions{ + Issues: []string{"CVE-2018-20843", "CVE-2019-15903"}, + Severity: "high", + Unfixed: true, + }, + }). + BuildSpec()). + Create() + Expect(err).ToNot(HaveOccurred()) + validateBuildRunToFail(testBuild, buildRun) + buildRun, err = testBuild.GetBR(buildRun.Name) + Expect(err).To(BeNil()) + Expect(buildRun).ToNot(BeNil()) + Expect(buildRun.Status).ToNot(BeNil()) + Expect(buildRun.Status.Output).ToNot(BeNil()) + Expect(buildRun.Status.Output.Vulnerabilities).ToNot(BeNil()) + Expect(len(buildRun.Status.Output.Vulnerabilities)).ToNot(BeZero()) + + }) + }) + }) +}) diff --git a/test/e2e/v1beta1/validators_test.go b/test/e2e/v1beta1/validators_test.go index 135df35096..933af80675 100644 --- a/test/e2e/v1beta1/validators_test.go +++ b/test/e2e/v1beta1/validators_test.go @@ -213,10 +213,13 @@ func validateBuildRunResultsFromBundleSource(testBuildRun *buildv1beta1.BuildRun func validateBuildRunToFail(testBuild *utils.TestBuild, testBuildRun *buildv1beta1.BuildRun) { trueCondition := corev1.ConditionTrue falseCondition := corev1.ConditionFalse + var err error // Ensure the BuildRun has been created - err := testBuild.CreateBR(testBuildRun) - Expect(err).ToNot(HaveOccurred(), "Failed to create BuildRun") + if _, err := testBuild.GetBR(testBuildRun.Name); err != nil { + Expect(testBuild.CreateBR(testBuildRun)). + ToNot(HaveOccurred(), "Failed to create BuildRun") + } // Ensure a BuildRun eventually moves to a succeeded FALSE status nextStatusLog := time.Now().Add(60 * time.Second)