feat(STONEBLD-1832): add tests to check parent sources are included

Signed-off-by: Chenxiong Qi <[email protected]>

go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/h2non/gock v1.2.0
 	github.com/magefile/mage v1.13.0
 	github.com/mitchellh/go-homedir v1.1.0
+	github.com/moby/buildkit v0.12.5
 	github.com/onsi/ginkgo/v2 v2.15.0
 	github.com/onsi/gomega v1.31.1
 	github.com/openshift-pipelines/pipelines-as-code v0.18.0
@@ -238,7 +239,6 @@ require (
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
-	github.com/moby/buildkit v0.12.5 // indirect
 	github.com/moby/locker v1.0.1 // indirect
 	github.com/moby/patternmatcher v0.5.0 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect

pkg/clients/tekton/taskruns.go

@@ -182,3 +182,17 @@ func (t *TektonController) GetTaskRunStatus(c crclient.Client, pr *pipeline.Pipe
 func (t *TektonController) DeleteAllTaskRunsInASpecificNamespace(namespace string) error {
 	return t.KubeRest().DeleteAllOf(context.Background(), &pipeline.TaskRun{}, crclient.InNamespace(namespace))
 }
+
+// GetTaskRunParam gets value of a TaskRun param.
+func (t *TektonController) GetTaskRunParam(c crclient.Client, pr *pipeline.PipelineRun, pipelineTaskName, paramName string) (string, error) {
+	taskRun, err := t.GetTaskRunFromPipelineRun(c, pr, pipelineTaskName)
+	if err != nil {
+		return "", err
+	}
+	for _, param := range taskRun.Spec.Params {
+		if param.Name == paramName {
+			return strings.TrimSpace(param.Value.StringVal), nil
+		}
+	}
+	return "", fmt.Errorf("cannot find param %s from TaskRun %s", pipelineTaskName, paramName)
+}

pkg/utils/build/image.go

@@ -6,6 +6,10 @@ import (
 	"os"
 	"time"
 
+	"github.com/google/go-containerregistry/pkg/authn"
+	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
 	. "github.com/onsi/ginkgo/v2"
 
 	"github.com/openshift/library-go/pkg/image/reference"
@@ -65,3 +69,40 @@ func ImageFromPipelineRun(pipelineRun *pipeline.PipelineRun) (*imageInfo.Image,
 	}
 	return image, nil
 }
+
+// FetchImageConfig fetches image config from remote registry.
+// It uses the registry authentication credentials stored in default place ~/.docker/config.json
+func FetchImageConfig(imagePullspec string) (*v1.ConfigFile, error) {
+	ref, err := name.ParseReference(imagePullspec)
+	if err != nil {
+		return nil, err
+	}
+	// Fetch the manifest using default credentials.
+	descriptor, err := remote.Get(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain))
+	if err != nil {
+		return nil, err
+	}
+
+	image, err := descriptor.Image()
+	if err != nil {
+		return nil, err
+	}
+	configFile, err := image.ConfigFile()
+	if err != nil {
+		return nil, err
+	}
+	return configFile, nil
+}
+
+func FetchImageDigest(imagePullspec string) (string, error) {
+	ref, err := name.ParseReference(imagePullspec)
+	if err != nil {
+		return "", err
+	}
+	// Fetch the manifest using default credentials.
+	descriptor, err := remote.Get(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain))
+	if err != nil {
+		return "", err
+	}
+	return descriptor.Digest.Hex, nil
+}

pkg/utils/build/source_image.go

@@ -1,11 +1,22 @@
 package build
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
+	"io"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
 
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/moby/buildkit/frontend/dockerfile/parser"
+	"github.com/openshift/library-go/pkg/image/reference"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/redhat-appstudio/e2e-tests/pkg/clients/tekton"
 	"github.com/redhat-appstudio/e2e-tests/pkg/utils"
 	pipeline "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1"
 )
@@ -165,3 +176,266 @@ func IsPreFetchDependencysFilesExists(absExtraSourceDirPath string, isHermetic b
 	}
 	return true, nil
 }
+
+// readDockerfile reads Dockerfile dockerfile from repository repoURL.
+// The Dockerfile is resolved by following the logic applied to the buildah task definition.
+func readDockerfile(pathContext, dockerfile, repoURL, repoRevision string) ([]byte, error) {
+	tempRepoDir, err := os.MkdirTemp("", "-test-repo")
+	if err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(tempRepoDir)
+	testRepo, err := git.PlainClone(tempRepoDir, false, &git.CloneOptions{URL: repoURL})
+	if err != nil {
+		return nil, err
+	}
+
+	// checkout to the revision. use go-git ResolveRevision since revision could be a branch, tag or commit hash
+	commitHash, err := testRepo.ResolveRevision(plumbing.Revision(repoRevision))
+	if err != nil {
+		return nil, err
+	}
+	workTree, err := testRepo.Worktree()
+	if err != nil {
+		return nil, err
+	}
+	if err := workTree.Checkout(&git.CheckoutOptions{Hash: *commitHash}); err != nil {
+		return nil, err
+	}
+
+	// check dockerfile in different paths
+	var dockerfilePath string
+	dockerfilePath = filepath.Join(tempRepoDir, dockerfile)
+	if content, err := os.ReadFile(dockerfilePath); err == nil {
+		return content, nil
+	}
+	dockerfilePath = filepath.Join(tempRepoDir, pathContext, dockerfile)
+	if content, err := os.ReadFile(dockerfilePath); err == nil {
+		return content, nil
+	}
+	if strings.HasPrefix(dockerfile, "https://") {
+		if resp, err := http.Get(dockerfile); err == nil {
+			defer resp.Body.Close()
+			if body, err := io.ReadAll(resp.Body); err == nil {
+				return body, err
+			} else {
+				return nil, err
+			}
+		} else {
+			return nil, err
+		}
+	}
+	return nil, fmt.Errorf(
+		fmt.Sprintf("resolveDockerfile: can't resolve Dockerfile from path context %s and dockerfile %s",
+			pathContext, dockerfile),
+	)
+}
+
+// ReadDockerfileUsedForBuild reads the Dockerfile and return its content.
+func ReadDockerfileUsedForBuild(c client.Client, tektonController *tekton.TektonController, pr *pipeline.PipelineRun) ([]byte, error) {
+	var paramDockerfileValue, paramPathContextValue string
+	var paramUrlValue, paramRevisionValue string
+	var err error
+	getParam := tektonController.GetTaskRunParam
+
+	if paramDockerfileValue, err = getParam(c, pr, "build-container", "dockerfile"); err != nil {
+		return nil, err
+	}
+
+	if paramPathContextValue, err = getParam(c, pr, "build-container", "path-context"); err != nil {
+		return nil, err
+	}
+
+	// get git-clone param url and revision
+	if paramUrlValue, err = getParam(c, pr, "clone-repository", "url"); err != nil {
+		return nil, err
+	}
+
+	if paramRevisionValue, err = getParam(c, pr, "clone-repository", "revision"); err != nil {
+		return nil, err
+	}
+
+	dockerfileContent, err := readDockerfile(paramPathContextValue, paramDockerfileValue, paramUrlValue, paramRevisionValue)
+	if err != nil {
+		return nil, err
+	}
+	return dockerfileContent, nil
+}
+
+type SourceBuildResult struct {
+	Status                  string `json:"status"`
+	Message                 string `json:"message,omitempty"`
+	DependenciesIncluded    bool   `json:"dependencies_included"`
+	BaseImageSourceIncluded bool   `json:"base_image_source_included"`
+	ImageUrl                string `json:"image_url"`
+	ImageDigest             string `json:"image_digest"`
+}
+
+// ReadSourceBuildResult reads source-build task result BUILD_RESULT and returns the decoded data.
+func ReadSourceBuildResult(c client.Client, tektonController *tekton.TektonController, pr *pipeline.PipelineRun) (*SourceBuildResult, error) {
+	sourceBuildResult, err := tektonController.GetTaskRunResult(c, pr, "build-source-image", "BUILD_RESULT")
+	if err != nil {
+		return nil, err
+	}
+	var buildResult SourceBuildResult
+	if err = json.Unmarshal([]byte(sourceBuildResult), &buildResult); err != nil {
+		return nil, err
+	}
+	return &buildResult, nil
+}
+
+type Dockerfile struct {
+	parsedContent *parser.Result
+}
+
+func ParseDockerfile(content []byte) (*Dockerfile, error) {
+	parsedContent, err := parser.Parse(bytes.NewReader(content))
+	if err != nil {
+		return nil, err
+	}
+	df := Dockerfile{
+		parsedContent: parsedContent,
+	}
+	return &df, nil
+}
+
+func (d *Dockerfile) ParentImages() []string {
+	parentImages := make([]string, 5)
+	for _, child := range d.parsedContent.AST.Children {
+		if child.Value == "FROM" {
+			parentImages = append(parentImages, child.Next.Value)
+		}
+	}
+	return parentImages
+}
+
+func (d *Dockerfile) IsBuildFromScratch() bool {
+	parentImages := d.ParentImages()
+	return parentImages[len(parentImages)-1] == "scratch"
+}
+
+// convertImageToBuildahOutputForm converts an image pullspec to the corresponding form within
+// BASE_IMAGES_DIGESTS output by buildah task..
+func convertImageToBuildahOutputForm(imagePullspec string) (string, error) {
+	ref, err := reference.Parse(imagePullspec)
+	if err != nil {
+		return "", err
+	}
+	tag := ref.Tag
+	if tag == "" {
+		tag = "<none>"
+	}
+	digest := ref.ID
+	if digest == "" {
+		val, err := FetchImageDigest(imagePullspec)
+		if err != nil {
+			return "", err
+		}
+		digest = val
+	}
+	return fmt.Sprintf("%s/%s/%s:%s@%s", ref.Registry, ref.Namespace, ref.Name, tag, digest), nil
+}
+
+// ConvertParentImagesToBaseImagesDigestsForm is a helper function for testing the order is matched
+// between BASE_IMAGES_DIGESTS and parent images within Dockerfile.
+// ConvertParentImagesToBaseImagesDigestsForm de-duplicates the images what buildah task does for BASE_IMAGES_DIGESTS.
+func (d *Dockerfile) ConvertParentImagesToBaseImagesDigestsForm() ([]string, error) {
+	convertedImagePullspecs := make([]string, 5)
+	seen := make(map[string]int)
+	parentImages := d.ParentImages()
+	for _, imagePullspec := range parentImages {
+		if imagePullspec == "scratch" {
+			continue
+		}
+		if _, exists := seen[imagePullspec]; exists {
+			continue
+		}
+		seen[imagePullspec] = 1
+		if converted, err := convertImageToBuildahOutputForm(imagePullspec); err == nil {
+			convertedImagePullspecs = append(convertedImagePullspecs, converted)
+		} else {
+			return nil, err
+		}
+	}
+	return convertedImagePullspecs, nil
+}
+
+func isRegistryAllowed(registry string) bool {
+	// For the list of allowed registries, refer to source-build task definition.
+	allowedRegistries := map[string]int{
+		"registry.access.redhat.com": 1,
+		"registry.redhat.io": 1,
+	}
+	_, exists := allowedRegistries[registry]
+	return exists
+}
+
+func IsImagePulledFromAllowedRegistry(imagePullspec string) (bool, error) {
+	if ref, err := reference.Parse(imagePullspec); err == nil {
+		return isRegistryAllowed(ref.Registry), nil
+	} else {
+		return false, err
+	}
+}
+
+func SourceBuildTaskRunLogsContain(
+	tektonController *tekton.TektonController, pr *pipeline.PipelineRun, message string) (bool, error) {
+	logs, err := tektonController.GetTaskRunLogs(pr.GetName(), "source-build", pr.GetNamespace())
+	if err != nil {
+		return false, err
+	}
+	for _, logMessage := range logs {
+		if strings.Contains(logMessage, message) {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func ResolveSourceImage(image string) (string, error) {
+	// Parse in order to dropping the tag for fetching config
+	ref, err := reference.Parse(image)
+	if err != nil {
+		return "", err
+	}
+	config, err := FetchImageConfig(ref.Exact())
+	if err != nil {
+		return "", err
+	}
+	labels := config.Config.Labels
+	var version, release string
+	var exists bool
+	if version, exists = labels["version"]; !exists {
+		return "", fmt.Errorf("cannot find out version label from image config")
+	}
+	if release, exists = labels["release"]; !exists {
+		return "", fmt.Errorf("cannot find out release label from image config")
+	}
+	return fmt.Sprintf("%s-%s-source", version, release), nil
+}
+
+func AllParentSourcesIncluded(parentSourceImage, builtSourceImage string) (bool, error) {
+	parentConfig, err := FetchImageConfig(parentSourceImage)
+	if err != nil {
+		return false, err
+	}
+	builtConfig, err := FetchImageConfig(builtSourceImage)
+	if err != nil {
+		return false, err
+	}
+	srpmSha256Sums := make(map[string]int)
+	var parts []string
+	for _, history := range builtConfig.History {
+		// Example history: #(nop) bsi version 0.2.0-dev adding artifact: 5f526f4
+		parts = strings.Split(history.CreatedBy, " ")
+		// The last part 5f526f4 is the checksum calculated from the file included in the generated blob.
+		srpmSha256Sums[parts[len(parts)-1]] = 1
+	}
+	for _, history := range parentConfig.History {
+		parts = strings.Split(history.CreatedBy, " ")
+		if _, exists := srpmSha256Sums[parts[len(parts)-1]]; !exists {
+			return false, nil
+		}
+	}
+	return true, nil
+}

tests/build/build_templates.go

@@ -278,9 +278,7 @@ var _ = framework.BuildSuiteDescribe("Build templates E2E test", Label("build",
 					Name:      binaryImageRef.Name,
 					Tag:       fmt.Sprintf("%s.src", strings.Replace(tagInfo.ManifestDigest, ":", "-", 1)),
 				}
-
 				srcImage := srcImageRef.String()
-
 				tagExists, err := build.DoesTagExistsInQuay(srcImage)
 				Expect(err).ShouldNot(HaveOccurred(),
 					fmt.Sprintf("failed to check existence of source container image %s", srcImage))
@@ -298,6 +296,8 @@ var _ = framework.BuildSuiteDescribe("Build templates E2E test", Label("build",
 				Expect(err).ShouldNot(HaveOccurred())
 				Expect(filesExists).To(BeTrue())
 
+				c := kubeadminClient.CommonController.KubeRest()
+				CheckParentSources(c, kubeadminClient.TektonController, pr)
 			})
 
 			When(fmt.Sprintf("Pipeline Results are stored for component with Git source URL %s", gitUrl), Label("pipeline"), func() {