Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

build, run: record hash or digest in image history for sources used in --mount #5691

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

build, run: record hash or digest in image history for sources used in --mount #5691

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 90 additions & 22 deletions imagebuildah/stage_executor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"github.com/containers/buildah/define"
buildahdocker "github.com/containers/buildah/docker"
"github.com/containers/buildah/internal"
internalParse "github.com/containers/buildah/internal/parse"
"github.com/containers/buildah/internal/tmpdir"
internalUtil "github.com/containers/buildah/internal/util"
"github.com/containers/buildah/pkg/parse"
Expand Down Expand Up @@ -1281,7 +1282,11 @@ func (s *StageExecutor) Execute(ctx context.Context, base string) (imgID string,
// No base image means there's nothing to put in a
// layer, so don't create one.
emptyLayer := (s.builder.FromImageID == "")
if imgID, ref, err = s.commit(ctx, s.getCreatedBy(nil, ""), emptyLayer, s.output, s.executor.squash || s.executor.confidentialWorkload.Convert, lastStage); err != nil {
createdBy, err := s.getCreatedBy(nil, "")
if err != nil {
return "", nil, false, err
}
if imgID, ref, err = s.commit(ctx, createdBy, emptyLayer, s.output, s.executor.squash || s.executor.confidentialWorkload.Convert, lastStage); err != nil {
return "", nil, false, fmt.Errorf("committing base container: %w", err)
}
} else {
Expand Down Expand Up @@ -1428,7 +1433,11 @@ func (s *StageExecutor) Execute(ctx context.Context, base string) (imgID string,
if s.executor.timestamp != nil {
timestamp = *s.executor.timestamp
}
s.builder.AddPrependedEmptyLayer(&timestamp, s.getCreatedBy(node, addedContentSummary), "", "")
createdBy, err := s.getCreatedBy(node, addedContentSummary)
if err != nil {
return "", nil, false, err
}
s.builder.AddPrependedEmptyLayer(&timestamp, createdBy, "", "")
continue
}
// This is the last instruction for this stage,
Expand All @@ -1438,7 +1447,11 @@ func (s *StageExecutor) Execute(ctx context.Context, base string) (imgID string,
// stage.
if lastStage || imageIsUsedLater {
logCommit(s.output, i)
imgID, ref, err = s.commit(ctx, s.getCreatedBy(node, addedContentSummary), false, s.output, s.executor.squash, lastStage && lastInstruction)
createdBy, err := s.getCreatedBy(node, addedContentSummary)
if err != nil {
return "", nil, false, err
}
imgID, ref, err = s.commit(ctx, createdBy, false, s.output, s.executor.squash, lastStage && lastInstruction)
if err != nil {
return "", nil, false, fmt.Errorf("committing container for step %+v: %w", *step, err)
}
Expand Down Expand Up @@ -1659,14 +1672,18 @@ func (s *StageExecutor) Execute(ctx context.Context, base string) (imgID string,
// We're not going to find any more cache hits, so we
// can stop looking for them.
checkForLayers = false
createdBy, err := s.getCreatedBy(node, addedContentSummary)
if err != nil {
return "", nil, false, err
}
// Create a new image, maybe with a new layer, with the
// name for this stage if it's the last instruction.
logCommit(s.output, i)
// While committing we always set squash to false here
// because at this point we want to save history for
// layers even if its a squashed build so that they
// can be part of the build cache.
imgID, ref, err = s.commit(ctx, s.getCreatedBy(node, addedContentSummary), !s.stepRequiresLayer(step), commitName, false, lastStage && lastInstruction)
imgID, ref, err = s.commit(ctx, createdBy, !s.stepRequiresLayer(step), commitName, false, lastStage && lastInstruction)
if err != nil {
return "", nil, false, fmt.Errorf("committing container for step %+v: %w", *step, err)
}
Expand Down Expand Up @@ -1697,12 +1714,16 @@ func (s *StageExecutor) Execute(ctx context.Context, base string) (imgID string,

if lastInstruction && lastStage {
if s.executor.squash || s.executor.confidentialWorkload.Convert || len(s.executor.sbomScanOptions) != 0 {
createdBy, err := s.getCreatedBy(node, addedContentSummary)
if err != nil {
return "", nil, false, err
}
// If this is the last instruction of the last stage,
// create a squashed or confidential workload
// version of the image if that's what we're after,
// or a normal one if we need to scan the image while
// committing it.
imgID, ref, err = s.commit(ctx, s.getCreatedBy(node, addedContentSummary), !s.stepRequiresLayer(step), commitName, s.executor.squash || s.executor.confidentialWorkload.Convert, lastStage && lastInstruction)
imgID, ref, err = s.commit(ctx, createdBy, !s.stepRequiresLayer(step), commitName, s.executor.squash || s.executor.confidentialWorkload.Convert, lastStage && lastInstruction)
if err != nil {
return "", nil, false, fmt.Errorf("committing final squash step %+v: %w", *step, err)
}
Expand Down Expand Up @@ -1794,54 +1815,58 @@ func historyEntriesEqual(base, derived v1.History) bool {
// that we're comparing.
// Used to verify whether a cache of the intermediate image exists and whether
// to run the build again.
func (s *StageExecutor) historyAndDiffIDsMatch(baseHistory []v1.History, baseDiffIDs []digest.Digest, child *parser.Node, history []v1.History, diffIDs []digest.Digest, addedContentSummary string, buildAddsLayer bool) bool {
func (s *StageExecutor) historyAndDiffIDsMatch(baseHistory []v1.History, baseDiffIDs []digest.Digest, child *parser.Node, history []v1.History, diffIDs []digest.Digest, addedContentSummary string, buildAddsLayer bool) (bool, error) {
// our history should be as long as the base's, plus one entry for what
// we're doing
if len(history) != len(baseHistory)+1 {
return false
return false, nil
}
// check that each entry in the base history corresponds to an entry in
// our history, and count how many of them add a layer diff
expectedDiffIDs := 0
for i := range baseHistory {
if !historyEntriesEqual(baseHistory[i], history[i]) {
return false
return false, nil
}
if !baseHistory[i].EmptyLayer {
expectedDiffIDs++
}
}
if len(baseDiffIDs) != expectedDiffIDs {
return false
return false, nil
}
if buildAddsLayer {
// we're adding a layer, so we should have exactly one more
// layer than the base image
if len(diffIDs) != expectedDiffIDs+1 {
return false
return false, nil
}
} else {
// we're not adding a layer, so we should have exactly the same
// layers as the base image
if len(diffIDs) != expectedDiffIDs {
return false
return false, nil
}
}
// compare the diffs for the layers that we should have in common
for i := range baseDiffIDs {
if diffIDs[i] != baseDiffIDs[i] {
return false
return false, nil
}
}
return history[len(baseHistory)].CreatedBy == s.getCreatedBy(child, addedContentSummary)
createdBy, err := s.getCreatedBy(child, addedContentSummary)
if err != nil {
return false, err
}
return history[len(baseHistory)].CreatedBy == createdBy, nil
}

// getCreatedBy returns the command the image at node will be created by. If
// the passed-in CompositeDigester is not nil, it is assumed to have the digest
// information for the content if the node is ADD or COPY.
func (s *StageExecutor) getCreatedBy(node *parser.Node, addedContentSummary string) string {
func (s *StageExecutor) getCreatedBy(node *parser.Node, addedContentSummary string) (string, error) {
if node == nil {
return "/bin/sh"
return "/bin/sh", nil
}
switch strings.ToUpper(node.Value) {
case "ARG":
Expand All @@ -1851,15 +1876,51 @@ func (s *StageExecutor) getCreatedBy(node *parser.Node, addedContentSummary stri
}
}
buildArgs := s.getBuildArgsKey()
return "/bin/sh -c #(nop) ARG " + buildArgs
return "/bin/sh -c #(nop) ARG " + buildArgs, nil
case "RUN":
shArg := ""
buildArgs := s.getBuildArgsResolvedForRun()
mountOptionSource := ""
mountOptionFrom := ""
appendCheckSum := ""
var err error
for _, flag := range node.Flags {
if strings.HasPrefix(flag, "--mount") {
mountOptionSource, mountOptionFrom = internalParse.GetFromAndSourceKeysFromMountFlag(flag)
}
}
// Source specificed is part of stage, image or additional-build-context.
if mountOptionFrom != "" {
// If this is not a stage then get digest of image or additional build context
if _, ok := s.executor.stages[mountOptionFrom]; !ok {
if builder, ok := s.executor.containerMap[mountOptionFrom]; ok {
// Found valid image, get image digest.
appendCheckSum = builder.FromImageDigest
} else {
// Found additional build context, get directory sha.
appendCheckSum, err = internalUtil.GeneratePathChecksum(filepath.Join(s.executor.contextDir, mountOptionSource))
if err != nil {
return "", fmt.Errorf("Unable to generate checksum for image history: %w", err)
}
}
}
} else {
if mountOptionSource != "" {
appendCheckSum, err = internalUtil.GeneratePathChecksum(filepath.Join(s.executor.contextDir, mountOptionSource))
if err != nil {
return "", fmt.Errorf("Unable to generate checksum for image history: %w", err)
}
}
}
if len(node.Original) > 4 {
shArg = node.Original[4:]
}
if appendCheckSum != "" {
// add a seperator to appendCheckSum
appendCheckSum = ":" + appendCheckSum
}
if buildArgs != "" {
return "|" + strconv.Itoa(len(strings.Split(buildArgs, " "))) + " " + buildArgs + " /bin/sh -c " + shArg
return "|" + strconv.Itoa(len(strings.Split(buildArgs, " "))) + " " + buildArgs + " /bin/sh -c " + shArg + appendCheckSum, nil
}
result := "/bin/sh -c " + shArg
if len(node.Heredocs) > 0 {
Expand All @@ -1868,15 +1929,15 @@ func (s *StageExecutor) getCreatedBy(node *parser.Node, addedContentSummary stri
result = result + "\n" + heredocContent
}
}
return result
return result + appendCheckSum, nil
case "ADD", "COPY":
destination := node
for destination.Next != nil {
destination = destination.Next
}
return "/bin/sh -c #(nop) " + strings.ToUpper(node.Value) + " " + addedContentSummary + " in " + destination.Value + " "
return "/bin/sh -c #(nop) " + strings.ToUpper(node.Value) + " " + addedContentSummary + " in " + destination.Value + " ", nil
default:
return "/bin/sh -c #(nop) " + node.Original
return "/bin/sh -c #(nop) " + node.Original, nil
}
}

Expand Down Expand Up @@ -2025,7 +2086,10 @@ func (s *StageExecutor) generateCacheKey(ctx context.Context, currNode *parser.N
fmt.Fprintln(hash, diffIDs[i].String())
}
}
createdBy := s.getCreatedBy(currNode, addedContentDigest)
createdBy, err := s.getCreatedBy(currNode, addedContentDigest)
if err != nil {
return "", err
}
fmt.Fprintf(hash, "%t", buildAddsLayer)
fmt.Fprintln(hash, createdBy)
fmt.Fprintln(hash, manifestType)
Expand Down Expand Up @@ -2205,7 +2269,11 @@ func (s *StageExecutor) intermediateImageExists(ctx context.Context, currNode *p
continue
}
// children + currNode is the point of the Dockerfile we are currently at.
if s.historyAndDiffIDsMatch(baseHistory, baseDiffIDs, currNode, history, diffIDs, addedContentDigest, buildAddsLayer) {
diff, err := s.historyAndDiffIDsMatch(baseHistory, baseDiffIDs, currNode, history, diffIDs, addedContentDigest, buildAddsLayer)
if err != nil {
return "", err
}
if diff {
return image.ID, nil
}
}
Expand Down
18 changes: 18 additions & 0 deletions internal/parse/parse.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,24 @@ import (
specs "github.com/opencontainers/runtime-spec/specs-go"
)

// Consumes mount flag in format of `--mount=type=bind,src=/path,from=image` and
// return values of `src` and `from`, values are empty if keys are not present in the option.
func GetFromAndSourceKeysFromMountFlag(mount string) (string, string) {
tokens := strings.Split(mount, ",")
source := ""
from := ""
for _, option := range tokens {
optionSplit := strings.Split(option, "=")
if optionSplit[0] == "src" || optionSplit[0] == "source" {
source = optionSplit[1]
}
if optionSplit[0] == "from" {
from = optionSplit[1]
}
}
return source, from
}

// ValidateVolumeMountHostDir validates the host path of buildah --volume
func ValidateVolumeMountHostDir(hostDir string) error {
if !filepath.IsAbs(hostDir) {
Expand Down
63 changes: 63 additions & 0 deletions internal/util/util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package util

import (
"crypto/sha256"
"fmt"
"io"
"os"
Expand All @@ -17,6 +18,68 @@ import (
v1 "github.com/opencontainers/image-spec/specs-go/v1"
)

// computeFileHash computes the SHA-256 checksum of a single file.
func computeFileHash(filePath string) ([]byte, error) {
file, err := os.Open(filePath)
if err != nil {
return nil, err
}
defer file.Close()

hash := sha256.New()
if _, err := io.Copy(hash, file); err != nil {
return nil, err
}

return hash.Sum(nil), nil
}

// computeDirectoryHash computes the SHA-256 checksum of all files in a directory.
func computeDirectoryHash(dirPath string) ([]byte, error) {
hash := sha256.New()

err := filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}

fileHash, err := computeFileHash(path)
if err != nil {
return err
}

_, err = hash.Write(fileHash)
if err != nil {
return err
}

return nil
})

if err != nil {
return nil, err
}

return hash.Sum(nil), nil
}

// GeneratePathChecksum generates the SHA-256 checksum for a file or a directory.
func GeneratePathChecksum(path string) (string, error) {
info, err := os.Stat(path)
if err != nil {
return "", err
}
if info.IsDir() {
checksum, err := computeDirectoryHash(path)
return fmt.Sprintf("%x", checksum), err
}
checksum, err := computeFileHash(path)
return fmt.Sprintf("%x", checksum), err
}

// LookupImage returns *Image to corresponding imagename or id
func LookupImage(ctx *types.SystemContext, store storage.Store, image string) (*libimage.Image, error) {
systemContext := ctx
Expand Down
Loading