cache Layer and ConfigFile

Signed-off-by: Naoki MATSUMOTO <[email protected]>
naoki9911 · Apr 14, 2024 · 06c4c9a · 06c4c9a
1 parent 279ac7a
commit 06c4c9a
Show file tree

Hide file tree

Showing 4 changed files with 136 additions and 18 deletions.
diff --git a/cmd/ctr-cli/convert/convert.go b/cmd/ctr-cli/convert/convert.go
@@ -54,7 +54,7 @@ var Flags = []cli.Flag{
 	},
 }
 
-var workDir = filepath.Join(os.TempDir(), "ctr-cli")
+var workDir = filepath.Join(os.TempDir(), "ctr-cli", "convert")
 
 type dockerImageManifest struct {
 	Config   string   `json:"Config"`
@@ -216,7 +216,7 @@ func Action(c *cli.Context) error {
 	}
 
 	// convert layer.tar to dimg
-	tempDir, err := os.MkdirTemp("/tmp/ctr-cli", "*")
+	tempDir, err := os.MkdirTemp("/tmp/ctr-cli/convert", "*")
 	if err != nil {
 		return err
 	}
@@ -237,12 +237,6 @@ func Action(c *cli.Context) error {
 		}
 	}
 
-	tempDiffDir, err := os.MkdirTemp("/tmp/ctr-cli", "*")
-	if err != nil {
-		return err
-	}
-	defer os.RemoveAll(tempDiffDir)
-
 	logger.Info("packing dimg")
 	dimgPath := filepath.Join(outputPath, "image.dimg")
 	err = di3fsImage.PackDir(tempDir, dimgPath, threadNum)

diff --git a/cmd/ctr-cli/convert2/convert.go b/cmd/ctr-cli/convert2/convert.go
@@ -80,7 +80,10 @@ func action(c *cli.Context) error {
 	img := c.String("image")
 	OS := c.String("os")
 	arch := c.String("arch")
-	puller := oci.NewPuller()
+	puller, err := oci.NewPuller()
+	if err != nil {
+		return fmt.Errorf("failed to create puller: %v", err)
+	}
 	logger.WithFields(logrus.Fields{"image": img, "os": OS, "arch": arch}).Info("started to pull image")
 	layer, config, err := puller.Pull(img, OS, arch)
 	if err != nil {

diff --git a/pkg/oci/cache.go b/pkg/oci/cache.go
@@ -0,0 +1,68 @@
+package oci
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+)
+
+var defaultCachePath = filepath.Join(os.TempDir(), "ctr-cli", "cache")
+
+type BlobCache struct {
+	cachePath string
+}
+
+func NewBlobCache() (*BlobCache, error) {
+	c := &BlobCache{
+		cachePath: defaultCachePath,
+	}
+	err := os.MkdirAll(c.cachePath, 0755)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create cache dir %s: %v", c.cachePath, err)
+	}
+
+	return c, nil
+}
+
+func (b *BlobCache) Store(d *v1.Hash, r io.Reader) error {
+	name := filepath.Join(b.cachePath, d.String())
+	f, err := os.Create(name)
+	if err != nil {
+		return fmt.Errorf("failed to create blob %s: %v", name, err)
+	}
+	defer f.Close()
+
+	_, err = io.Copy(f, r)
+	if err != nil {
+		return fmt.Errorf("failed to copy: %v", err)
+	}
+	return nil
+}
+
+func (b *BlobCache) StoreBytes(d *v1.Hash, bytes []byte) error {
+	name := filepath.Join(b.cachePath, d.String())
+	f, err := os.Create(name)
+	if err != nil {
+		return fmt.Errorf("failed to create blob %s: %v", name, err)
+	}
+	defer f.Close()
+
+	_, err = f.Write(bytes)
+	if err != nil {
+		return fmt.Errorf("failed to write content: %v", err)
+	}
+	return nil
+}
+
+func (b *BlobCache) Get(d *v1.Hash) (io.ReadCloser, error) {
+	name := filepath.Join(b.cachePath, d.String())
+	f, err := os.Open(name)
+	if err != nil {
+		return nil, err
+	}
+
+	return f, nil
+}
diff --git a/pkg/oci/pull.go b/pkg/oci/pull.go
@@ -24,15 +24,21 @@ var acceptableMediaTypes = map[types.MediaType]bool{
 }
 
 type Puller struct {
+	cache  *BlobCache
 	logger *logrus.Entry
 }
 
-func NewPuller() *Puller {
+func NewPuller() (*Puller, error) {
+	cache, err := NewBlobCache()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create BlobCache: %v", err)
+	}
 	p := &Puller{
+		cache:  cache,
 		logger: log.G(context.TODO()),
 	}
 
-	return p
+	return p, nil
 }
 
 func (p *Puller) Pull(imageName string, os, arch string) (v1.Layer, *v1.ConfigFile, error) {
@@ -103,15 +109,15 @@ func (p *Puller) Pull(imageName string, os, arch string) (v1.Layer, *v1.ConfigFi
 	return layer, config, nil
 }
 
-func (p *Puller) retrieveFlattenLayerFromIndex(idx v1.ImageIndex, os, arch string) (v1.Layer, *v1.ConfigFile, error) {
+func (p *Puller) retrieveFlattenLayerFromIndex(idx v1.ImageIndex, OS, arch string) (v1.Layer, *v1.ConfigFile, error) {
 	im, err := idx.IndexManifest()
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to IndexManifest: %v", err)
 	}
 
 	manifests := []v1.Descriptor{}
 	for i, m := range im.Manifests {
-		if m.Platform.OS != os {
+		if m.Platform.OS != OS {
 			continue
 		}
 		if m.Platform.Architecture != arch {
@@ -121,7 +127,7 @@ func (p *Puller) retrieveFlattenLayerFromIndex(idx v1.ImageIndex, os, arch strin
 	}
 
 	if len(manifests) == 0 {
-		return nil, nil, fmt.Errorf("no available Image found for os=%s arch=%s", os, arch)
+		return nil, nil, fmt.Errorf("no available Image found for os=%s arch=%s", OS, arch)
 	}
 
 	if len(manifests) > 1 {
@@ -133,12 +139,13 @@ func (p *Puller) retrieveFlattenLayerFromIndex(idx v1.ImageIndex, os, arch strin
 	}
 
 	m := manifests[0]
+
 	img, err := idx.Image(m.Digest)
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to Image with %s: %v", m.Digest, err)
 	}
 
-	layer, config, err := p.retrieveFlattenLayerFromImage(img, os, arch)
+	layer, config, err := p.retrieveFlattenLayerFromImage(img, OS, arch)
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to flatten: %v", err)
 	}
@@ -147,19 +154,55 @@ func (p *Puller) retrieveFlattenLayerFromIndex(idx v1.ImageIndex, os, arch strin
 		return nil, nil, fmt.Errorf("unexpected architecture in config expected=%s actual=%s", arch, config.Architecture)
 	}
 
-	if config.OS != os {
-		return nil, nil, fmt.Errorf("unexpected os in config expected=%s actual=%s", os, config.OS)
+	if config.OS != OS {
+		return nil, nil, fmt.Errorf("unexpected os in config expected=%s actual=%s", OS, config.OS)
 	}
 
 	return layer, config, nil
 }
 
 func (p *Puller) retrieveFlattenLayerFromImage(img v1.Image, os, arch string) (v1.Layer, *v1.ConfigFile, error) {
-	l, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { return mutate.Extract(img), nil })
+	imgDigest, err := img.Digest()
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to get image digest: %v", err)
+	}
+
+	configName, err := img.ConfigName()
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to get ConfigName: %v", err)
+	}
+
+	l, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { return p.cache.Get(&imgDigest) })
+	if err == nil {
+		configReader, err := p.cache.Get(&configName)
+		if err == nil {
+			c, err := v1.ParseConfigFile(configReader)
+			if err == nil {
+				return l, c, nil
+			} else {
+				p.logger.Warnf("failed to parse config blob %s from cache", configName)
+			}
+		} else {
+			p.logger.Warnf("failed to get config blob %s from cache", configName)
+		}
+	} else {
+		p.logger.Warnf("failed to get layer blob %s from cache", imgDigest)
+	}
+
+	l, err = tarball.LayerFromOpener(func() (io.ReadCloser, error) { return mutate.Extract(img), nil })
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed tarball.LayerFromOpener: %v", err)
 	}
 
+	comp, err := l.Compressed()
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to get Layer.Compressed(): %v", err)
+	}
+	err = p.cache.Store(&imgDigest, comp)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to store %s: %v", imgDigest, err)
+	}
+
 	config, err := img.ConfigFile()
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed img.ConfigFile: %v", err)
@@ -173,5 +216,15 @@ func (p *Puller) retrieveFlattenLayerFromImage(img v1.Image, os, arch string) (v
 		return nil, nil, fmt.Errorf("unexpected os in config expected=%s actual=%s", os, config.OS)
 	}
 
+	configBytes, err := img.RawConfigFile()
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to get RawConfigFile: %v", err)
+	}
+
+	err = p.cache.StoreBytes(&configName, configBytes)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to store ConfigFile: %v", err)
+	}
+
 	return l, config, nil
 }