diff --git a/cmd/ctr-cli/convert/convert.go b/cmd/ctr-cli/convert/convert.go index 5fffe1e..4c3d0b5 100644 --- a/cmd/ctr-cli/convert/convert.go +++ b/cmd/ctr-cli/convert/convert.go @@ -54,7 +54,7 @@ var Flags = []cli.Flag{ }, } -var workDir = filepath.Join(os.TempDir(), "ctr-cli") +var workDir = filepath.Join(os.TempDir(), "ctr-cli", "convert") type dockerImageManifest struct { Config string `json:"Config"` @@ -216,7 +216,7 @@ func Action(c *cli.Context) error { } // convert layer.tar to dimg - tempDir, err := os.MkdirTemp("/tmp/ctr-cli", "*") + tempDir, err := os.MkdirTemp("/tmp/ctr-cli/convert", "*") if err != nil { return err } @@ -237,12 +237,6 @@ func Action(c *cli.Context) error { } } - tempDiffDir, err := os.MkdirTemp("/tmp/ctr-cli", "*") - if err != nil { - return err - } - defer os.RemoveAll(tempDiffDir) - logger.Info("packing dimg") dimgPath := filepath.Join(outputPath, "image.dimg") err = di3fsImage.PackDir(tempDir, dimgPath, threadNum) diff --git a/cmd/ctr-cli/convert2/convert.go b/cmd/ctr-cli/convert2/convert.go index 1f97433..a7f277f 100644 --- a/cmd/ctr-cli/convert2/convert.go +++ b/cmd/ctr-cli/convert2/convert.go @@ -80,7 +80,10 @@ func action(c *cli.Context) error { img := c.String("image") OS := c.String("os") arch := c.String("arch") - puller := oci.NewPuller() + puller, err := oci.NewPuller() + if err != nil { + return fmt.Errorf("failed to create puller: %v", err) + } logger.WithFields(logrus.Fields{"image": img, "os": OS, "arch": arch}).Info("started to pull image") layer, config, err := puller.Pull(img, OS, arch) if err != nil { diff --git a/pkg/oci/cache.go b/pkg/oci/cache.go new file mode 100644 index 0000000..4e4f592 --- /dev/null +++ b/pkg/oci/cache.go @@ -0,0 +1,68 @@ +package oci + +import ( + "fmt" + "io" + "os" + "path/filepath" + + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +var defaultCachePath = filepath.Join(os.TempDir(), "ctr-cli", "cache") + +type BlobCache struct { + cachePath string +} + +func NewBlobCache() (*BlobCache, error) { + c := &BlobCache{ + cachePath: defaultCachePath, + } + err := os.MkdirAll(c.cachePath, 0755) + if err != nil { + return nil, fmt.Errorf("failed to create cache dir %s: %v", c.cachePath, err) + } + + return c, nil +} + +func (b *BlobCache) Store(d *v1.Hash, r io.Reader) error { + name := filepath.Join(b.cachePath, d.String()) + f, err := os.Create(name) + if err != nil { + return fmt.Errorf("failed to create blob %s: %v", name, err) + } + defer f.Close() + + _, err = io.Copy(f, r) + if err != nil { + return fmt.Errorf("failed to copy: %v", err) + } + return nil +} + +func (b *BlobCache) StoreBytes(d *v1.Hash, bytes []byte) error { + name := filepath.Join(b.cachePath, d.String()) + f, err := os.Create(name) + if err != nil { + return fmt.Errorf("failed to create blob %s: %v", name, err) + } + defer f.Close() + + _, err = f.Write(bytes) + if err != nil { + return fmt.Errorf("failed to write content: %v", err) + } + return nil +} + +func (b *BlobCache) Get(d *v1.Hash) (io.ReadCloser, error) { + name := filepath.Join(b.cachePath, d.String()) + f, err := os.Open(name) + if err != nil { + return nil, err + } + + return f, nil +} diff --git a/pkg/oci/pull.go b/pkg/oci/pull.go index 5bca0f6..200d37d 100644 --- a/pkg/oci/pull.go +++ b/pkg/oci/pull.go @@ -24,15 +24,21 @@ var acceptableMediaTypes = map[types.MediaType]bool{ } type Puller struct { + cache *BlobCache logger *logrus.Entry } -func NewPuller() *Puller { +func NewPuller() (*Puller, error) { + cache, err := NewBlobCache() + if err != nil { + return nil, fmt.Errorf("failed to create BlobCache: %v", err) + } p := &Puller{ + cache: cache, logger: log.G(context.TODO()), } - return p + return p, nil } func (p *Puller) Pull(imageName string, os, arch string) (v1.Layer, *v1.ConfigFile, error) { @@ -103,7 +109,7 @@ func (p *Puller) Pull(imageName string, os, arch string) (v1.Layer, *v1.ConfigFi return layer, config, nil } -func (p *Puller) retrieveFlattenLayerFromIndex(idx v1.ImageIndex, os, arch string) (v1.Layer, *v1.ConfigFile, error) { +func (p *Puller) retrieveFlattenLayerFromIndex(idx v1.ImageIndex, OS, arch string) (v1.Layer, *v1.ConfigFile, error) { im, err := idx.IndexManifest() if err != nil { return nil, nil, fmt.Errorf("failed to IndexManifest: %v", err) @@ -111,7 +117,7 @@ func (p *Puller) retrieveFlattenLayerFromIndex(idx v1.ImageIndex, os, arch strin manifests := []v1.Descriptor{} for i, m := range im.Manifests { - if m.Platform.OS != os { + if m.Platform.OS != OS { continue } if m.Platform.Architecture != arch { @@ -121,7 +127,7 @@ func (p *Puller) retrieveFlattenLayerFromIndex(idx v1.ImageIndex, os, arch strin } if len(manifests) == 0 { - return nil, nil, fmt.Errorf("no available Image found for os=%s arch=%s", os, arch) + return nil, nil, fmt.Errorf("no available Image found for os=%s arch=%s", OS, arch) } if len(manifests) > 1 { @@ -133,12 +139,13 @@ func (p *Puller) retrieveFlattenLayerFromIndex(idx v1.ImageIndex, os, arch strin } m := manifests[0] + img, err := idx.Image(m.Digest) if err != nil { return nil, nil, fmt.Errorf("failed to Image with %s: %v", m.Digest, err) } - layer, config, err := p.retrieveFlattenLayerFromImage(img, os, arch) + layer, config, err := p.retrieveFlattenLayerFromImage(img, OS, arch) if err != nil { return nil, nil, fmt.Errorf("failed to flatten: %v", err) } @@ -147,19 +154,55 @@ func (p *Puller) retrieveFlattenLayerFromIndex(idx v1.ImageIndex, os, arch strin return nil, nil, fmt.Errorf("unexpected architecture in config expected=%s actual=%s", arch, config.Architecture) } - if config.OS != os { - return nil, nil, fmt.Errorf("unexpected os in config expected=%s actual=%s", os, config.OS) + if config.OS != OS { + return nil, nil, fmt.Errorf("unexpected os in config expected=%s actual=%s", OS, config.OS) } return layer, config, nil } func (p *Puller) retrieveFlattenLayerFromImage(img v1.Image, os, arch string) (v1.Layer, *v1.ConfigFile, error) { - l, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { return mutate.Extract(img), nil }) + imgDigest, err := img.Digest() + if err != nil { + return nil, nil, fmt.Errorf("failed to get image digest: %v", err) + } + + configName, err := img.ConfigName() + if err != nil { + return nil, nil, fmt.Errorf("failed to get ConfigName: %v", err) + } + + l, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { return p.cache.Get(&imgDigest) }) + if err == nil { + configReader, err := p.cache.Get(&configName) + if err == nil { + c, err := v1.ParseConfigFile(configReader) + if err == nil { + return l, c, nil + } else { + p.logger.Warnf("failed to parse config blob %s from cache", configName) + } + } else { + p.logger.Warnf("failed to get config blob %s from cache", configName) + } + } else { + p.logger.Warnf("failed to get layer blob %s from cache", imgDigest) + } + + l, err = tarball.LayerFromOpener(func() (io.ReadCloser, error) { return mutate.Extract(img), nil }) if err != nil { return nil, nil, fmt.Errorf("failed tarball.LayerFromOpener: %v", err) } + comp, err := l.Compressed() + if err != nil { + return nil, nil, fmt.Errorf("failed to get Layer.Compressed(): %v", err) + } + err = p.cache.Store(&imgDigest, comp) + if err != nil { + return nil, nil, fmt.Errorf("failed to store %s: %v", imgDigest, err) + } + config, err := img.ConfigFile() if err != nil { return nil, nil, fmt.Errorf("failed img.ConfigFile: %v", err) @@ -173,5 +216,15 @@ func (p *Puller) retrieveFlattenLayerFromImage(img v1.Image, os, arch string) (v return nil, nil, fmt.Errorf("unexpected os in config expected=%s actual=%s", os, config.OS) } + configBytes, err := img.RawConfigFile() + if err != nil { + return nil, nil, fmt.Errorf("failed to get RawConfigFile: %v", err) + } + + err = p.cache.StoreBytes(&configName, configBytes) + if err != nil { + return nil, nil, fmt.Errorf("failed to store ConfigFile: %v", err) + } + return l, config, nil }