config,storage: support populating directories from archives

Tarballs are ubiquitous as a binary release format, and without any
builtin ability to fetch, validate, and extract them, we are left
with half-baked hacks to do all of this from the confines of a oneshot
systemd service, or worse, extracting and providing the entirety of the
archive contents as files and directory entries in the ignition config,
resulting in very large json documents.

Let's not do this. Instead, this commit formalizes the use of archives
via adding a new (optional) "contents" key under a directory entry.

This new contents key is identical in function as its eponymous version
in the "files" entries, except that it incorporates a new "archive"
subkey to specify the archive format rather than guessing with
heuristics. Today, only "archive": "tar" is supported, though this
commit is structured to allow the addition of other archive types if
needed.

config/shared/errors/errors.go

@@ -84,6 +84,9 @@ var (
 	ErrInvalidProxy              = errors.New("proxies must be http(s)")
 	ErrInsecureProxy             = errors.New("insecure plaintext HTTP proxy specified for HTTPS resources")
 	ErrPathConflictsSystemd      = errors.New("path conflicts with systemd unit or dropin")
+	ErrUnsupportedArchiveType    = errors.New("unsupported archive type")
+	ErrArchiveTypeRequired       = errors.New("archive type is required")
+	ErrOverwriteMustBeTrue       = errors.New("overwrite must be true when specifying directory contents")
 
 	// Systemd section errors
 	ErrInvalidSystemdExt       = errors.New("invalid systemd unit extension")

config/v3_4_experimental/schema/ignition.json

@@ -40,6 +40,21 @@
         }
       }
     },
+    "archiveResource": {
+      "allOf": [
+        {
+          "$ref": "#/definitions/resource"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "archive": {
+              "type": ["string", "null"]
+            }
+          }
+        }
+      ]
+    },
     "verification": {
       "type": "object",
       "properties": {
@@ -394,6 +409,9 @@
               "properties": {
                 "mode": {
                   "type": ["integer", "null"]
+                },
+                "contents": {
+                  "$ref": "#/definitions/archiveResource"
                 }
               }
             }

config/v3_4_experimental/types/directory.go

@@ -15,12 +15,27 @@
 package types
 
 import (
+	"github.com/coreos/ignition/v2/config/shared/errors"
+	"github.com/coreos/ignition/v2/config/util"
+
 	"github.com/coreos/vcontext/path"
 	"github.com/coreos/vcontext/report"
 )
 
 func (d Directory) Validate(c path.ContextPath) (r report.Report) {
 	r.Merge(d.Node.Validate(c))
 	r.AddOnError(c.Append("mode"), validateMode(d.Mode))
+	if !util.NilOrEmpty(d.Contents.Archive) && (d.Overwrite == nil || !*d.Overwrite) {
+		r.AddOnError(c.Append("overwrite"), errors.ErrOverwriteMustBeTrue)
+	}
 	return
 }
+
+func (d Directory) KeyPrefix() string {
+	if util.NilOrEmpty(d.Contents.Archive) {
+		return ""
+	}
+	// If a directory is populated by an archive, all other file/directory entries
+	// in the config must conflict with any files under said directory.
+	return d.Path
+}

config/v3_4_experimental/types/resource.go

@@ -89,3 +89,24 @@ func (res Resource) validateRequiredSource() error {
 	}
 	return validateURL(*res.Source)
 }
+
+func (res ArchiveResource) Validate(c path.ContextPath) (r report.Report) {
+	r.Merge(res.Resource.Validate(c))
+	r.AddOnError(c.Append("archive"), res.validateArchive())
+	return
+}
+
+func (res ArchiveResource) validateArchive() error {
+	if util.NilOrEmpty(res.Source) {
+		// archive can be omitted iff the contents are omitted
+		return nil
+	}
+	if util.NilOrEmpty(res.Archive) {
+		return errors.ErrArchiveTypeRequired
+	}
+	switch *res.Archive {
+	case "tar":
+		return nil
+	}
+	return errors.ErrUnsupportedArchiveType
+}

config/v3_4_experimental/types/schema.go

@@ -2,6 +2,15 @@ package types
 
 // generated by "schematyper --package=types config/v3_4_experimental/schema/ignition.json -o config/v3_4_experimental/types/schema.go --root-type=Config" -- DO NOT EDIT
 
+type ArchiveResource struct {
+	Resource
+	ArchiveResourceEmbedded1
+}
+
+type ArchiveResourceEmbedded1 struct {
+	Archive *string `json:"archive,omitempty"`
+}
+
 type Clevis struct {
 	Custom    ClevisCustom `json:"custom,omitempty"`
 	Tang      []Tang       `json:"tang,omitempty"`
@@ -31,7 +40,8 @@ type Directory struct {
 }
 
 type DirectoryEmbedded1 struct {
-	Mode *int `json:"mode,omitempty"`
+	Contents ArchiveResource `json:"contents,omitempty"`
+	Mode     *int            `json:"mode,omitempty"`
 }
 
 type Disk struct {

docs/configuration-v3_4_experimental.md

@@ -111,6 +111,15 @@ The Ignition configuration is a JSON document conforming to the following specif
     * **_group_** (object): specifies the directory's group.
       * **_id_** (integer): the group ID of the group.
       * **_name_** (string): the group name of the group.
+    * **_contents_** (object): options related to the contents of the directory. If specified, `overwrite` must be `true`. Directories populated from an archive own all files under it. This means that specifying files, directories, and links under the path of this directory always result in a conflict error during config validation.
+      * **archive** (string): format of the archive to extract into the directory. Must be `tar`. If `tar` is specified, the source must be a USTAR, PAX, or GNU tarball. Only regular files, directories, and links (both hard links and symlinks) are extracted, other file types are ignored and emit a warning. Note that for `tar` archives, sparse files are not supported and processing an archive with one will result in an error.
+      * **_compression_** (string): the type of compression used on the archive (null or gzip). Compression cannot be used with S3.
+      * **_source_** (string): the URL of the archive to extract. Supported schemes are `http`, `https`, `tftp`, `s3`, `arn`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. If source is omitted and a directory already exists at the path, Ignition will do nothing. If source is omitted and no directory exists, an empty directory will be created.
+      * **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only.
+        * **name** (string): the header name.
+        * **_value_** (string): the header contents.
+      * **_verification_** (object): options related to the verification of the archive file.
+        * **_hash_** (string): the hash of the archive file, in the form `<type>-<value>` where type is either `sha512` or `sha256`.
   * **_links_** (list of objects): the list of links to be created. Every file, directory, and link must have a unique `path`.
     * **path** (string): the absolute path to the link
     * **_overwrite_** (boolean): whether to delete preexisting nodes at the path. If overwrite is false and a matching link exists at the path, Ignition will only set the owner and group. Defaults to false.

docs/release-notes.md

@@ -12,6 +12,7 @@ nav_order: 9
 ### Features
 
 - Ship aarch64 macOS ignition-validate binary in GitHub release artifacts
+- Add the ability to populate directory from tar archives.
 
 ### Changes
 

internal/exec/stages/files/filesystemEntries.go

@@ -17,6 +17,7 @@ package files
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -285,7 +286,7 @@ func (tmp fileEntry) create(l *log.Logger, u util.Util) error {
 
 	for _, op := range fetchOps {
 		msg := "writing file %q"
-		if op.Append {
+		if op.Mode == util.FetchAppend {
 			msg = "appending to file %q"
 		}
 		if err := l.LogOp(
@@ -323,6 +324,33 @@ func (tmp dirEntry) create(l *log.Logger, u util.Util) error {
 		return fmt.Errorf("error creating directory %s: A non-directory already exists and overwrite is false", d.Path)
 	}
 
+	if d.Contents.Archive != nil {
+		dirf, err := os.Open(d.Path)
+		if err != nil {
+			return fmt.Errorf("open() failed on %s: %v", d.Path, err)
+		}
+		switch _, err := dirf.Readdirnames(1); {
+		case err == nil:
+			return fmt.Errorf("refusing to populate directory %s: directory is not empty and overwrite is false", d.Path)
+		case err != io.EOF:
+			return fmt.Errorf("readdirnames() failed on %s: %v", d.Path, err)
+		}
+
+		fetch, err := util.MakeFetchOp(l, d.Node, d.Contents.Resource)
+		if err != nil {
+			return fmt.Errorf("failed to resolve directory %q: %v", d.Path, err)
+		}
+		fetch.Mode = util.FetchExtract
+		fetch.ArchiveType = util.ArchiveType(*d.Contents.Archive)
+
+		op := func() error {
+			return u.PerformFetch(fetch)
+		}
+		if err := l.LogOp(op, "populating directory %q", d.Path); err != nil {
+			return fmt.Errorf("failed to populate directory %q: %v", d.Path, err)
+		}
+	}
+
 	if err := u.SetPermissions(d.Mode, d.Node); err != nil {
 		return fmt.Errorf("error setting directory permissions for %s: %v", d.Path, err)
 	}