copier: handle globbing with "**" path components

Handle glob patterns with "**" path components by expanding "**" to the
set of subdirectories and calling filepath.Glob() on the results.

Signed-off-by: Nalin Dahyabhai <[email protected]>

copier/copier.go

@@ -41,6 +41,74 @@ func init() {
 	reexec.Register(copierCommand, copierMain)
 }
 
+// extendedGlob calls filepath.Glob() on the passed-in patterns.  If there is a
+// "**" component in the pattern, filepath.Glob() will be called with the "**"
+// replaced with all of the subdirectories under that point, and the results
+// will be concatenated.
+func extendedGlob(pattern string) (matches []string, err error) {
+	subdirectories := func(dir string) []string {
+		var subdirectories []string
+		if err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, _ error) error {
+			if d.IsDir() {
+				if rel, err := filepath.Rel(dir, path); err == nil {
+					subdirectories = append(subdirectories, rel)
+				}
+			}
+			return nil
+		}); err != nil {
+			subdirectories = []string{"."}
+		}
+		return subdirectories
+	}
+	expandPatterns := func(pattern string) []string {
+		patterns := []string{pattern}
+		dir, file := filepath.Split(pattern)
+		var rest string
+		for dir != "" && dir != string(os.PathSeparator) {
+			if file == "**" {
+				subdirectories := subdirectories(dir)
+				patterns := []string{}
+				for _, subdirectory := range subdirectories {
+					if rest == "" {
+						patterns = append(patterns, filepath.Join(dir, subdirectory))
+					} else {
+						patterns = append(patterns, filepath.Join(dir, subdirectory, rest))
+					}
+				}
+				return patterns
+			}
+			if rest == "" {
+				rest = file
+			} else {
+				rest = filepath.Join(file, rest)
+			}
+			if dir != "" && dir[len(dir)-1] == os.PathSeparator {
+				dir = dir[:len(dir)-1]
+			}
+			dir, file = filepath.Split(dir)
+		}
+		return patterns
+	}
+	patterns := []string{pattern}
+	i := 0
+	for i < len(patterns) {
+		expanded := expandPatterns(patterns[i])
+		if len(expanded) > 1 {
+			patterns = append(append(patterns[:i], expanded...), patterns[i+1:]...)
+		} else {
+			i++
+		}
+	}
+	for _, pattern := range patterns {
+		theseMatches, err := filepath.Glob(pattern)
+		if err != nil {
+			return nil, err
+		}
+		matches = append(matches, theseMatches...)
+	}
+	return matches, nil
+}
+
 // isArchivePath returns true if the specified path can be read like a (possibly
 // compressed) tarball.
 func isArchivePath(path string) bool {
@@ -999,7 +1067,7 @@ func copierHandlerStat(req request, pm *fileutils.PatternMatcher) *response {
 			Glob: req.preservedGlobs[i],
 		}
 		// glob this pattern
-		globMatched, err := filepath.Glob(glob)
+		globMatched, err := extendedGlob(glob)
 		if err != nil {
 			s.Error = fmt.Sprintf("copier: stat: %q while matching glob pattern %q", err.Error(), glob)
 		}
@@ -1127,7 +1195,7 @@ func copierHandlerGet(bulkWriter io.Writer, req request, pm *fileutils.PatternMa
 	var queue []string
 	globMatchedCount := 0
 	for _, glob := range req.Globs {
-		globMatched, err := filepath.Glob(glob)
+		globMatched, err := extendedGlob(glob)
 		if err != nil {
 			return errorResponse("copier: get: glob %q: %v", glob, err)
 		}

copier/copier_test.go

@@ -1871,3 +1871,30 @@ func testRemove(t *testing.T) {
 		})
 	}
 }
+
+func TestExtendedGlob(t *testing.T) {
+	tmpdir := t.TempDir()
+	buf := []byte("buffer")
+	var expected1, expected2 []string
+	require.NoError(t, os.Mkdir(filepath.Join(tmpdir, "a"), 0o700))
+	require.NoError(t, os.Mkdir(filepath.Join(tmpdir, "a", "b"), 0o700))
+	require.NoError(t, os.WriteFile(filepath.Join(tmpdir, "a", "b", "a.dat"), buf, 0o600))
+	expected1 = append(expected1, filepath.Join(tmpdir, "a", "b", "a.dat"))
+	require.NoError(t, os.Mkdir(filepath.Join(tmpdir, "b"), 0o700))
+	require.NoError(t, os.Mkdir(filepath.Join(tmpdir, "b", "c"), 0o700))
+	require.NoError(t, os.Mkdir(filepath.Join(tmpdir, "c"), 0o700))
+	require.NoError(t, os.Mkdir(filepath.Join(tmpdir, "c", "d"), 0o700))
+	require.NoError(t, os.WriteFile(filepath.Join(tmpdir, "c", "d", "c.dat"), buf, 0o600))
+	expected1 = append(expected1, filepath.Join(tmpdir, "c", "d", "c.dat"))
+	expected2 = append(expected2, filepath.Join(tmpdir, "c", "d", "c.dat"))
+	require.NoError(t, os.Mkdir(filepath.Join(tmpdir, "d"), 0o700))
+	require.NoError(t, os.WriteFile(filepath.Join(tmpdir, "d", "d.dat"), buf, 0o600))
+	expected1 = append(expected1, filepath.Join(tmpdir, "d", "d.dat"))
+	expected2 = append(expected2, filepath.Join(tmpdir, "d", "d.dat"))
+	matched, err := extendedGlob(filepath.Join(tmpdir, "**", "*.dat"))
+	require.NoError(t, err, "globbing")
+	require.ElementsMatch(t, expected1, matched)
+	matched, err = extendedGlob(filepath.Join(tmpdir, "**", "d", "*.dat"))
+	require.NoError(t, err, "globbing")
+	require.ElementsMatch(t, expected2, matched)
+}

tests/conformance/conformance_test.go

@@ -3264,6 +3264,25 @@ var internalTestCases = []testCase{
 		contextDir:        "header-builtin",
 		dockerUseBuildKit: true,
 	},
+
+	{
+		name:              "copyglob-1",
+		contextDir:        "copyglob",
+		dockerUseBuildKit: true,
+		buildArgs:         map[string]string{"SOURCE": "**/*.txt"},
+	},
+	{
+		name:              "copyglob-2",
+		contextDir:        "copyglob",
+		dockerUseBuildKit: true,
+		buildArgs:         map[string]string{"SOURCE": "**/sub/*.txt"},
+	},
+	{
+		name:              "copyglob-3",
+		contextDir:        "copyglob",
+		dockerUseBuildKit: true,
+		buildArgs:         map[string]string{"SOURCE": "e/**/*sub/*.txt"},
+	},
 }
 
 func TestCommit(t *testing.T) {

tests/conformance/testdata/copyglob/Beach.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/Dockerfile

@@ -0,0 +1,3 @@
+FROM scratch
+ARG SOURCE
+COPY $SOURCE /

tests/conformance/testdata/copyglob/a/sub/subsub/protopatriarchal.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/a/sub/subsub/undestructible.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/b/.sub/gade.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/b/.sub/parcae.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/b/heliacally.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/b/overgoing.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/b/sub/ileosigmoidostomy.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/b/sub/overdilation.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/c/sub/disadvise.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/c/sub/subsub/subsubsub/fiddlecome.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/c/sub/subsub/subsubsub/unweariableness.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/c/sub/travel-sick.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/d/.sub/restocks.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/d/.sub/unblazoned.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/d/sub/alkalinity.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/d/sub/glandules.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/e/.sub/Tytonidae.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/e/.sub/vice-guilty.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/e/Towroy.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/e/sub/subsub/near-blindness.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/e/sub/subsub/paymaster-generalship.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/e/subjectivities.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/f/.sub/bilobate.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/f/.sub/fine-headed.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/f/Etnean.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/f/Sheya.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/f/sub/Vernaccia.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/f/sub/inaccordance.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/f/sub/subsub/subsubsub/ankylosing.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/f/sub/subsub/subsubsub/ocean-born.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/g/sub/huerta.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/g/sub/smopple.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/g/triple-throw.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/g/unexciting.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/h/.sub/Ellston.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/h/.sub/mine-run.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/h/sub/chromaticness.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/h/sub/noninhabitancy.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/i/.sub/Auer.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/i/.sub/hexachlorocyclohexane.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/i/sub/subsub/Minnnie.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/i/sub/subsub/papsquidder.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/i/sub/subsub/subsubsub/Croghan.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/i/sub/subsub/subsubsub/stacc..txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/j/sub/Hamon.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/j/sub/subsub/subsubsub/anapaestic.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/j/sub/subsub/subsubsub/castlelike.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/j/sub/topsy-turvydom.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/k/coembody.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/k/unvoweled.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/l/.sub/galliots.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/l/.sub/minning.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/l/sub/subsub/misidentification.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/l/sub/subsub/palling.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/l/torpifying.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/l/unmarring.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/m/sub/cache.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/m/sub/ribbon-marked.txt

@@ -0,0 +1 @@
+text

tests/conformance/testdata/copyglob/pasteups.txt

@@ -0,0 +1 @@
+text