Add sparse_directories input to support cloning specific directorie…

…s of a repo. (#153)

### Checklist

- [x] I've read and accepted the [Contribution Guidelines](https://github.com/bitrise-steplib/.github/blob/main/CONTRIBUTING.md)
- Requires MINOR [version update](https://semver.org/)

### Context

Add `sparse_directories` input to support cloning specific directories of a repo.

Resolves: [STEP-626](https://bitrise.atlassian.net/browse/STEP-626)

### Changes

- `sparse_directories` can be a single directory or a multiple directories
 ```
- sparse_directories: client/android
OR
- sparse_directories: |-
                client/android
                client/common
 ```
- If `sparse_directories` is specified the `--filter=tree:0` flag will be passed to `git fetch` and `--depth` flag will be removed. More info on [--filter=tree:0](https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/#user-content-treeless-clones)

gitclone/checkout.go

@@ -269,12 +269,35 @@ func createCheckoutStrategy(checkoutMethod CheckoutMethod, cfg Config, patchFile
 
 }
 
-func selectFetchOptions(checkoutStrategy CheckoutMethod, cloneDepth int, fetchTags, fetchSubmodules bool) fetchOptions {
-	return fetchOptions{
+func selectFetchOptions(checkoutStrategy CheckoutMethod, cloneDepth int, fetchTags, fetchSubmodules bool, filterTree bool) fetchOptions {
+	opts := fetchOptions{
 		depth:           cloneDepth,
 		tags:            fetchTags,
 		fetchSubmodules: fetchSubmodules,
 	}
+
+	opts = selectFilterTreeFetchOption(checkoutStrategy, opts, filterTree)
+
+	return opts
+}
+
+func selectFilterTreeFetchOption(checkoutStrategy CheckoutMethod, opts fetchOptions, filterTree bool) fetchOptions {
+	if !filterTree {
+		return opts
+	}
+
+	switch checkoutStrategy {
+	case CheckoutCommitMethod,
+		CheckoutTagMethod,
+		CheckoutBranchMethod,
+		CheckoutHeadBranchCommitMethod,
+		CheckoutForkCommitMethod:
+		opts.filterTree = true
+		opts.depth = 0
+	default:
+	}
+
+	return opts
 }
 
 func selectFallbacks(checkoutStrategy CheckoutMethod, fetchOpts fetchOptions) fallbackRetry {

gitclone/checkout_helper.go

@@ -22,6 +22,9 @@ type fetchOptions struct {
 	// Sets '--no-recurse-submodules' flag
 	// More info: https://git-scm.com/docs/git-fetch#Documentation/git-fetch.txt---no-recurse-submodules
 	fetchSubmodules bool
+	// Sets `--filter=tree:0` flag
+	// More info: https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/#user-content-treeless-clones
+	filterTree bool
 }
 
 func (t fetchOptions) IsFullDepth() bool {
@@ -38,6 +41,9 @@ func fetch(gitCmd git.Git, remote string, ref string, traits fetchOptions) error
 	if traits.depth != 0 {
 		opts = append(opts, "--depth="+strconv.Itoa(traits.depth))
 	}
+	if traits.filterTree {
+		opts = append(opts, `--filter=tree:0`)
+	}
 
 	if traits.tags {
 		opts = append(opts, "--tags")

gitclone/gitclone.go

@@ -23,11 +23,12 @@ type Config struct {
 	PRMergeBranch         string `env:"pull_request_merge_branch"`
 	PRHeadBranch          string `env:"pull_request_head_branch"`
 
-	ResetRepository           bool `env:"reset_repository,opt[Yes,No]"`
-	CloneDepth                int  `env:"clone_depth"`
-	FetchTags                 bool `env:"fetch_tags,opt[yes,no]"`
-	LimitSubmoduleUpdateDepth bool `env:"limit_submodule_update_depth,opt[yes,no]"`
-	ShouldMergePR             bool `env:"merge_pr,opt[yes,no]"`
+	ResetRepository           bool     `env:"reset_repository,opt[Yes,No]"`
+	CloneDepth                int      `env:"clone_depth"`
+	FetchTags                 bool     `env:"fetch_tags,opt[yes,no]"`
+	LimitSubmoduleUpdateDepth bool     `env:"limit_submodule_update_depth,opt[yes,no]"`
+	ShouldMergePR             bool     `env:"merge_pr,opt[yes,no]"`
+	SparseDirectories         []string `env:"sparse_directories,multiline"`
 
 	BuildURL         string `env:"build_url"`
 	BuildAPIToken    string `env:"build_api_token"`
@@ -40,6 +41,7 @@ const (
 	originRemoteName        = "origin"
 	forkRemoteName          = "fork"
 	updateSubmodelFailedTag = "update_submodule_failed"
+	sparseCheckoutFailedTag = "sparse_checkout_failed"
 )
 
 func printLogAndExportEnv(gitCmd git.Git, format, env string, maxEnvLength int) error {
@@ -72,7 +74,7 @@ func getMaxEnvLength() (int, error) {
 
 func checkoutState(gitCmd git.Git, cfg Config, patch patchSource) error {
 	checkoutMethod, diffFile := selectCheckoutMethod(cfg, patch)
-	fetchOpts := selectFetchOptions(checkoutMethod, cfg.CloneDepth, cfg.FetchTags, cfg.UpdateSubmodules)
+	fetchOpts := selectFetchOptions(checkoutMethod, cfg.CloneDepth, cfg.FetchTags, cfg.UpdateSubmodules, len(cfg.SparseDirectories) != 0)
 
 	checkoutStrategy, err := createCheckoutStrategy(checkoutMethod, cfg, diffFile)
 	if err != nil {
@@ -102,6 +104,32 @@ func updateSubmodules(gitCmd git.Git, cfg Config) error {
 	return nil
 }
 
+func setupSparseCheckout(gitCmd git.Git, sparseDirectories []string) error {
+	if len(sparseDirectories) == 0 {
+		return nil
+	}
+
+	initCommand := gitCmd.SparseCheckoutInit(true)
+	if err := runner.Run(initCommand); err != nil {
+		return newStepError(
+			sparseCheckoutFailedTag,
+			fmt.Errorf("initializing sparse-checkout config failed: %v", err),
+			"Initializing sparse-checkout config has failed",
+		)
+	}
+
+	sparseSetCommand := gitCmd.SparseCheckoutSet(sparseDirectories...)
+	if err := runner.Run(sparseSetCommand); err != nil {
+		return newStepError(
+			sparseCheckoutFailedTag,
+			fmt.Errorf("updating sparse-checkout config failed: %v", err),
+			"Updating sparse-checkout config has failed",
+		)
+	}
+
+	return nil
+}
+
 // Execute is the entry point of the git clone process
 func Execute(cfg Config) error {
 	maxEnvLength, err := getMaxEnvLength()
@@ -157,6 +185,10 @@ func Execute(cfg Config) error {
 		}
 	}
 
+	if err := setupSparseCheckout(gitCmd, cfg.SparseDirectories); err != nil {
+		return err
+	}
+
 	if err := checkoutState(gitCmd, cfg, defaultPatchSource{}); err != nil {
 		return err
 	}

gitclone/gitclone_test.go

@@ -559,6 +559,57 @@ var testCases = [...]struct {
 			`git "checkout" "--detach"`,
 		},
 	},
+
+	// ** Sparse-checkout **
+	{
+		name: "Checkout commit - sparse",
+		cfg: Config{
+			Commit:            "76a934a",
+			CloneDepth:        1,
+			SparseDirectories: []string{"client/android"},
+		},
+		wantCmds: []string{
+			`git "fetch" "--filter=tree:0" "--no-tags" "--no-recurse-submodules"`,
+			`git "checkout" "76a934a"`,
+		},
+	},
+	{
+		name: "Checkout commit, branch specified - sparse",
+		cfg: Config{
+			Commit:            "76a934ae",
+			Branch:            "hcnarb",
+			SparseDirectories: []string{"client/android"},
+		},
+		wantCmds: []string{
+			`git "fetch" "--filter=tree:0" "--no-tags" "--no-recurse-submodules" "origin" "refs/heads/hcnarb"`,
+			`git "checkout" "76a934ae"`,
+		},
+	},
+	{
+		name: "Checkout branch - sparse",
+		cfg: Config{
+			Branch:            "hcnarb",
+			CloneDepth:        1,
+			SparseDirectories: []string{"client/android"},
+		},
+		wantCmds: []string{
+			`git "fetch" "--filter=tree:0" "--no-tags" "--no-recurse-submodules" "origin" "refs/heads/hcnarb"`,
+			`git "checkout" "hcnarb"`,
+			`git "merge" "origin/hcnarb"`,
+		},
+	},
+	{
+		name: "Checkout tag - sparse",
+		cfg: Config{
+			Tag:               "gat",
+			CloneDepth:        1,
+			SparseDirectories: []string{"client/android"},
+		},
+		wantCmds: []string{
+			`git "fetch" "--filter=tree:0" "--no-tags" "--no-recurse-submodules" "origin" "refs/tags/gat:refs/tags/gat"`,
+			`git "checkout" "gat"`,
+		},
+	},
 }
 
 func Test_checkoutState(t *testing.T) {
@@ -633,6 +684,47 @@ func Test_SubmoduleUpdate(t *testing.T) {
 	}
 }
 
+// SetupSparseCechkout
+var sparseCheckoutTestCases = [...]struct {
+	name              string
+	sparseDirectories []string
+	wantCmds          []string
+}{
+	{
+		name:              "Sparse-checkout single directory",
+		sparseDirectories: []string{"client/android"},
+		wantCmds: []string{
+			`git "sparse-checkout" "init" "--cone"`,
+			`git "sparse-checkout" "set" "client/android"`,
+		},
+	},
+	{
+		name:              "Sparse-checkout multiple directory",
+		sparseDirectories: []string{"client/android", "client/ios"},
+		wantCmds: []string{
+			`git "sparse-checkout" "init" "--cone"`,
+			`git "sparse-checkout" "set" "client/android" "client/ios"`,
+		},
+	},
+}
+
+func Test_SetupSparseCheckout(t *testing.T) {
+	for _, tt := range sparseCheckoutTestCases {
+		t.Run(tt.name, func(t *testing.T) {
+			// Given
+			mockRunner := givenMockRunnerSucceeds()
+			runner = mockRunner
+
+			// When
+			actualErr := setupSparseCheckout(git.Git{}, tt.sparseDirectories)
+
+			// Then
+			assert.NoError(t, actualErr)
+			assert.Equal(t, tt.wantCmds, mockRunner.Cmds())
+		})
+	}
+}
+
 // Mocks
 func givenMockRunner() *MockRunner {
 	mockRunner := new(MockRunner)

go.mod

@@ -5,11 +5,11 @@ go 1.16
 require (
 	github.com/bitrise-io/bitrise-init v0.0.0-20201117094539-a984f01dd477
 	github.com/bitrise-io/envman v0.0.0-20200512105748-919e33f391ee
-	github.com/bitrise-io/go-steputils v0.0.0-20201016102104-03ae3a6ded35
-	github.com/bitrise-io/go-utils v0.0.0-20210316133228-449620935158
+	github.com/bitrise-io/go-steputils v0.0.0-20210324082442-21a1b8f2b237
+	github.com/bitrise-io/go-utils v0.0.0-20210323091856-00429d8e1e87
 	github.com/bitrise-io/goinp v0.0.0-20190611131639-bd18a8681e27 // indirect
 	github.com/bitrise-steplib/bitrise-step-export-universal-apk v0.0.0-20200729103519-a582681d23d6
-	github.com/stretchr/testify v1.5.1
+	github.com/stretchr/testify v1.6.1
 	golang.org/x/crypto v0.0.0-20201116153603-4be66e5b6582 // indirect
 	golang.org/x/sys v0.0.0-20201116194326-cc9327a14d48 // indirect
 	gopkg.in/yaml.v2 v2.3.0 // indirect

go.sum

@@ -4,10 +4,15 @@ github.com/bitrise-io/envman v0.0.0-20200512105748-919e33f391ee h1:zZiMNcVl2WhYx
 github.com/bitrise-io/envman v0.0.0-20200512105748-919e33f391ee/go.mod h1:m8pTp1o3Sw9uzDxb1WRm5IBRnMau2iOvPMSnRCAhQNI=
 github.com/bitrise-io/go-steputils v0.0.0-20201016102104-03ae3a6ded35 h1:iKtx/RxSrA9xcjZ17W3yu3jMzwowNB6nxLF/1BwVYQ8=
 github.com/bitrise-io/go-steputils v0.0.0-20201016102104-03ae3a6ded35/go.mod h1:GXgBV3Frd3qcnsg+NryQTyx1CHjZHr/2w7Bx4WAcB4o=
+github.com/bitrise-io/go-steputils v0.0.0-20210324082442-21a1b8f2b237 h1:pf2j3cJjJ2gwMOED21U8QbIu3xg3+k3Th3HkVK8MwkY=
+github.com/bitrise-io/go-steputils v0.0.0-20210324082442-21a1b8f2b237/go.mod h1:Z/Vll330H6Ws7INi8bpUNpWP0+N+dLSN4d0Fnc+j5B0=
+github.com/bitrise-io/go-utils v0.0.0-20200629150542-0c47c16813a4/go.mod h1:tTEsKvbz1LbzuN/KpVFHXnLtcAPdEgIdM41s0lL407s=
 github.com/bitrise-io/go-utils v0.0.0-20201019131314-6cc2aa4d248a h1:i1hLZTX2L30oYmaeSnSjDRPQDvwgsFkH4OCr16Vyy+8=
 github.com/bitrise-io/go-utils v0.0.0-20201019131314-6cc2aa4d248a/go.mod h1:tTEsKvbz1LbzuN/KpVFHXnLtcAPdEgIdM41s0lL407s=
 github.com/bitrise-io/go-utils v0.0.0-20210316133228-449620935158 h1:GZjx3KhYSFlfzEXN7Uw48MgFgt0QVU2z3R9KVnbcZ4Y=
 github.com/bitrise-io/go-utils v0.0.0-20210316133228-449620935158/go.mod h1:tTEsKvbz1LbzuN/KpVFHXnLtcAPdEgIdM41s0lL407s=
+github.com/bitrise-io/go-utils v0.0.0-20210323091856-00429d8e1e87 h1:RKsBLQLMUwrYQAiQxVUtfs2yDSGatUeMfJn9zc/jy1U=
+github.com/bitrise-io/go-utils v0.0.0-20210323091856-00429d8e1e87/go.mod h1:tTEsKvbz1LbzuN/KpVFHXnLtcAPdEgIdM41s0lL407s=
 github.com/bitrise-io/goinp v0.0.0-20190611131639-bd18a8681e27 h1:NuGIfwKcZvdR1RT4sQ32kE8h35w9XQjQrrCJPJqS4ek=
 github.com/bitrise-io/goinp v0.0.0-20190611131639-bd18a8681e27/go.mod h1:G0M1sK06a1l0KAg4rQei7q5dDKC/JrZoaXFqak91osU=
 github.com/bitrise-steplib/bitrise-step-export-universal-apk v0.0.0-20200729103519-a582681d23d6 h1:7UWHsApY8/iIGw1jVbDiL3FMnI70Y/t0BxG9nQupHhI=
@@ -20,8 +25,13 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As=
+github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -51,3 +61,6 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

step.yml

@@ -130,8 +130,18 @@ inputs:
         - `yes`: The default setting. Merges the source branch into the destination branch.
         - `no`: Treats Pull Request events as Push events on the source branch.
       value_options:
-      - "yes"
-      - "no"
+        - "yes"
+        - "no"
+  - sparse_directories: ""
+    opts:
+      category: "Checkout options"
+      title: "Specify which directories to clone."
+      description: |-
+        Limit which directories should be cloned during the build. This could be useful if a repository contains multiple platforms, so called monorepositories, and the build is only targeting a single platform.
+        For example, specifying "src/android" the Step will only clone:
+        - contents of the root directory and
+        - contents of the "src/android" directory and all subdirectories of "src/android".
+        On the other hand, "src/ios" and any other directories will not be cloned.
   - reset_repository: "No"
     opts:
       category: Debug
@@ -149,8 +159,8 @@ inputs:
         Prefer to do a manual `git merge` by default.
         When the Pull Request is from a GitHub or Bitbucket private fork repository set this to `no`.
       value_options:
-      - "yes"
-      - "no"
+        - "yes"
+        - "no"
   - fetch_tags: "no"
     opts:
       category: Clone Config
@@ -159,8 +169,8 @@ inputs:
         yes - fetch all tags from the remote by adding `--tags` flag to git fetch calls  
         no - disable automatic tag following by adding `--no-tags` flag to git fetch calls
       value_options:
-      - "yes"
-      - "no"
+        - "yes"
+        - "no"
   - build_url: "$BITRISE_BUILD_URL"
     opts:
       category: Debug