Skip to content

Commit

Permalink
Add experimental support for private http repositories (#212)
Browse files Browse the repository at this point in the history
  • Loading branch information
gaborszakacs authored Jun 19, 2023
1 parent f29bfca commit 67b0d01
Show file tree
Hide file tree
Showing 11 changed files with 229 additions and 2 deletions.
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,8 @@ You can also run this step directly with [Bitrise CLI](https://github.com/bitris
| `reset_repository` | Reset repository contents with `git reset --hard HEAD` and `git clean -f` before fetching. | | `No` |
| `build_url` | Unique build URL of this build on Bitrise.io | | `$BITRISE_BUILD_URL` |
| `build_api_token` | The build's API Token for the build on Bitrise.io | sensitive | `$BITRISE_BUILD_API_TOKEN` |
| `git_http_username` | Username for establishing an HTTP(S) connection to the repository | sensitive | `$GIT_HTTP_USERNAME` |
| `git_http_password` | Personal access token (or password) for establishing an HTTP(S) connection to the repository | sensitive | `$GIT_HTTP_PASSWORD` |
</details>

<details>
Expand Down
30 changes: 30 additions & 0 deletions e2e/bitrise.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ app:
- SPARSE_DIRECTORIES: ""
# define these envs in your .bitrise.secrets.yml
- GIT_CLONE_SSH_PRIVATE_KEY: $GIT_CLONE_SSH_PRIVATE_KEY
- GIT_HTTP_PASSWORD_FOR_TEST: $GIT_HTTP_PASSWORD_FOR_TEST

workflows:
test_generate_changelog_by_commit:
Expand Down Expand Up @@ -444,6 +445,35 @@ workflows:
- _check_too_long_message
- _teardown

test_private_http_repository:
envs:
- TEST_REPO_URL: https://github.com/websitebot/git-repo-fixture-private.git
- BRANCH: main
# Doesn't matter for GitHub
- GIT_HTTP_USERNAME: username
# Fine grained personal access token for the single repo having Contents rights,
# will expire at 2024.06.01
- GIT_HTTP_PASSWORD: $GIT_HTTP_PASSWORD_FOR_TEST
before_run:
- _setup
after_run:
- _run
- _check_outputs
- _teardown

test_private_http_repository_without_username_set:
envs:
- TEST_REPO_URL: https://github.com/websitebot/git-repo-fixture-private.git
- BRANCH: main
- GIT_HTTP_USERNAME: ""
- GIT_HTTP_PASSWORD: $GIT_HTTP_PASSWORD_FOR_TEST
before_run:
- _setup
after_run:
- _run
- _check_outputs
- _teardown

_check_changelog:
steps:
- generate-changelog:
Expand Down
3 changes: 1 addition & 2 deletions gitclone/gitclone.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,11 @@ import (
"fmt"
"time"

"github.com/bitrise-io/go-utils/command/git"
"github.com/bitrise-io/go-utils/v2/command"
"github.com/bitrise-io/go-utils/v2/log"
"github.com/bitrise-steplib/steps-git-clone/gitclone/bitriseapi"
"github.com/bitrise-steplib/steps-git-clone/gitclone/tracker"

"github.com/bitrise-io/go-utils/command/git"
)

const (
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ require (
github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.15
github.com/bitrise-io/go-utils v1.0.1
github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.15
github.com/bitrise-steplib/steps-authenticate-host-with-netrc v0.0.0-20230216105320-8cb845d52e28
github.com/hashicorp/go-retryablehttp v0.7.1
github.com/stretchr/testify v1.8.1
)
Expand Down
5 changes: 5 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ github.com/bitrise-io/bitrise v0.0.0-20210513161711-09b1442cf3e0/go.mod h1:Jqf2P
github.com/bitrise-io/bitrise-init v0.0.0-20210518121553-1e678625c45d h1:JXSwWXDsEXIXYtlXpFnOUc7kMW8hGZIzlYLxFaLawvc=
github.com/bitrise-io/bitrise-init v0.0.0-20210518121553-1e678625c45d/go.mod h1:2bxvLp7xgXC3HsacG2Z5nn6WOjk8kbNOmLWjinnhnKg=
github.com/bitrise-io/colorstring v0.0.0-20180614154802-a8cd70115192/go.mod h1:CIHVcxZUvsG99XUJV6JlR7okNsMMGY81jMvPC20W+O0=
github.com/bitrise-io/depman v0.0.0-20160708153333-4c59bc31f02a/go.mod h1:jeK3Koe+wXMSwfl8t6M7kovGgXsayUJ5ybqo3gDJgts=
github.com/bitrise-io/envman v0.0.0-20200512105748-919e33f391ee/go.mod h1:m8pTp1o3Sw9uzDxb1WRm5IBRnMau2iOvPMSnRCAhQNI=
github.com/bitrise-io/envman v0.0.0-20210517135508-b2b4fe89eac5 h1:3jVXjsejV+1EW9Sp7ef+sMinD0J3HFsPgh4Du6ctqZc=
github.com/bitrise-io/envman v0.0.0-20210517135508-b2b4fe89eac5/go.mod h1:m8pTp1o3Sw9uzDxb1WRm5IBRnMau2iOvPMSnRCAhQNI=
Expand All @@ -14,6 +15,7 @@ github.com/bitrise-io/go-steputils v1.0.5 h1:OBH7CPXeqIWFWJw6BOUMQnUb8guspwKr2Rh
github.com/bitrise-io/go-steputils v1.0.5/go.mod h1:YIUaQnIAyK4pCvQG0hYHVkSzKNT9uL2FWmkFNW4mfNI=
github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.15 h1:wG037NV+pS8cEwtalE5K58bmKLyUkU0+4m4IuXjTzmo=
github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.15/go.mod h1:M09BbxYoh6B7KJnXk/yvtuU5nZPh7RQBJGKc1dp+0hQ=
github.com/bitrise-io/go-utils v0.0.0-20170404153453-c8163959178a/go.mod h1:Pp48eQd8BSNEzWWTBxdFUd/ufS5b5Bgkmt9D5NedjWg=
github.com/bitrise-io/go-utils v0.0.0-20201019131314-6cc2aa4d248a/go.mod h1:tTEsKvbz1LbzuN/KpVFHXnLtcAPdEgIdM41s0lL407s=
github.com/bitrise-io/go-utils v0.0.0-20210505091801-98b7dc39ee61/go.mod h1:nhdaDQFvaMny1CugVV6KjK92/q97ENo0RuKSW5I4fbA=
github.com/bitrise-io/go-utils v0.0.0-20210505121718-07411d72e36e/go.mod h1:nhdaDQFvaMny1CugVV6KjK92/q97ENo0RuKSW5I4fbA=
Expand All @@ -35,6 +37,8 @@ github.com/bitrise-io/stepman v0.0.0-20210505110307-5c2296bcc558/go.mod h1:WLh58
github.com/bitrise-io/stepman v0.0.0-20210517135458-203f7a48d37a/go.mod h1:WLh58JYBgbD1Z/yyw1AkFz/90F6oBL0HS/luBpUW9dI=
github.com/bitrise-io/xcode-project v0.0.0-20201203153351-7ad13a1dd021/go.mod h1:R2iDrjJlNQtVwIdXT+F9bcx/YTNJPdd0tXfFRJsxHaM=
github.com/bitrise-io/xcode-project v0.0.0-20210302080829-f3e0bfbcd5cb/go.mod h1:t9Gj5Pe/FBDTUrkFmw2lK6PbzESB6o27eE97ukZj8Rs=
github.com/bitrise-steplib/steps-authenticate-host-with-netrc v0.0.0-20230216105320-8cb845d52e28 h1:Xwe7VK9RT9QhfQ689qD2tFBK3OxeUqgeaiErOybZXGE=
github.com/bitrise-steplib/steps-authenticate-host-with-netrc v0.0.0-20230216105320-8cb845d52e28/go.mod h1:X1+WYJT3/vJpfeU5gbQwNdAq5Me0TD0ysWpAcXs01Uw=
github.com/bmatcuk/doublestar/v4 v4.2.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
Expand Down Expand Up @@ -86,6 +90,7 @@ github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.1.5-0.20170130113145-4d4bfba8f1d1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
Expand Down
12 changes: 12 additions & 0 deletions step.yml
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,18 @@ inputs:
- "yes"
- "no"

- git_http_username: $GIT_HTTP_USERNAME
opts:
title: Username for establishing an HTTP(S) connection to the repository
is_dont_change_value: true
is_sensitive: true

- git_http_password: $GIT_HTTP_PASSWORD
opts:
title: Personal access token (or password) for establishing an HTTP(S) connection to the repository
is_dont_change_value: true
is_sensitive: true

# Clone options
- clone_into_dir: $BITRISE_SOURCE_DIR
opts:
Expand Down
12 changes: 12 additions & 0 deletions step/step.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,15 @@ import (
"github.com/bitrise-steplib/steps-git-clone/gitclone"
"github.com/bitrise-steplib/steps-git-clone/gitclone/bitriseapi"
"github.com/bitrise-steplib/steps-git-clone/gitclone/tracker"
"github.com/bitrise-steplib/steps-git-clone/transport"
)

type Input struct {
ShouldMergePR bool `env:"merge_pr,opt[yes,no]"`

GitHTTPUsername string `env:"git_http_username"`
GitHTTPPassword string `env:"git_http_password"`

CloneIntoDir string `env:"clone_into_dir,required"`
CloneDepth int `env:"clone_depth"`
UpdateSubmodules bool `env:"update_submodules,opt[yes,no]"`
Expand Down Expand Up @@ -69,6 +73,14 @@ func (g GitCloneStep) ProcessConfig() (Config, error) {
}

func (g GitCloneStep) Run(cfg Config) (gitclone.CheckoutStateResult, error) {
if err := transport.Setup(transport.Config{
URL: cfg.RepositoryURL,
HTTPUsername: cfg.GitHTTPUsername,
HTTPPassword: cfg.GitHTTPPassword,
}); err != nil {
return gitclone.CheckoutStateResult{}, err
}

gitCloneCfg := convertConfig(cfg)
patchSource := bitriseapi.NewPatchSource(cfg.BuildURL, cfg.BuildAPIToken)
mergeRefChecker := bitriseapi.NewMergeRefChecker(cfg.BuildURL, cfg.BuildAPIToken, retry.NewHTTPClient(), g.logger, g.tracker)
Expand Down
82 changes: 82 additions & 0 deletions transport/transport.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
package transport

import (
"fmt"
"net/url"
"strings"
"time"

"github.com/bitrise-io/go-utils/fileutil"
"github.com/bitrise-io/go-utils/log"
"github.com/bitrise-io/go-utils/pathutil"
"github.com/bitrise-steplib/steps-authenticate-host-with-netrc/netrcutil"
)

type Config struct {
URL string
HTTPUsername string
HTTPPassword string
}

func Setup(cfg Config) error {
// We only deal with http URLs for now
if !strings.HasPrefix(cfg.URL, "http") {
return nil
}

// Setup is a no-op if no password is provided
if cfg.HTTPPassword == "" {
return nil
}

url, err := url.Parse(cfg.URL)
if err != nil {
return fmt.Errorf("failed to parse URL: %w", err)
}
host := url.Host
username := cfg.HTTPUsername
// Some providers (e.g. GitHub) doesn't care about the username, so we don't ask for it from the user
// But something still needs to be provided when making the network call
if username == "" {
username = "bitrise-git-clone-step"
}
password := cfg.HTTPPassword

netRC := netrcutil.New()

// based on https://github.com/bitrise-steplib/steps-authenticate-host-with-netrc/blob/master/main.go
netRC.AddItemModel(netrcutil.NetRCItemModel{Machine: host, Login: username, Password: password})

isExists, err := pathutil.IsPathExists(netRC.OutputPth)
if err != nil {
return fmt.Errorf("failed to check path (%s): %w", netRC.OutputPth, err)
}

if !isExists {
log.Debugf("No .netrc file found at (%s), creating new...", netRC.OutputPth)

if err := netRC.CreateFile(); err != nil {
return fmt.Errorf("failed to create .netrc file: %w", err)
}
} else {
log.Warnf(".netrc file already exists at (%s)", netRC.OutputPth)

backupPth := fmt.Sprintf("%s%s", strings.Replace(netRC.OutputPth, ".netrc", ".bk.netrc", -1), time.Now().Format("2006_01_02_15_04_05"))

if originalContent, err := fileutil.ReadBytesFromFile(netRC.OutputPth); err != nil {
return fmt.Errorf("failed to read file (%s): %w", netRC.OutputPth, err)
} else if err := fileutil.WriteBytesToFile(backupPth, originalContent); err != nil {
return fmt.Errorf("failed to write file (%s): %w", backupPth, err)
} else {
log.Warnf("Backup created at: %s", backupPth)
}

log.Debugf("Appending config to the existing .netrc file...")

if err := netRC.Append(); err != nil {
return fmt.Errorf("failed to append to .netrc file: %w", err)
}
}

return nil
}

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions vendor/modules.txt
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@ github.com/bitrise-io/go-utils/v2/retryhttp
# github.com/bitrise-io/goinp v0.0.0-20210504152833-8559b0680ab1
## explicit; go 1.16
github.com/bitrise-io/goinp/goinp
# github.com/bitrise-steplib/steps-authenticate-host-with-netrc v0.0.0-20230216105320-8cb845d52e28
## explicit; go 1.17
github.com/bitrise-steplib/steps-authenticate-host-with-netrc/netrcutil
# github.com/davecgh/go-spew v1.1.1
## explicit
github.com/davecgh/go-spew/spew
Expand Down

0 comments on commit 67b0d01

Please sign in to comment.