Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: configurable serve as proxy #10

Closed
wants to merge 4 commits into from
Closed

feat: configurable serve as proxy #10

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
fd97c66
feat: allow enable/disable webdav-proxy with cli arg
saw-jan Aug 1, 2023
608a73e
rename setAuthForWebDAV to getVfs
saw-jan Jan 10, 2024
2278a30
raise Exception if the provider is not webdav when using proxy mode
saw-jan Jan 10, 2024
0a79dff
fix tests
saw-jan Jan 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 25 additions & 22 deletions cmd/serve/s3/backend.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,21 +40,24 @@ func newBackend(opt *Options, w *Server) gofakes3.Backend {
}
}

func (b *s3Backend) setAuthForWebDAV(accessKey string) *vfs.VFS {
// new VFS
if _, ok := b.w.f.(*webdav.Fs); ok {
info, name, remote, config, _ := fs.ConfigFs(b.w.f.Name() + ":")
f, _ := info.NewFs(context.Background(), name+accessKey, remote, config)
vf := vfs.New(f, &vfsflags.Opt)
vf.Fs().(*webdav.Fs).SetBearerToken(accessKey)
return vf
}
return vfs.New(b.w.f, &vfsflags.Opt)
func (b *s3Backend) getVfs(accessKey string) *vfs.VFS {
if b.opt.proxyMode {
// new VFS
if _, ok := b.w.f.(*webdav.Fs); ok {
info, name, remote, config, _ := fs.ConfigFs(b.w.f.Name() + ":")
f, _ := info.NewFs(context.Background(), name+accessKey, remote, config)
vf := vfs.New(f, &vfsflags.Opt)
vf.Fs().(*webdav.Fs).SetBearerToken(accessKey)
return vf
}
return vfs.New(b.w.f, &vfsflags.Opt)
}
return b.w.vfs
}

// ListBuckets always returns the default bucket.
func (b *s3Backend) ListBuckets(accessKey string) ([]gofakes3.BucketInfo, error) {
vf := b.setAuthForWebDAV(accessKey)
vf := b.getVfs(accessKey)

dirEntries, err := getDirEntries("/", vf)
if err != nil {
Expand All @@ -76,7 +79,7 @@ func (b *s3Backend) ListBuckets(accessKey string) ([]gofakes3.BucketInfo, error)

// ListBucket lists the objects in the given bucket.
func (b *s3Backend) ListBucket(accessKey string, bucket string, prefix *gofakes3.Prefix, page gofakes3.ListBucketPage) (*gofakes3.ObjectList, error) {
vf := b.setAuthForWebDAV(accessKey)
vf := b.getVfs(accessKey)

_, err := vf.Stat(bucket)
if err != nil {
Expand Down Expand Up @@ -112,7 +115,7 @@ func (b *s3Backend) ListBucket(accessKey string, bucket string, prefix *gofakes3
//
// Note that the metadata is not supported yet.
func (b *s3Backend) HeadObject(accessKey string, bucketName, objectName string) (*gofakes3.Object, error) {
vf := b.setAuthForWebDAV(accessKey)
vf := b.getVfs(accessKey)

_, err := vf.Stat(bucketName)
if err != nil {
Expand Down Expand Up @@ -161,7 +164,7 @@ func (b *s3Backend) HeadObject(accessKey string, bucketName, objectName string)

// GetObject fetchs the object from the filesystem.
func (b *s3Backend) GetObject(accessKey string, bucketName, objectName string, rangeRequest *gofakes3.ObjectRangeRequest) (obj *gofakes3.Object, err error) {
vf := b.setAuthForWebDAV(accessKey)
vf := b.getVfs(accessKey)

_, err = vf.Stat(bucketName)
if err != nil {
Expand Down Expand Up @@ -237,7 +240,7 @@ func (b *s3Backend) GetObject(accessKey string, bucketName, objectName string, r

// TouchObject creates or updates meta on specified object.
func (b *s3Backend) TouchObject(accessKey string, fp string, meta map[string]string) (result gofakes3.PutObjectResult, err error) {
vf := b.setAuthForWebDAV(accessKey)
vf := b.getVfs(accessKey)

_, err = vf.Stat(fp)
if err == vfs.ENOENT {
Expand Down Expand Up @@ -284,7 +287,7 @@ func (b *s3Backend) PutObject(
meta map[string]string,
input io.Reader, size int64,
) (result gofakes3.PutObjectResult, err error) {
vf := b.setAuthForWebDAV(accessKey)
vf := b.getVfs(accessKey)

_, err = vf.Stat(bucketName)
if err != nil {
Expand Down Expand Up @@ -351,7 +354,7 @@ func (b *s3Backend) PutObject(

// DeleteMulti deletes multiple objects in a single request.
func (b *s3Backend) DeleteMulti(accessKey string, bucketName string, objects ...string) (result gofakes3.MultiDeleteResult, rerr error) {
vf := b.setAuthForWebDAV(accessKey)
vf := b.getVfs(accessKey)

for _, object := range objects {
if err := b.deleteObject(vf, bucketName, object); err != nil {
Expand All @@ -373,7 +376,7 @@ func (b *s3Backend) DeleteMulti(accessKey string, bucketName string, objects ...

// DeleteObject deletes the object with the given name.
func (b *s3Backend) DeleteObject(accessKey string, bucketName, objectName string) (result gofakes3.ObjectDeleteResult, rerr error) {
vf := b.setAuthForWebDAV(accessKey)
vf := b.getVfs(accessKey)

return result, b.deleteObject(vf, bucketName, objectName)
}
Expand All @@ -399,7 +402,7 @@ func (b *s3Backend) deleteObject(vf *vfs.VFS, bucketName, objectName string) err

// CreateBucket creates a new bucket.
func (b *s3Backend) CreateBucket(accessKey string, name string) error {
vf := b.setAuthForWebDAV(accessKey)
vf := b.getVfs(accessKey)

_, err := vf.Stat(name)
if err != nil && err != vfs.ENOENT {
Expand All @@ -418,7 +421,7 @@ func (b *s3Backend) CreateBucket(accessKey string, name string) error {

// DeleteBucket deletes the bucket with the given name.
func (b *s3Backend) DeleteBucket(accessKey string, name string) error {
vf := b.setAuthForWebDAV(accessKey)
vf := b.getVfs(accessKey)

_, err := vf.Stat(name)
if err != nil {
Expand All @@ -434,7 +437,7 @@ func (b *s3Backend) DeleteBucket(accessKey string, name string) error {

// BucketExists checks if the bucket exists.
func (b *s3Backend) BucketExists(accessKey string, name string) (exists bool, err error) {
vf := b.setAuthForWebDAV(accessKey)
vf := b.getVfs(accessKey)

_, err = vf.Stat(name)
if err != nil {
Expand All @@ -446,7 +449,7 @@ func (b *s3Backend) BucketExists(accessKey string, name string) (exists bool, er

// CopyObject copy specified object from srcKey to dstKey.
func (b *s3Backend) CopyObject(accessKey string, srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (result gofakes3.CopyObjectResult, err error) {
vf := b.setAuthForWebDAV(accessKey)
vf := b.getVfs(accessKey)

fp := path.Join(srcBucket, srcKey)
if srcBucket == dstBucket && srcKey == dstKey {
Expand Down
9 changes: 9 additions & 0 deletions cmd/serve/s3/s3.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ import (
"context"
_ "embed"

"github.com/rclone/rclone/backend/webdav"
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/hash"
httplib "github.com/rclone/rclone/lib/http"
Expand All @@ -19,6 +21,7 @@ var DefaultOpt = Options{
hashName: "MD5",
hashType: hash.MD5,
noCleanup: false,
proxyMode: false,
HTTP: httplib.DefaultCfg(),
}

Expand All @@ -35,6 +38,7 @@ func init() {
flags.StringVarP(flagSet, &Opt.hashName, "etag-hash", "", Opt.hashName, "Which hash to use for the ETag, or auto or blank for off", "")
flags.StringArrayVarP(flagSet, &Opt.authPair, "auth-key", "", Opt.authPair, "Set key pair for v4 authorization: access_key_id,secret_access_key", "")
flags.BoolVarP(flagSet, &Opt.noCleanup, "no-cleanup", "", Opt.noCleanup, "Not to cleanup empty folder after object is deleted", "")
flags.BoolVarP(flagSet, &Opt.proxyMode, "proxy-mode", "", Opt.proxyMode, "Serve as a proxy for s3 clients", "")
}

//go:embed serve_s3.md
Expand All @@ -54,6 +58,11 @@ var Command = &cobra.Command{
cmd.CheckArgs(1, 1, command, args)
f := cmd.NewFsSrc(args)

if _, ok := f.(*webdav.Fs); !ok && Opt.proxyMode {
fs.Logf("serve s3", "Proxy mode is not supported for %s provider", f.Name())
return fs.ErrorNotImplemented
}

if Opt.hashName == "auto" {
Opt.hashType = f.Hashes().GetOne()
} else if Opt.hashName != "" {
Expand Down
11 changes: 6 additions & 5 deletions cmd/serve/s3/s3_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,13 +167,14 @@ const (
)

// Configure and serve the server
func serveS3(f fs.Fs, keyid string, keysec string) (testURL string) {
func serveS3(f fs.Fs, keyid string, keysec string, proxyMode bool) (testURL string) {
serveropt := &Options{
HTTP: httplib.DefaultCfg(),
pathBucketMode: true,
hashName: "",
hashType: hash.None,
}
serveropt.proxyMode = proxyMode
if keyid != "" && keysec != "" {
serveropt.authPair = []string{fmt.Sprintf("%s,%s", keyid, keysec)}
}
Expand Down Expand Up @@ -206,7 +207,7 @@ func TestS3(t *testing.T) {
start := func(f fs.Fs) (configmap.Simple, func()) {
keyid := RandString(16)
keysec := RandString(16)
testURL := serveS3(f, keyid, keysec)
testURL := serveS3(f, keyid, keysec, false)
// Config for the backend we'll use to connect to the server
config := configmap.Simple{
"type": "s3",
Expand Down Expand Up @@ -310,7 +311,7 @@ func TestForwardAccessKeyToWebDav(t *testing.T) {
})
f, clean := prepareWebDavFs(t, handler)
defer clean()
endpoint := serveS3(f, keyid, keysec)
endpoint := serveS3(f, keyid, keysec, true)
testURL, _ := url.Parse(endpoint)
minioClient, err := minio.New(testURL.Host, &minio.Options{
Creds: credentials.NewStaticV4(keyid, keysec, ""),
Expand Down Expand Up @@ -365,7 +366,7 @@ func TestForwardAccessKeyToWebDavParallelRequests(t *testing.T) {
var wg sync.WaitGroup
wg.Add(len(keyids))

endpoint := serveS3(f, "", "")
endpoint := serveS3(f, "", "", true)
testURL, _ := url.Parse(endpoint)

responseChannel := make(chan error)
Expand Down Expand Up @@ -455,7 +456,7 @@ func TestEncodingWithMinioClient(t *testing.T) {
assert.NoError(t, err)
keyid := RandString(16)
keysec := RandString(16)
endpoint := serveS3(f, keyid, keysec)
endpoint := serveS3(f, keyid, keysec, false)
testURL, _ := url.Parse(endpoint)
minioClient, err := minio.New(testURL.Host, &minio.Options{
Creds: credentials.NewStaticV4(keyid, keysec, ""),
Expand Down
1 change: 1 addition & 0 deletions cmd/serve/s3/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ type Options struct {
hashType hash.Type
authPair []string
noCleanup bool
proxyMode bool
HTTP httplib.Config
}

Expand Down