Skip to content

Commit

Permalink
Cleaning up logging around S&D & Gdrive (#412)
Browse files Browse the repository at this point in the history
* Adding sync & delete support
  • Loading branch information
fclairamb authored Jul 27, 2021
1 parent bc2187f commit de43c5a
Show file tree
Hide file tree
Showing 11 changed files with 142 additions and 13 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@ ftpserver
.idea
.vscode
*.json
__debug__bin
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ At the current stage, supported backend are:
- [SFTP](https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol) through [afero's sftpfs](https://github.com/spf13/afero/)
- Email through [go-mail](https://github.com/go-mail/mail) thanks to [@x-way](https://github.com/x-way)

And with those are supported common parameters to switch them to read-only, enable logging access, or use a temporary directory file (see [doc](https://github.com/fclairamb/ftpserver/tree/master/fs)).

## Current status of the project

### Features
Expand Down
2 changes: 1 addition & 1 deletion config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import (
"errors"
"os"

"github.com/fclairamb/ftpserverlib/log"
log "github.com/fclairamb/go-log"

"github.com/fclairamb/ftpserver/config/confpar"
"github.com/fclairamb/ftpserver/fs"
Expand Down
94 changes: 94 additions & 0 deletions fs/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
# File system loading

On top of the supported file system and their associated config,
they are few parameters that can be enabled.

## Read-only
This blocks any writing call to the underlying file system. It can be abled by the `read_only` boolean parameter.

```json
{
"version": 1,
"accesses": [
{
"read_only": true,

// The usual FS config:
"user": "test",
"pass": "test",
"fs": "os",
"params": {
"basePath": "/tmp"
}
}
]
}
```

## Shared
This makes sure the underlying filesystem is loaded once per use. It can be enabled by the `shared` boolean parameter.

```json
{
"version": 1,
"accesses": [
{
"shared": true,
// The usual FS config:
"user": "test",
"pass": "test",
"fs": "os",
"params": {
"basePath": "/tmp"
}
}
]
}
```

## Sync & Delete
This creates a temporary local folder that will be replicated to a destination file system.


```json
{
"version": 1,
"accesses": [
{
"sync_and_delete": {
"enable": true, // To enable it
"directory": "/tmp/dir" // Temporary directory (optional)
},
// The usual FS config:
"user": "test",
"pass": "test",
"fs": "os",
"params": {
"basePath": "/target"
}
}
]
}
```

## Logging
```json
{
"version": 1,
"accesses": [
{
"logging": {
"file_accesses": true, // This logs every single file access
"ftp_exchanges": true // This logs every single FTP command
},
// The usual FS config:
"user": "test",
"pass": "test",
"fs": "os",
"params": {
"basePath": "/target"
}
}
]
}
```
5 changes: 3 additions & 2 deletions fs/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
"fmt"

snd "github.com/fclairamb/afero-snd"
"github.com/fclairamb/ftpserverlib/log"
log "github.com/fclairamb/go-log"
"github.com/spf13/afero"

"github.com/fclairamb/ftpserver/config/confpar"
Expand Down Expand Up @@ -42,7 +42,7 @@ func LoadFs(access *confpar.Access, logger log.Logger) (afero.Fs, error) {
case "mail":
fs, err = mail.LoadFs(access)
case "gdrive":
fs, err = gdrive.LoadFs(access, logger)
fs, err = gdrive.LoadFs(access, logger.With("component", "gdrive"))
case "dropbox":
fs, err = dropbox.LoadFs(access)
default:
Expand All @@ -64,6 +64,7 @@ func LoadFs(access *confpar.Access, logger log.Logger) (afero.Fs, error) {
fs, err = snd.NewFs(&snd.Config{
Destination: fs,
Temporary: temp,
Logger: logger.With("component", "snd"),
})
}

Expand Down
2 changes: 1 addition & 1 deletion fs/fslog/fslog.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (

"github.com/spf13/afero"

"github.com/fclairamb/ftpserverlib/log"
log "github.com/fclairamb/go-log"
)

// File is a wrapper to log interactions around file accesses
Expand Down
14 changes: 9 additions & 5 deletions fs/gdrive/gdrive.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (

drv "github.com/fclairamb/afero-gdrive"
drvoa "github.com/fclairamb/afero-gdrive/oauthhelper"
"github.com/fclairamb/ftpserverlib/log"
log "github.com/fclairamb/go-log"
"github.com/spf13/afero"
"golang.org/x/oauth2"

Expand All @@ -22,7 +22,6 @@ var ErrMissingGoogleClientCredentials = errors.New("missing the google client cr

// LoadFs loads a file system from an access description
func LoadFs(access *confpar.Access, logger log.Logger) (afero.Fs, error) {
logger = logger.With("fs", "gdrive")
googleClientID := access.Params["google_client_id"]
googleClientSecret := access.Params["google_client_secret"]
tokenFile := access.Params["token_file"]
Expand Down Expand Up @@ -80,10 +79,15 @@ func LoadFs(access *confpar.Access, logger log.Logger) (afero.Fs, error) {
}

if saveToken {
if err := drvoa.StoreTokenToFile(tokenFile, auth.Token); err != nil {
return nil, fmt.Errorf("token couldn't be saved: %w", err)
if errStoreToken := drvoa.StoreTokenToFile(tokenFile, auth.Token); errStoreToken != nil {
return nil, fmt.Errorf("token couldn't be saved: %w", errStoreToken)
}
}

return drv.New(httpClient)
gdriveFs, err := drv.New(httpClient)
if err == nil {
gdriveFs.Logger = logger
}

return gdriveFs, err
}
24 changes: 22 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,39 @@ module github.com/fclairamb/ftpserver
go 1.16

require (
github.com/apache/thrift v0.13.0 // indirect
github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a // indirect
github.com/aws/aws-lambda-go v1.13.3 // indirect
github.com/aws/aws-sdk-go v1.40.7
github.com/fclairamb/afero-dropbox v0.1.0
github.com/fclairamb/afero-gdrive v0.3.0
github.com/fclairamb/afero-s3 v0.3.0
github.com/fclairamb/afero-snd v0.0.0-20210725192044-a2d6c522f7ed
github.com/fclairamb/ftpserverlib v0.14.0
github.com/fclairamb/afero-snd v0.1.0
github.com/fclairamb/ftpserverlib v0.14.1-0.20210727203929-61f871ae46fc
github.com/fclairamb/go-log v0.1.0
github.com/go-mail/mail v2.3.1+incompatible
github.com/go-sql-driver/mysql v1.4.0 // indirect
github.com/hashicorp/go-version v1.2.0 // indirect
github.com/hashicorp/go.net v0.0.1 // indirect
github.com/lightstep/lightstep-tracer-go v0.18.1 // indirect
github.com/mitchellh/gox v0.4.0 // indirect
github.com/mitchellh/iochan v1.0.0 // indirect
github.com/oklog/oklog v0.3.2 // indirect
github.com/oklog/run v1.0.0 // indirect
github.com/opentracing/basictracer-go v1.0.0 // indirect
github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5 // indirect
github.com/pact-foundation/pact-go v1.0.4 // indirect
github.com/pborman/uuid v1.2.0 // indirect
github.com/pkg/sftp v1.13.2
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da // indirect
github.com/spf13/afero v1.6.0
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 // indirect
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee // indirect
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97
golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
gopkg.in/mail.v2 v2.3.1 // indirect
sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 // indirect
)

// replace github.com/fclairamb/ftpserverlib => /Users/florent/go/src/github.com/fclairamb/ftpserverlib
Expand Down
7 changes: 7 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -131,10 +131,17 @@ github.com/fclairamb/afero-s3 v0.3.0 h1:yqgJDvVY9XHetHqSrUkXy/YoNWpVvuCRX7yfC6pc
github.com/fclairamb/afero-s3 v0.3.0/go.mod h1:jOjRl8+IamjtYgZTDagbLP6+cdarjuigsm6V7cP4LZA=
github.com/fclairamb/afero-snd v0.0.0-20210725192044-a2d6c522f7ed h1:sCJUKutjhNiGMCB5ElkKScjP9Xh5TiLe9FutIUJMnJY=
github.com/fclairamb/afero-snd v0.0.0-20210725192044-a2d6c522f7ed/go.mod h1:3KupVhrtzquMmydX3IGUbhLVbW/KXrg9QonyJtmHITY=
github.com/fclairamb/afero-snd v0.1.0 h1:eQ28zAnjamm+4wxl/NlzP7LYZaSTx9fNqPHPvNLfQE0=
github.com/fclairamb/afero-snd v0.1.0/go.mod h1:3KupVhrtzquMmydX3IGUbhLVbW/KXrg9QonyJtmHITY=
github.com/fclairamb/ftpserverlib v0.14.0 h1:hF7cOVgihzmUwC4+i31iZ8MeCwK5IUipSZEDi4g6G4w=
github.com/fclairamb/ftpserverlib v0.14.0/go.mod h1:ATLgn4bHgiM9+vfZbK+rMu/dqgkxO5nk94x/9f8ffDI=
github.com/fclairamb/ftpserverlib v0.14.1-0.20210727203929-61f871ae46fc h1:U5mPcdTTchLyqeQxOtizTg74PxSkJqSpVRFyDJRi5mM=
github.com/fclairamb/ftpserverlib v0.14.1-0.20210727203929-61f871ae46fc/go.mod h1:KNoFofEUEz2PvH8vzZi+lUWZ2/NgCIdAjL0s+AMTbRQ=
github.com/fclairamb/go-log v0.0.0-20210717204555-d370617e3635 h1:LnQGTX50HKMdM3yGt5RlYN4JR8gOYuwa+qB4LkSzfuM=
github.com/fclairamb/go-log v0.0.0-20210717204555-d370617e3635/go.mod h1:iqmym8aI6xBbZXnZSPjElrmQrlEwjwEemOmIzKaTBM8=
github.com/fclairamb/go-log v0.0.0-20210725225107-80efc81cb386/go.mod h1:iqmym8aI6xBbZXnZSPjElrmQrlEwjwEemOmIzKaTBM8=
github.com/fclairamb/go-log v0.1.0 h1:fNoqk8w62i4EDEuRzDgHdDVTqMYSyr3DS981R7F2x/Y=
github.com/fclairamb/go-log v0.1.0/go.mod h1:iqmym8aI6xBbZXnZSPjElrmQrlEwjwEemOmIzKaTBM8=
github.com/fclairamb/go-log v0.0.0-20210725225107-80efc81cb386 h1:f/v/Fr+f7LiOKbMJlBuQPRmqrsYUXDzm1dpCJ29j9yo=
github.com/fclairamb/go-log v0.0.0-20210725225107-80efc81cb386/go.mod h1:iqmym8aI6xBbZXnZSPjElrmQrlEwjwEemOmIzKaTBM8=
github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
Expand Down
2 changes: 1 addition & 1 deletion main.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
"time"

ftpserver "github.com/fclairamb/ftpserverlib"
"github.com/fclairamb/ftpserverlib/log/gokit"
"github.com/fclairamb/go-log/gokit"

"github.com/fclairamb/ftpserver/config"
"github.com/fclairamb/ftpserver/server"
Expand Down
2 changes: 1 addition & 1 deletion server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
"github.com/spf13/afero"

serverlib "github.com/fclairamb/ftpserverlib"
"github.com/fclairamb/ftpserverlib/log"
log "github.com/fclairamb/go-log"

"github.com/fclairamb/ftpserver/config"
"github.com/fclairamb/ftpserver/config/confpar"
Expand Down

0 comments on commit de43c5a

Please sign in to comment.