Open AWS storage to being used with non-AWS MySQL and S3 services

cmd/conformance/aws/main.go

@@ -24,6 +24,9 @@ import (
 	"net/http"
 	"time"
 
+	aaws "github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	tessera "github.com/transparency-dev/trillian-tessera"
 	"github.com/transparency-dev/trillian-tessera/storage/aws"
 	"golang.org/x/mod/sumdb/note"
@@ -42,6 +45,10 @@ var (
 	dbPassword        = flag.String("db_password", "", "AuroraDB user")
 	dbMaxConns        = flag.Int("db_max_conns", 0, "Maximum connections to the database, defaults to 0, i.e unlimited")
 	dbMaxIdle         = flag.Int("db_max_idle_conns", 2, "Maximum idle database connections in the connection pool, defaults to 2")
+	s3Endpoint        = flag.String("s3_endpoint", "", "Endpoint for custom non-AWS S3 service")
+	s3AccessKeyID     = flag.String("s3_access_key", "", "Access key ID for custom non-AWS S3 service")
+	s3SecretAccessKey = flag.String("s3_secret", "", "Secret access key for custom non-AWS S3 service")
+	s3UseSSL          = flag.Bool("s3_use_ssl", false, "Whether to use SSL for custom non-AWS S3 service")
 	signer            = flag.String("signer", "", "Note signer to use to sign checkpoints")
 	publishInterval   = flag.Duration("publish_interval", 3*time.Second, "How frequently to publish updated checkpoints")
 	additionalSigners = []string{}
@@ -138,8 +145,27 @@ func storageConfigFromFlags() aws.Config {
 		*dbUser, *dbPassword, dbEndpoint, *dbName,
 	)
 
+	// Configure to use MinIO Server
+	var awsConfig *aaws.Config
+	var s3Opts func(o *s3.Options)
+	if *s3Endpoint != "" {
+		const defaultRegion = "us-east-1"
+		s3Opts = func(o *s3.Options) {
+			o.BaseEndpoint = aaws.String(*s3Endpoint)
+			o.Credentials = credentials.NewStaticCredentialsProvider(*s3AccessKeyID, *s3SecretAccessKey, "")
+			o.Region = defaultRegion
+			o.UsePathStyle = true
+		}
+
+		awsConfig = &aaws.Config{
+			Region: defaultRegion,
+		}
+	}
+
 	return aws.Config{
 		Bucket:       *bucket,
+		SDKConfig:    awsConfig,
+		S3Options:    s3Opts,
 		DSN:          dsn,
 		MaxOpenConns: *dbMaxConns,
 		MaxIdleConns: *dbMaxIdle,

storage/aws/README.md

@@ -58,6 +58,22 @@ Two experimental implementations have been tested which uses either Aurora MySQL
 or a local bbolt database to store the `<identity_hash>` --> `sequence` mapping.
 They work well, but call for further stress testing and cost analysis.
 
+## Compatibility
+
+This storage implementation is intended to be used with AWS services.
+
+However, given that it's based on services which are compatible with MySQL and
+S3 protocols, it's possible that it will work with other non-AWS-based backends
+which are compatible with these protocols.
+
+Given the vast array of combinations of backend implementations and versions, we
+can't offer anything more than a "best effort" level of support for using this storage
+implementation outside of AWS.
+
+Similarly, PRs raised against it relating to its use outside of AWS are unlikely to 
+be accepted unless it's shown that they have no detremental effect to the implementation's
+performance on AWS.
+
 ### Alternatives considered
 
 Other transactional storage systems are available on AWS, e.g. Redshift, RDS or

storage/aws/aws.go

@@ -110,6 +110,11 @@ type consumeFunc func(ctx context.Context, from uint64, entries []storage.Sequen
 
 // Config holds AWS project and resource configuration for a storage instance.
 type Config struct {
+	// SDKConfig is an optional AWS config to use when configuring service clients, e.g. S3.
+	// If nil, the value from config.LoadDefaultConfig() will be used.
+	SDKConfig *aws.Config
+	// S3Options is an optional function which can be used to configure the S3 library.
+	S3Options func(*s3.Options)
 	// Bucket is the name of the S3 bucket to use for storing log state.
 	Bucket string
 	// DSN is the DSN of the MySQL instance to use.
@@ -133,11 +138,14 @@ func New(ctx context.Context, cfg Config, opts ...func(*options.StorageOptions))
 		return nil, fmt.Errorf("requested CheckpointInterval (%v) is less than minimum permitted %v", opt.CheckpointInterval, minCheckpointInterval)
 	}
 
-	sdkConfig, err := config.LoadDefaultConfig(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("failed to load default AWS configuration: %v", err)
+	if cfg.SDKConfig == nil {
+		sdkConfig, err := config.LoadDefaultConfig(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("failed to load default AWS configuration: %v", err)
+		}
+		cfg.SDKConfig = &sdkConfig
 	}
-	c := s3.NewFromConfig(sdkConfig)
+	c := s3.NewFromConfig(*cfg.SDKConfig, cfg.S3Options)
 
 	seq, err := newMySQLSequencer(ctx, cfg.DSN, uint64(opt.PushbackMaxOutstanding), cfg.MaxOpenConns, cfg.MaxIdleConns)
 	if err != nil {