バックアップ機能

go.mod

@@ -25,6 +25,7 @@ require (
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.19 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.37 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.23 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.23 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect

go.sum

@@ -34,6 +34,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.44 h1:qqfs5kulLUHUEXlHEZXLJkgGoF3
 github.com/aws/aws-sdk-go-v2/credentials v1.17.44/go.mod h1:0Lm2YJ8etJdEdw23s+q/9wTpOeo2HhNE97XcRa7T8MA=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.19 h1:woXadbf0c7enQ2UGCi8gW/WuKmE0xIzxBF/eD94jMKQ=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.19/go.mod h1:zminj5ucw7w0r65bP6nhyOd3xL6veAUMc3ElGMoLVb4=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.37 h1:jHKR76E81sZvz1+x1vYYrHMxphG5LFBJPhSqEr4CLlE=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.37/go.mod h1:iMkyPkmoJWQKzSOtaX+8oEJxAuqr7s8laxcqGDSHeII=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.23 h1:A2w6m6Tmr+BNXjDsr7M90zkWjsu4JXHwrzPg235STs4=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.23/go.mod h1:35EVp9wyeANdujZruvHiQUAo9E3vbhnIO1mTCAxMlY0=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.23 h1:pgYW9FCabt2M25MoHYCfMrVY2ghiiBKYWUVXfwZs+sU=

restore/main.go

@@ -0,0 +1,166 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"cloud.google.com/go/storage"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/golang/snappy"
+	"github.com/joho/godotenv"
+	"google.golang.org/api/iterator"
+	"google.golang.org/api/option"
+)
+
+// S3設定（バケットも含む）
+type s3ConfigStruct struct {
+	Region         string
+	EndPoint       string
+	AccessKey      string
+	SecretKey      string
+	Bucket         string
+	ForcePathStyle bool
+}
+
+var s3Config s3ConfigStruct
+
+// GCP設定
+type gcpConfigStruct struct {
+	CredentialsPath string
+	ProjectID       string
+	Region          string
+	Bucket          string
+}
+
+var gcpConfig gcpConfigStruct
+
+func init() {
+	err := godotenv.Load("restore/.env")
+	if err != nil {
+		log.Fatal("Error: Failed to load .env file")
+	}
+
+	// 環境変数の読み込み
+	s3Config.EndPoint = os.Getenv("S3_ENDPOINT")
+	s3Config.Region = os.Getenv("S3_REGION")
+	s3Config.Bucket = os.Getenv("S3_BUCKET")
+	s3Config.AccessKey = os.Getenv("S3_ACCESS_KEY")
+	s3Config.SecretKey = os.Getenv("S3_SECRET_KEY")
+	s3Config.ForcePathStyle = true
+
+	gcpConfig.CredentialsPath = os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")
+	gcpConfig.ProjectID = os.Getenv("GCP_PROJECT_ID")
+	gcpConfig.Region = os.Getenv("GCS_REGION")
+	gcpConfig.Bucket = os.Getenv("GCS_BUCKET")
+}
+
+func main() {
+	// S3クライアントの作成
+	s3Credential := credentials.NewStaticCredentialsProvider(s3Config.AccessKey, s3Config.SecretKey, "")
+	cfg, err := config.LoadDefaultConfig(context.TODO(),
+		config.WithCredentialsProvider(s3Credential),
+		config.WithRegion(s3Config.Region),
+	)
+	if err != nil {
+		log.Fatalf("Error: Failed to load configuration: %v", err)
+	}
+	s3Client := s3.NewFromConfig(cfg, func(opt *s3.Options) {
+		opt.UsePathStyle = s3Config.ForcePathStyle
+		opt.BaseEndpoint = aws.String(s3Config.EndPoint)
+	})
+
+	// GCSクライアントの作成
+	ctx := context.Background()
+	gcsClient, err := storage.NewClient(ctx, option.WithCredentialsFile(gcpConfig.CredentialsPath))
+	if err != nil {
+		log.Fatalf("Error: Failed to create GCS client: %v", err)
+	}
+	defer gcsClient.Close()
+
+	// GCSバケットの取得、存在判定
+	gcsBucket := gcsClient.Bucket(gcpConfig.Bucket)
+	_, err = gcsBucket.Attrs(ctx)
+	if err != nil {
+		log.Fatalf("Error: Failed to get bucket attributes. Please check that the bucket exists: %v", err)
+	}
+
+	// バケットが存在しない場合は作成
+	_, err = s3Client.HeadBucket(ctx, &s3.HeadBucketInput{
+		Bucket: aws.String(s3Config.Bucket),
+	})
+	if err != nil {
+		_, err = s3Client.CreateBucket(ctx, &s3.CreateBucketInput{
+			Bucket: aws.String(s3Config.Bucket),
+		})
+		if err != nil {
+			log.Fatalf("Error: Failed to create bucket: %v", err)
+		}
+	}
+
+	fmt.Println("Target bucket:")
+	fmt.Printf(" - %s -> %s\n", gcpConfig.Bucket, s3Config.Bucket)
+
+	// 改行
+	fmt.Println()
+
+	// 復元計測用変数
+	//restoreStartTime := time.Now()
+
+	fmt.Println("Restoring objects: ")
+
+	// オブジェクトの取得
+	allObjects := gcsBucket.Objects(ctx, nil)
+
+	// オブジェクト数
+	totalObjects := 0
+	// エラー数
+	totalError := 0
+	// TODO: 並列処理
+	// TODO: プログレスバー表示、cheggaaa/pbをイテレーターに対して使う方法が分からない or 使えない？
+
+	for {
+		// GCSオブジェクトの取得
+		object, err := allObjects.Next()
+		if err == iterator.Done {
+			break
+		} else if err != nil {
+			log.Printf("Error: Failed to get object: %v", err)
+			totalError++
+			continue
+		}
+		totalObjects++
+		fmt.Printf(" - %s\n", object.Name)
+		gcsObjectReader, err := gcsBucket.Object(object.Name).NewReader(ctx)
+		if err != nil {
+			log.Printf("Error: Failed to get object reader: %v", err)
+			totalError++
+			continue
+		}
+
+		// snappy解凍してS3にアップロード
+		snappyReader := snappy.NewReader(gcsObjectReader)
+		s3Uploader := manager.NewUploader(s3Client)
+		_, err = s3Uploader.Upload(ctx, &s3.PutObjectInput{
+			Bucket: aws.String(s3Config.Bucket),
+			Key:    aws.String(object.Name),
+			Body:   snappyReader,
+		})
+		if err != nil {
+			log.Printf("Error: Failed to put object: %v", err)
+			totalError++
+			continue
+		}
+	}
+
+	// 復元終了
+	//restoreEndTime := time.Now()
+	//restoreDuration := restoreEndTime.Sub(restoreStartTime)
+
+	fmt.Printf("Restore completed: %d objects, %d errors\n", totalObjects, totalError)
+}

restore/sample.env

@@ -0,0 +1,11 @@
+S3_ENDPOINT=http://127.0.0.1:9000
+S3_REGION=ap-northeast-1
+S3_ACCESS_KEY=ROOT
+S3_SECRET_KEY=PASSWORD
+S3_BUCKET=traq
+S3_FORCE_PATH_STYLE=true
+
+GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials/json
+GCP_PROJECT_ID=
+GCS_REGION=asia-northeast1
+GCS_BUCKET=traq.bucket.tokyotech.org