Skip to content

Commit

Permalink
メタデータが欠落する問題を修正
Browse files Browse the repository at this point in the history
  • Loading branch information
Kentaro1043 committed Nov 14, 2024
1 parent bee2fa9 commit 25075d5
Show file tree
Hide file tree
Showing 4 changed files with 53 additions and 4 deletions.
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@
```go
go run restore/main.go
```
`GCS_BUCKET`から`S3_BUCKET`に復元されます。
`GCS_BUCKET`から`S3_BUCKET`に復元されます。
メタデータを修復するため、DBに接続する必要があります。

## 単一ファイル復元

Expand Down
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ require (
cloud.google.com/go/iam v1.2.1 // indirect
cloud.google.com/go/monitoring v1.21.1 // indirect
cloud.google.com/go/storage v1.46.0 // indirect
filippo.io/edwards25519 v1.1.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.1 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect
Expand Down Expand Up @@ -48,6 +49,7 @@ require (
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-sql-driver/mysql v1.8.1 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/s2a-go v0.1.8 // indirect
Expand Down
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ cloud.google.com/go/monitoring v1.21.1 h1:zWtbIoBMnU5LP9A/fz8LmWMGHpk4skdfeiaa66
cloud.google.com/go/monitoring v1.21.1/go.mod h1:Rj++LKrlht9uBi8+Eb530dIrzG/cU/lB8mt+lbeFK1c=
cloud.google.com/go/storage v1.46.0 h1:OTXISBpFd8KaA2ClT3K3oRk8UGOcTHtrZ1bW88xKiic=
cloud.google.com/go/storage v1.46.0/go.mod h1:lM+gMAW91EfXIeMTBmixRsKL/XCxysytoAgduVikjMk=
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.1 h1:pB2F2JKCj1Znmp2rwxxt1J0Fg0wezTMgWYk5Mpbi1kg=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.1/go.mod h1:itPGVDKf9cC/ov4MdvJ2QZ0khw4bfoo9jzwTJlaxy2k=
Expand Down Expand Up @@ -92,6 +94,8 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
Expand Down
48 changes: 45 additions & 3 deletions restore/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package main

import (
"context"
"database/sql"
"fmt"
"log"
"os"
Expand All @@ -12,6 +13,7 @@ import (
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
_ "github.com/go-sql-driver/mysql"
"github.com/golang/snappy"
"github.com/joho/godotenv"
"google.golang.org/api/iterator"
Expand Down Expand Up @@ -40,6 +42,17 @@ type gcpConfigStruct struct {

var gcpConfig gcpConfigStruct

// DB設定
type dbConfigStruct struct {
Host string
Port string
User string
Password string
database string
}

var dbConfig dbConfigStruct

func init() {
err := godotenv.Load("restore/.env")
if err != nil {
Expand All @@ -58,6 +71,13 @@ func init() {
gcpConfig.ProjectID = os.Getenv("GCP_PROJECT_ID")
gcpConfig.Region = os.Getenv("GCS_REGION")
gcpConfig.Bucket = os.Getenv("GCS_BUCKET")

dbConfig.Host = os.Getenv("MYSQL_HOST")
dbConfig.Port = os.Getenv("MYSQL_PORT")
dbConfig.User = os.Getenv("MYSQL_USER")
dbConfig.Password = os.Getenv("MYSQL_PASSWORD")
dbConfig.database = os.Getenv("MYSQL_DATABASE")

}

func main() {
Expand All @@ -83,6 +103,16 @@ func main() {
}
defer gcsClient.Close()

// DB接続
db, err := sql.Open("mysql", dbConfig.User+":"+dbConfig.Password+"@tcp("+dbConfig.Host+":"+dbConfig.Port+")/"+dbConfig.database)
if err != nil {
log.Fatalf("Error: Failed to connect to database: %v", err)
}
defer db.Close()
if err = db.Ping(); err != nil {
log.Fatalf("Error: Failed to ping database: %v", err)
}

// GCSバケットの取得、存在判定
gcsBucket := gcsClient.Bucket(gcpConfig.Bucket)
_, err = gcsBucket.Attrs(ctx)
Expand Down Expand Up @@ -143,13 +173,25 @@ func main() {
continue
}

// ファイルのデータをDBから取得
var fileName string
var fileMime string

if err := db.QueryRow("SELECT name,mime FROM files WHERE id=?", object.Name).Scan(&fileName, &fileMime); err != nil {
log.Printf("Error: Failed to get file data: %v", err)
totalError++
continue
}

// snappy解凍してS3にアップロード
snappyReader := snappy.NewReader(gcsObjectReader)
s3Uploader := manager.NewUploader(s3Client)
_, err = s3Uploader.Upload(ctx, &s3.PutObjectInput{
Bucket: aws.String(s3Config.Bucket),
Key: aws.String(object.Name),
Body: snappyReader,
Bucket: aws.String(s3Config.Bucket),
Key: aws.String(object.Name),
Body: snappyReader,
ContentType: aws.String(fileMime),
ContentDisposition: aws.String(fmt.Sprintf("attachment; filename*=UTF-8''%s", fileName)),
})
if err != nil {
log.Printf("Error: Failed to put object: %v", err)
Expand Down

0 comments on commit 25075d5

Please sign in to comment.