Skip to content

Commit

Permalink
Fix versioning. Remove objMap.
Browse files Browse the repository at this point in the history
  • Loading branch information
@brucen1030
brucen1030 committed Feb 7, 2020
1 parent a2cfcda commit 26f0c37
Show file tree
Hide file tree
Showing 12 changed files with 149 additions and 267 deletions.
2 changes: 1 addition & 1 deletion api/object-handlers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -963,7 +963,7 @@ func (api ObjectAPIHandlers) AppendObjectHandler(w http.ResponseWriter, r *http.
return
}

if objInfo != nil && objInfo.Type != meta.ObjectTypeAppendable {
if objInfo != nil && (objInfo.Type != meta.ObjectTypeAppendable || objInfo.DeleteMarker) {
WriteErrorResponse(w, r, ErrObjectNotAppendable)
return
}
Expand Down
9 changes: 3 additions & 6 deletions meta/client/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package client

import (
"database/sql"

"github.com/journeymidnight/yig/api/datatype"
. "github.com/journeymidnight/yig/meta/types"
)
Expand All @@ -16,7 +17,7 @@ type Client interface {
GetObject(bucketName, objectName, version string) (object *Object, err error)
GetAllObject(bucketName, objectName, version string) (object []*Object, err error)
PutObject(object *Object, tx DB) error
UpdateAppendObject(object *Object, tx DB) error
UpdateAppendObject(object *Object, tx DB, versionId string) error
RenameObjectPart(object *Object, sourceObject string, tx DB) (err error)
RenameObject(object *Object, sourceObject string, tx DB) (err error)
ReplaceObjectMetas(object *Object, tx DB) (err error)
Expand All @@ -29,7 +30,7 @@ type Client interface {
PutBucket(bucket Bucket) error
CheckAndPutBucket(bucket Bucket) (bool, error)
DeleteBucket(bucket Bucket) error
ListObjects(bucketName, marker, verIdMarker, prefix, delimiter string, versioned bool, maxKeys int) (retObjects []*Object, prefixes []string, truncated bool, nextMarker, nextVerIdMarker string, err error)
ListObjects(bucketName, marker, verIdMarker, prefix, delimiter string, versioned bool, maxKeys int, withDeleteMarker bool) (retObjects []*Object, prefixes []string, truncated bool, nextMarker, nextVerIdMarker string, err error)
UpdateUsage(bucketName string, size int64, tx DB) error

//multipart
Expand All @@ -38,10 +39,6 @@ type Client interface {
PutObjectPart(multipart *Multipart, part *Part, tx DB) (err error)
DeleteMultipart(multipart *Multipart, tx DB) (err error)
ListMultipartUploads(bucketName, keyMarker, uploadIdMarker, prefix, delimiter, encodingType string, maxUploads int) (uploads []datatype.Upload, prefixs []string, isTruncated bool, nextKeyMarker, nextUploadIdMarker string, err error)
//objmap
GetObjectMap(bucketName, objectName string) (objMap *ObjMap, err error)
PutObjectMap(objMap *ObjMap, tx DB) error
DeleteObjectMap(objMap *ObjMap, tx DB) error
//cluster
GetClusters() (cluster []Cluster, err error)
//lc
Expand Down
23 changes: 10 additions & 13 deletions meta/client/tidbclient/bucket.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package tidbclient
import (
"database/sql"
"encoding/json"
"strconv"
"strings"
"time"

Expand Down Expand Up @@ -146,10 +145,7 @@ func (t *TidbClient) CheckAndPutBucket(bucket Bucket) (bool, error) {
return processed, err
}

func (t *TidbClient) ListObjects(bucketName, marker, verIdMarker, prefix, delimiter string, versioned bool, maxKeys int) (retObjects []*Object, prefixes []string, truncated bool, nextMarker, nextVerIdMarker string, err error) {
if versioned {
return
}
func (t *TidbClient) ListObjects(bucketName, marker, verIdMarker, prefix, delimiter string, versioned bool, maxKeys int, withDeleteMarker bool) (retObjects []*Object, prefixes []string, truncated bool, nextMarker, nextVerIdMarker string, err error) {
var count int
var exit bool
objectMap := make(map[string]struct{})
Expand All @@ -175,7 +171,7 @@ func (t *TidbClient) ListObjects(bucketName, marker, verIdMarker, prefix, delimi
loopcount += 1
//fetch related date
var bucketname, name string
var version uint64
var version uint64 // Internal version, the same as in DB.
var nullversion, deletemarker bool
err = rows.Scan(
&bucketname,
Expand All @@ -201,13 +197,15 @@ func (t *TidbClient) ListObjects(bucketName, marker, verIdMarker, prefix, delimi
continue
}
//filte by objectname
if _, ok := objectMap[name]; !ok {
objectMap[name] = struct{}{}
} else {
continue
if !versioned {
if _, ok := objectMap[name]; !ok {
objectMap[name] = struct{}{}
} else {
continue
}
}
//filte by deletemarker
if deletemarker {
if deletemarker && !withDeleteMarker {
continue
}
if name == omarker {
Expand Down Expand Up @@ -236,8 +234,7 @@ func (t *TidbClient) ListObjects(bucketName, marker, verIdMarker, prefix, delimi
}
}
var o *Object
Strver := strconv.FormatUint(version, 10)
o, err = t.GetObject(bucketname, name, Strver)
o, err = t.GetObject(bucketname, name, ConvertRawVersionToS3Version(version))
if err != nil {
return
}
Expand Down
53 changes: 40 additions & 13 deletions meta/client/tidbclient/object.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"time"

. "github.com/journeymidnight/yig/error"
"github.com/journeymidnight/yig/helper"
. "github.com/journeymidnight/yig/meta/types"
"github.com/xxtea/xxtea-go/xxtea"
)
Expand All @@ -23,9 +24,16 @@ func (t *TidbClient) GetObject(bucketName, objectName, version string) (object *
if version == "" {
sqltext += "order by bucketname,name,version limit 1;"
row = t.Client.QueryRow(sqltext, bucketName, objectName)
} else if version == ObjectNullVersion {
sqltext += "and nullversion=1 limit 1;" // There should be only one NullVersion object.
row = t.Client.QueryRow(sqltext, bucketName, objectName)
} else {
sqltext += "and version=?;"
row = t.Client.QueryRow(sqltext, bucketName, objectName, version)
internalVersion, err := ConvertS3VersionToRawVersion(version)
if err != nil {
return nil, ErrInternalError
}
row = t.Client.QueryRow(sqltext, bucketName, objectName, internalVersion)
}
object = &Object{}
err = row.Scan(
Expand Down Expand Up @@ -79,32 +87,44 @@ func (t *TidbClient) GetObject(bucketName, objectName, version string) (object *
}
object.PartsIndex = &SimpleIndex{Index: sortedPartNum}
}
var reversedTime uint64
timestamp := math.MaxUint64 - reversedTime
timeData := []byte(strconv.FormatUint(timestamp, 10))
object.VersionId = hex.EncodeToString(xxtea.Encrypt(timeData, XXTEA_KEY))
object.VersionId = ConvertRawVersionToS3Version(iversion)

return
}

func ConvertRawVersionToS3Version(rawVersion uint64) string {
return hex.EncodeToString(xxtea.Encrypt([]byte(strconv.FormatUint(rawVersion, 10)), XXTEA_KEY))
}

func ConvertS3VersionToRawVersion(s3Version string) (string, error) {
versionEncryped, err := hex.DecodeString(s3Version)
if err != nil {
helper.Logger.Error("Err in DecodeString()", s3Version)
return "", ErrInternalError
}

return string(xxtea.Decrypt(versionEncryped, XXTEA_KEY)), nil
}

func (t *TidbClient) GetAllObject(bucketName, objectName, version string) (object []*Object, err error) {
sqltext := "select version from objects where bucketname=? and name=?;"
var versions []string
var versions []uint64
rows, err := t.Client.Query(sqltext, bucketName, objectName)
if err != nil {
return
}
defer rows.Close()
for rows.Next() {
var sversion string
err = rows.Scan(&sversion)
var iversion uint64
err = rows.Scan(&iversion)
if err != nil {
return
}
versions = append(versions, sversion)
versions = append(versions, iversion)
}
for _, v := range versions {
var obj *Object
obj, err = t.GetObject(bucketName, objectName, v)
obj, err = t.GetObject(bucketName, objectName, ConvertRawVersionToS3Version(v))
if err != nil {
return
}
Expand Down Expand Up @@ -143,11 +163,15 @@ func (t *TidbClient) ReplaceObjectMetas(object *Object, tx DB) (err error) {
return
}

func (t *TidbClient) UpdateAppendObject(object *Object, tx DB) (err error) {
func (t *TidbClient) UpdateAppendObject(object *Object, tx DB, versionId string) (err error) {
if tx == nil {
tx = t.Client
}
sql, args := object.GetAppendSql()
rawVersionId, err := ConvertS3VersionToRawVersion(versionId)
if err != nil {
return err
}
sql, args := object.GetAppendSql(rawVersionId)
_, err = tx.Exec(sql, args...)
return err
}
Expand All @@ -167,7 +191,8 @@ func (t *TidbClient) PutObject(object *Object, tx DB) (err error) {
}
}()
}
sql, args := object.GetCreateSql()
sql, args, iversion := object.GetCreateSql()
object.VersionId = ConvertRawVersionToS3Version(iversion)
_, err = tx.Exec(sql, args...)
if object.Parts != nil {
v := math.MaxUint64 - uint64(object.LastModifiedTime.UnixNano())
Expand Down Expand Up @@ -204,11 +229,13 @@ func (t *TidbClient) DeleteObject(object *Object, tx DB) (err error) {
sqltext := "delete from objects where name=? and bucketname=? and version=?;"
_, err = tx.Exec(sqltext, object.Name, object.BucketName, version)
if err != nil {
helper.Logger.Error(object.Name, object.BucketName, version)
return err
}
sqltext = "delete from objectpart where objectname=? and bucketname=? and version=?;"
_, err = tx.Exec(sqltext, object.Name, object.BucketName, version)
if err != nil {
helper.Logger.Error(object.Name, object.BucketName, version)
return err
}
return nil
Expand Down
40 changes: 0 additions & 40 deletions meta/client/tidbclient/objmap.go
View file

This file was deleted.

36 changes: 8 additions & 28 deletions meta/object.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package meta

import (
"database/sql"

. "github.com/journeymidnight/yig/error"
"github.com/journeymidnight/yig/helper"
. "github.com/journeymidnight/yig/meta/types"
Expand Down Expand Up @@ -46,11 +47,6 @@ func (m *Meta) GetAllObject(bucketName string, objectName string) (object []*Obj
return m.Client.GetAllObject(bucketName, objectName, "")
}

func (m *Meta) GetObjectMap(bucketName, objectName string) (objMap *ObjMap, err error) {
m.Client.GetObjectMap(bucketName, objectName)
return
}

func (m *Meta) GetObjectVersion(bucketName, objectName, version string, willNeed bool) (object *Object, err error) {
getObjectVersion := func() (o interface{}, err error) {
object, err := m.Client.GetObject(bucketName, objectName, version)
Expand Down Expand Up @@ -81,7 +77,7 @@ func (m *Meta) GetObjectVersion(bucketName, objectName, version string, willNeed
return object, nil
}

func (m *Meta) PutObject(object *Object, multipart *Multipart, objMap *ObjMap, updateUsage bool) error {
func (m *Meta) PutObject(object *Object, multipart *Multipart, updateUsage bool) error {
tx, err := m.Client.NewTrans()
if err != nil {
return err
Expand All @@ -97,13 +93,6 @@ func (m *Meta) PutObject(object *Object, multipart *Multipart, objMap *ObjMap, u
return err
}

if objMap != nil {
err = m.Client.PutObjectMap(objMap, tx)
if err != nil {
return err
}
}

if multipart != nil {
err = m.Client.DeleteMultipart(multipart, tx)
if err != nil {
Expand Down Expand Up @@ -145,12 +134,7 @@ func (m *Meta) ReplaceObjectMetas(object *Object) error {
return err
}

func (m *Meta) PutObjMapEntry(objMap *ObjMap) error {
err := m.Client.PutObjectMap(objMap, nil)
return err
}

func (m *Meta) DeleteObject(object *Object, DeleteMarker bool, objMap *ObjMap) (err error) {
func (m *Meta) DeleteObject(object *Object, DeleteMarker bool) (err error) {
var tx *sql.Tx
tx, err = m.Client.NewTrans()
if err != nil {
Expand All @@ -167,20 +151,16 @@ func (m *Meta) DeleteObject(object *Object, DeleteMarker bool, objMap *ObjMap) (

err = m.Client.DeleteObject(object, tx)
if err != nil {
helper.Logger.Error(err)
return err
}

if objMap != nil {
err = m.Client.DeleteObjectMap(objMap, tx)
if err != nil {
return err
}
}

if DeleteMarker {
return nil
}

helper.Logger.Info("DeleteObject meta: gc")

err = m.Client.PutObjectToGarbageCollection(object, tx)
if err != nil {
return err
Expand All @@ -189,7 +169,7 @@ func (m *Meta) DeleteObject(object *Object, DeleteMarker bool, objMap *ObjMap) (
return m.Client.UpdateUsage(object.BucketName, -object.Size, tx)
}

func (m *Meta) AppendObject(object *Object, isExist bool) error {
func (m *Meta) AppendObject(object *Object, isExist bool, versionId string) error {
tx, err := m.Client.NewTrans()
if err != nil {
return err
Expand All @@ -202,7 +182,7 @@ func (m *Meta) AppendObject(object *Object, isExist bool) error {
if !isExist {
err = m.Client.PutObject(object, tx)
} else {
err = m.Client.UpdateAppendObject(object, tx)
err = m.Client.UpdateAppendObject(object, tx, versionId)
}
if err != nil {
return err
Expand Down
Loading

0 comments on commit 26f0c37

Please sign in to comment.