Fix versioning. Remove objMap.

.travis.yml

@@ -33,6 +33,7 @@ script:
   # use chmod to reset the permission of go.mod and go.sum to make go test happy
 - sudo chmod 644 go.mod go.sum
 - sudo python test/sanity.py
+- sudo chmod 644 go.mod go.sum
 - pushd test/go
 - go test -v
 - popd
@@ -53,7 +54,7 @@ deploy:
     file_glob: true 
     skip_cleanup: true
     on:
-      repo: journeymidnight/yig
+      repo: woclouds3/yig
       tags: true
 
   - provider: script

Makefile

@@ -26,6 +26,7 @@ build_internal:
 	cp -f delete $(PWD)/build/bin/
 	cp -f getrediskeys $(PWD)/build/bin/
 	cp -f lc $(PWD)/build/bin/
+
 pkg:
 	docker run --rm -v $(PWD):/work -w /work journeymidnight/yig bash -c 'bash package/rpmbuild.sh'
 image:

admin-server.go

@@ -55,7 +55,7 @@ func getUsage(w http.ResponseWriter, r *http.Request) {
 	claims := r.Context().Value("claims").(jwt.MapClaims)
 	bucketName := claims["bucket"].(string)
 
-	usage, err := adminServer.Yig.MetaStorage.GetUsage(bucketName)
+	usage, err := adminServer.Yig.MetaStorage.GetUsage(r.Context(), bucketName)
 	if err != nil {
 		api.WriteErrorResponse(w, r, err)
 		return
@@ -69,8 +69,8 @@ func getBucketInfo(w http.ResponseWriter, r *http.Request) {
 	claims := r.Context().Value("claims").(jwt.MapClaims)
 	bucketName := claims["bucket"].(string)
 
-	helper.Debugln("bucketName:", bucketName)
-	bucket, err := adminServer.Yig.MetaStorage.GetBucketInfo(bucketName)
+	helper.Debugln("[", api.RequestIdFromContext(r.Context()), "]", "bucketName:", bucketName)
+	bucket, err := adminServer.Yig.MetaStorage.GetBucketInfo(r.Context(), bucketName)
 	if err != nil {
 		api.WriteErrorResponse(w, r, err)
 		return
@@ -85,12 +85,12 @@ func getUserInfo(w http.ResponseWriter, r *http.Request) {
 	claims := r.Context().Value("claims").(jwt.MapClaims)
 	uid := claims["uid"].(string)
 
-	buckets, err := adminServer.Yig.MetaStorage.GetUserInfo(uid)
+	buckets, err := adminServer.Yig.MetaStorage.GetUserInfo(r.Context(), uid)
 	if err != nil {
 		api.WriteErrorResponse(w, r, err)
 		return
 	}
-	helper.Debugln("enter getUserInfo", uid, buckets)
+	helper.Debugln("[", api.RequestIdFromContext(r.Context()), "]", "enter getUserInfo", uid, buckets)
 
 	var keys []common.Credential
 	if helper.CONFIG.DebugMode == false {
@@ -106,12 +106,12 @@ func getUserInfo(w http.ResponseWriter, r *http.Request) {
 }
 
 func getObjectInfo(w http.ResponseWriter, r *http.Request) {
-	helper.Debugln("enter getObjectInfo")
+	helper.Debugln("[", api.RequestIdFromContext(r.Context()), "]", "enter getObjectInfo")
 	claims := r.Context().Value("claims").(jwt.MapClaims)
 	bucketName := claims["bucket"].(string)
 	objectName := claims["object"].(string)
 
-	object, err := adminServer.Yig.MetaStorage.GetObject(bucketName, objectName, true)
+	object, err := adminServer.Yig.MetaStorage.GetObject(r.Context(), bucketName, objectName, true)
 	if err != nil {
 		api.WriteErrorResponse(w, r, err)
 		return
@@ -122,7 +122,7 @@ func getObjectInfo(w http.ResponseWriter, r *http.Request) {
 }
 
 func getCacheHitRatio(w http.ResponseWriter, r *http.Request) {
-	helper.Debugln("enter getCacheHitRatio")
+	helper.Debugln("[", api.RequestIdFromContext(r.Context()), "]", "enter getCacheHitRatio")
 
 	rate := adminServer.Yig.MetaStorage.Cache.GetCacheHitRatio()
 	b, _ := json.Marshal(cacheJson{HitRate: rate})

api/access-log-handler.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"net/http"
+	"sync"
 	"time"
 
 	"github.com/journeymidnight/yig/helper"
@@ -40,23 +41,38 @@ type AccessLogHandler struct {
 	handler          http.Handler
 	responseRecorder *ResponseRecorder
 	format           string
+	mapPool          *sync.Pool
+	replacePool      *sync.Pool
 }
 
 func (a AccessLogHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	tstart := time.Now()
 	a.responseRecorder = NewResponseRecorder(w)
 
 	startTime := time.Now()
 	a.handler.ServeHTTP(a.responseRecorder, r)
 	finishTime := time.Now()
 	a.responseRecorder.requestTime = finishTime.Sub(startTime)
 
-	newReplacer := NewReplacer(r, a.responseRecorder, "-")
+	valueMap := a.mapPool.Get().(map[string]string)
+	newReplacer := NewReplacer(r, a.responseRecorder, "-", valueMap, a.replacePool)
+	defer func() {
+		for k := range valueMap {
+			delete(valueMap, k)
+		}
+		a.mapPool.Put(valueMap)
+	}()
 	response := newReplacer.Replace(a.format)
 
-	helper.AccessLogger.Println(5, response)
+	helper.AccessLogger.Println(20, response)
 	// send the entrys in access logger to message bus.
 	elems := newReplacer.GetReplacedValues()
 	a.notify(elems)
+	tend := time.Now()
+	dur := tend.Sub(tstart).Nanoseconds() / 1000000
+	if dur >= 100 {
+		helper.Logger.Printf(5, "slow log: access_log_handler(%s) spent %d", response, dur)
+	}
 }
 
 func (a AccessLogHandler) notify(elems map[string]string) {
@@ -99,5 +115,15 @@ func NewAccessLogHandler(handler http.Handler, _ *meta.Meta) http.Handler {
 	return AccessLogHandler{
 		handler: handler,
 		format:  CombinedLogFormat,
+		mapPool: &sync.Pool{
+			New: func() interface{} {
+				return make(map[string]string)
+			},
+		},
+		replacePool: &sync.Pool{
+			New: func() interface{} {
+				return make([]byte, 256)
+			},
+		},
 	}
 }

api/api-resources.go

@@ -17,16 +17,18 @@
 package api
 
 import (
+	"context"
 	"net/url"
 	"strconv"
 
+	"unicode/utf8"
+
 	. "github.com/journeymidnight/yig/api/datatype"
 	. "github.com/journeymidnight/yig/error"
 	"github.com/journeymidnight/yig/helper"
-	"unicode/utf8"
 )
 
-func parseListObjectsQuery(query url.Values) (request ListObjectsRequest, err error) {
+func parseListObjectsQuery(ctx context.Context, query url.Values) (request ListObjectsRequest, err error) {
 	if query.Get("list-type") == "2" {
 		request.Version = 2
 		request.ContinuationToken = query.Get("continuation-token")
@@ -60,7 +62,7 @@ func parseListObjectsQuery(query url.Values) (request ListObjectsRequest, err er
 	} else {
 		request.MaxKeys, err = strconv.Atoi(query.Get("max-keys"))
 		if err != nil {
-			helper.Debugln("Error parsing max-keys:", err)
+			helper.Debugln("[", RequestIdFromContext(ctx), "]", "Error parsing max-keys:", err)
 			return request, ErrInvalidMaxKeys
 		}
 		if request.MaxKeys > MaxObjectList || request.MaxKeys < 1 {

api/api-response.go

@@ -60,7 +60,7 @@ func GetObjectLocation(bucketName string, key string) string {
 // Takes an array of Bucket metadata information for serialization
 // input: array of bucket metadata
 // output: populated struct that can be serialized to match xml and json api spec output
-func GenerateListBucketsResponse(buckets []meta.Bucket, credential common.Credential) ListBucketsResponse {
+func GenerateListBucketsResponse(buckets []*meta.Bucket, credential common.Credential) ListBucketsResponse {
 	var listBuckets []Bucket
 	var data = ListBucketsResponse{}
 	var owner = Owner{}
@@ -224,24 +224,24 @@ func WriteSuccessNoContent(w http.ResponseWriter) {
 
 // writeErrorResponse write error headers
 func WriteErrorResponse(w http.ResponseWriter, r *http.Request, err error) {
-	WriteErrorResponseHeaders(w, err)
+	WriteErrorResponseHeaders(w, r, err)
 	WriteErrorResponseNoHeader(w, r, err, r.URL.Path)
 }
 
 func WriteErrorResponseWithResource(w http.ResponseWriter, r *http.Request, err error, resource string) {
-	WriteErrorResponseHeaders(w, err)
+	WriteErrorResponseHeaders(w, r, err)
 	WriteErrorResponseNoHeader(w, r, err, resource)
 }
 
-func WriteErrorResponseHeaders(w http.ResponseWriter, err error) {
+func WriteErrorResponseHeaders(w http.ResponseWriter, r *http.Request, err error) {
 	var status int
 	apiErrorCode, ok := err.(ApiError)
 	if ok {
 		status = apiErrorCode.HttpStatusCode()
 	} else {
 		status = http.StatusInternalServerError
 	}
-	helper.Logger.Println(5, "Response status code:", status, "err:", err)
+	helper.Logger.Println(5, "[", RequestIdFromContext(r.Context()), "]", "Response status code:", status, "err:", err)
 
 	//ResponseRecorder
 	w.(*ResponseRecorder).status = status
@@ -266,7 +266,7 @@ func WriteErrorResponseNoHeader(w http.ResponseWriter, req *http.Request, err er
 		errorResponse.Message = "We encountered an internal error, please try again."
 	}
 	errorResponse.Resource = resource
-	errorResponse.RequestId = requestIdFromContext(req.Context())
+	errorResponse.RequestId = RequestIdFromContext(req.Context())
 	errorResponse.HostId = helper.CONFIG.InstanceId
 
 	encodedErrorResponse := EncodeResponse(errorResponse)

api/auth-handler.go

@@ -23,16 +23,16 @@ func checkRequestAuth(api ObjectAPIHandlers, r *http.Request, action policy.Acti
 	authType := signature.GetRequestAuthType(r)
 	switch authType {
 	case signature.AuthTypeUnknown:
-		helper.Logger.Println(5, "ErrAccessDenied: AuthTypeUnknown")
+		helper.Logger.Println(5, "[", RequestIdFromContext(r.Context()), "]", "ErrAccessDenied: AuthTypeUnknown")
 		return c, ErrAccessDenied
 	case signature.AuthTypeSignedV4, signature.AuthTypePresignedV4,
 		signature.AuthTypePresignedV2, signature.AuthTypeSignedV2:
-		helper.Logger.Println(5, "AuthTypeSigned:", authType)
+		helper.Logger.Println(5, "[", RequestIdFromContext(r.Context()), "]", "AuthTypeSigned:", authType)
 		if c, err := signature.IsReqAuthenticated(r); err != nil {
-			helper.Logger.Println(5, "ErrAccessDenied: IsReqAuthenticated return false:", err)
+			helper.Logger.Println(5, "[", RequestIdFromContext(r.Context()), "]", "ErrAccessDenied: IsReqAuthenticated return false:", err)
 			return c, err
 		} else {
-			helper.Logger.Println(5, "Credential:", c)
+			helper.Logger.Println(5, "[", RequestIdFromContext(r.Context()), "]", "Credential:", c)
 			// check bucket policy
 			c, err = IsBucketPolicyAllowed(c, api, r, action, bucketName, objectName)
 			return c, err
@@ -45,16 +45,16 @@ func checkRequestAuth(api ObjectAPIHandlers, r *http.Request, action policy.Acti
 }
 
 func IsBucketPolicyAllowed(c common.Credential, api ObjectAPIHandlers, r *http.Request, action policy.Action, bucketName, objectName string) (common.Credential, error) {
-	bucket, err := api.ObjectAPI.GetBucket(bucketName)
+	bucket, err := api.ObjectAPI.GetBucket(r.Context(), bucketName)
 	if err != nil {
-		helper.Logger.Println(5, "GetBucket", bucketName, "err:", err)
+		helper.Logger.Println(5, "[", RequestIdFromContext(r.Context()), "]", "GetBucket", bucketName, "err:", err)
 		return c, err
 	}
 	if bucket.OwnerId == c.UserId {
 		return c, nil
 	}
-	helper.Logger.Println(5, "bucket.OwnerId:", bucket.OwnerId, "not equals c.UserId:", c.UserId)
-	helper.Debugln("GetBucketPolicy:", bucket.Policy)
+	helper.Logger.Println(5, "[", RequestIdFromContext(r.Context()), "]", "bucket.OwnerId:", bucket.OwnerId, "not equals c.UserId:", c.UserId)
+	helper.Debugln("[", RequestIdFromContext(r.Context()), "]", "GetBucketPolicy:", bucket.Policy)
 	policyResult := bucket.Policy.IsAllowed(policy.Args{
 		AccountName:     c.UserId,
 		Action:          action,
@@ -65,10 +65,12 @@ func IsBucketPolicyAllowed(c common.Credential, api ObjectAPIHandlers, r *http.R
 	})
 	if policyResult == policy.PolicyAllow {
 		c.AllowOtherUserAccess = true
-		helper.Debugln("Allow", c.UserId, "access", bucketName, "with", action, objectName)
+		helper.Debugln("[", RequestIdFromContext(r.Context()), "]",
+			"Allow", c.UserId, "access", bucketName, "with", action, objectName)
 		return c, nil
 	} else if policyResult == policy.PolicyDeny {
-		helper.Debugln("ErrAccessDenied: NotAllow", c.UserId, "access", bucketName, "with", action, objectName)
+		helper.Debugln("[", RequestIdFromContext(r.Context()), "]",
+			"ErrAccessDenied: NotAllow", c.UserId, "access", bucketName, "with", action, objectName)
 		return c, ErrAccessDenied
 	} else {
 		return c, nil