Merge pull request #78 from peng225/refactor3

refactor3

.github/workflows/lint.yml

@@ -19,7 +19,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v3
       with:
-        go-version: "1.20"
+        go-version: "1.21"
 
     - name: Lint
       run: make lint

.github/workflows/local-run.yml

@@ -22,7 +22,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v3
       with:
-        go-version: "1.20"
+        go-version: "1.21"
 
     - name: Locally run oval with MinIO
       env:

.github/workflows/main.yml

@@ -20,7 +20,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v3
       with:
-        go-version: "1.20"
+        go-version: "1.21"
 
     - name: Build
       run: make

Dockerfile

@@ -1,5 +1,5 @@
 # Stage 1
-FROM golang:1.20 AS builder
+FROM golang:1.21 AS builder
 
 WORKDIR /go/src/github.com/
 COPY . oval

go.mod

@@ -1,12 +1,11 @@
 module github.com/peng225/oval
 
-go 1.20
+go 1.21
 
 require (
 	github.com/aws/aws-sdk-go-v2 v1.22.1
 	github.com/aws/aws-sdk-go-v2/config v1.22.0
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.42.0
-	github.com/dsnet/golib/memfile v1.0.0
 	github.com/pkg/profile v1.7.0
 	github.com/spf13/cobra v1.7.0
 	github.com/stretchr/testify v1.8.4

go.sum

@@ -41,11 +41,10 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dsnet/golib/memfile v1.0.0 h1:J9pUspY2bDCbF9o+YGwcf3uG6MdyITfh/Fk3/CaEiFs=
-github.com/dsnet/golib/memfile v1.0.0/go.mod h1:tXGNW9q3RwvWt1VV2qrRKlSSz0npnh12yftCSCy2T64=
 github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
 github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
 github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
 github.com/google/pprof v0.0.0-20221219190121-3cb0bae90811 h1:wORs2YN3R3ona/CXYuTvLM31QlgoNKHvlCNuArCDDCU=
 github.com/google/pprof v0.0.0-20221219190121-3cb0bae90811/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=

internal/argparser/argparser_test.go

@@ -35,6 +35,11 @@ func TestParseMultipartThresh(t *testing.T) {
 		},
 		{
 			multipartThreshStr:      "8g",
+			expectedMultipartThresh: 8 * 1024 * 1024 * 1024,
+			expectedErr:             false,
+		},
+		{
+			multipartThreshStr:      "5t",
 			expectedMultipartThresh: 0,
 			expectedErr:             true,
 		},

internal/argparser/follower_list.go

@@ -2,13 +2,12 @@ package argparser
 
 import (
 	"fmt"
-	"strings"
+	"slices"
 )
 
-func ParseFollowerList(followerListStr string) ([]string, error) {
-	followerList := strings.Split(followerListStr, ",")
-	if len(followerList) == 0 || followerList[0] == "" {
-		return nil, fmt.Errorf("invalid follower list format %v", followerList)
+func ValidateFollowerList(followerList []string) error {
+	if len(followerList) == 0 || slices.Contains(followerList, "") {
+		return fmt.Errorf("invalid follower list format %v", followerList)
 	}
-	return followerList, nil
+	return nil
 }

internal/argparser/size.go

@@ -38,8 +38,9 @@ func parseSizeUnit(s string) (int, error) {
 	unit := map[string]int{
 		"k": 1024,
 		"m": 1024 * 1024,
+		"g": 1024 * 1024 * 1024,
 	}
-	r := regexp.MustCompile("^[1-9][0-9]*[km]*$")
+	r := regexp.MustCompile("^[1-9][0-9]*[kmg]*$")
 	if r.MatchString(s) {
 		if size, err := strconv.Atoi(s); err == nil {
 			return size, nil
@@ -51,7 +52,7 @@ func parseSizeUnit(s string) (int, error) {
 			return baseNum * unit[s[len(s)-1:]], nil
 		}
 	}
-	return 0, fmt.Errorf("Illegal size format: %v\n", s)
+	return 0, fmt.Errorf("illegal size format: %v\n", s)
 }
 
 func ParseMultipartThresh(s string) (int, error) {

internal/cmd/leader.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"os"
 
+	"github.com/peng225/oval/internal/argparser"
 	"github.com/peng225/oval/internal/multiprocess"
 	"github.com/spf13/cobra"
 )
@@ -49,16 +50,23 @@ var leaderCmd = &cobra.Command{
 			if err != nil {
 				log.Fatal(err)
 			}
-			if len(followerList) == 0 {
-				log.Fatalf("Invalid config file: %s", configFileName)
-			}
 		}
 
-		err := multiprocess.StartFollower(followerList, execContext,
-			opeRatio, execTime.Milliseconds(), multipartThresh)
+		err := argparser.ValidateFollowerList(followerList)
 		if err != nil {
 			log.Fatal(err)
 		}
+
+		err = multiprocess.StartFollower(followerList, execContext,
+			opeRatio, execTime.Milliseconds(), multipartThresh)
+		if err != nil {
+			log.Printf("StartFollower failed. err: %v", err)
+			cancelErr := multiprocess.CancelFollowerWorkload(followerList)
+			if cancelErr != nil {
+				log.Printf("Failed to cancel followers' workload. err: %v\n", cancelErr)
+			}
+			os.Exit(1)
+		}
 		log.Println("Sent start requests to all followers.")
 
 		successAll, report, err := multiprocess.GetResultFromAllFollower(followerList)

internal/cmd/root.go

@@ -162,10 +162,10 @@ func handleCommonFlags() {
 func defineCommonFlags(cmd *cobra.Command) {
 	cmd.Flags().IntVar(&numObj, "num_obj", 10, "The maximum number of objects per process.")
 	cmd.Flags().IntVar(&numWorker, "num_worker", 1, "The number of workers per process.")
-	cmd.Flags().StringVar(&sizePattern, "size", "4k", "The size of object. Should be in the form like \"8k\" or \"4k-2m\". Only \"k\" and \"m\" is allowed as an unit.")
+	cmd.Flags().StringVar(&sizePattern, "size", "4k", `The size of object. Should be in the form like "8k" or "4k-2m". Only "k", "m" and "g" is allowed as an unit.`)
 	cmd.Flags().DurationVar(&execTime, "time", time.Second*3, "Time duration for run the workload.")
 	cmd.Flags().StringSliceVar(&bucketNames, "bucket", nil, "The name list of the buckets. e.g. \"bucket1,bucket2\"")
 	cmd.Flags().StringVar(&opeRatioStr, "ope_ratio", "1,1,1,0", "The ration of put, get, delete and list operations. e.g. \"2,3,1,1\"")
 	cmd.Flags().StringVar(&endpoint, "endpoint", "", "The endpoint URL and TCP port number. e.g. \"http://127.0.0.1:9000\"")
-	cmd.Flags().StringVar(&multipartThreshStr, "multipart_thresh", "100m", "The threshold of the object size to switch to the multipart upload. Only \"k\" and \"m\" is allowed as an unit.")
+	cmd.Flags().StringVar(&multipartThreshStr, "multipart_thresh", "100m", `The threshold of the object size to switch to the multipart upload. Only "k", "m" and "g" is allowed as an unit.`)
 }

internal/multiprocess/leader.go

@@ -91,7 +91,7 @@ func GetResultFromAllFollower(followerList []string) (bool, string, error) {
 					canceled = true
 					returnedErr = err
 					successAll = false
-					cancelErr := cancelFollowerWorkload(followerList)
+					cancelErr := CancelFollowerWorkload(followerList)
 					if cancelErr != nil {
 						log.Printf("Failed to cancel followers' workload. err: %v\n", cancelErr)
 					}
@@ -102,7 +102,7 @@ func GetResultFromAllFollower(followerList []string) (bool, string, error) {
 				if !canceled {
 					canceled = true
 					successAll = false
-					cancelErr := cancelFollowerWorkload(followerList)
+					cancelErr := CancelFollowerWorkload(followerList)
 					if cancelErr != nil {
 						log.Printf("Failed to cancel followers' workload. err: %v\n", cancelErr)
 					}
@@ -151,7 +151,7 @@ func getResultFromFollower(follower string) (bool, string, error) {
 	return (string(body) == successMessage), report, nil
 }
 
-func cancelFollowerWorkload(followerList []string) error {
+func CancelFollowerWorkload(followerList []string) error {
 	var returnedErr error
 	for _, follower := range followerList {
 		path, err := url.JoinPath(follower, "cancel")

internal/object/object.go

@@ -23,8 +23,8 @@ type Object struct {
 }
 
 type ObjectMeta struct {
-	ObjectList          []Object `json:"objectList"`
-	ExistingObjectIDs   []int64  `json:"existingObjectIDs"`
+	ObjectList          []*Object `json:"objectList"`
+	ExistingObjectIDs   []int64   `json:"existingObjectIDs"`
 	existingObjectIDMap map[int64]struct{}
 	KeyIDOffset         int64 `json:"keyIDOffset"`
 	KeyPrefix           string
@@ -47,11 +47,16 @@ func generateKey(objID int64) string {
 	return fmt.Sprintf("%s%010x", KeyShortPrefix, objID)
 }
 
+func getObjIDFromKey(key string) (int64, error) {
+	return strconv.ParseInt(key[KeyPrefixLength:], 16, 64)
+
+}
+
 func NewObjectMeta(numObj int, keyIDOffset int64) *ObjectMeta {
 	om := &ObjectMeta{}
-	om.ObjectList = make([]Object, numObj)
+	om.ObjectList = make([]*Object, numObj)
 	for objID := 0; objID < numObj; objID++ {
-		om.ObjectList[objID] = *NewObject(keyIDOffset + int64(objID))
+		om.ObjectList[objID] = NewObject(keyIDOffset + int64(objID))
 	}
 	om.ExistingObjectIDs = make([]int64, 0, int(math.Sqrt(float64(numObj))))
 	om.existingObjectIDMap = make(map[int64]struct{})
@@ -63,11 +68,11 @@ func NewObjectMeta(numObj int, keyIDOffset int64) *ObjectMeta {
 
 func (om *ObjectMeta) GetRandomObject() *Object {
 	objID := rand.Intn(len(om.ObjectList))
-	return &om.ObjectList[objID]
+	return om.ObjectList[objID]
 }
 
 func (om *ObjectMeta) RegisterToExistingList(key string) {
-	objID, err := strconv.ParseInt(key[KeyPrefixLength:], 16, 64)
+	objID, err := getObjIDFromKey(key)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -96,7 +101,7 @@ func (om *ObjectMeta) PopExistingRandomObject() *Object {
 		log.Fatalf("objID 0x%x found in ExistingObjectIDs, but not in existingObjectIDMap.", objID)
 	}
 	delete(om.existingObjectIDMap, objID)
-	return &om.ObjectList[objID]
+	return om.ObjectList[objID]
 }
 
 func (om *ObjectMeta) GetExistingRandomObject() *Object {
@@ -106,11 +111,11 @@ func (om *ObjectMeta) GetExistingRandomObject() *Object {
 	eoIDIndex := rand.Intn(len(om.ExistingObjectIDs))
 
 	objID := om.ExistingObjectIDs[eoIDIndex]
-	return &om.ObjectList[objID]
+	return om.ObjectList[objID]
 }
 
 func (om *ObjectMeta) Exist(key string) bool {
-	objID, err := strconv.ParseInt(key[KeyPrefixLength:], 16, 64)
+	objID, err := getObjIDFromKey(key)
 	if err != nil {
 		log.Fatal(err)
 	}

internal/runner/runner.go

@@ -101,12 +101,13 @@ func (r *Runner) init() {
 	}
 	if r.loadFileName == "" {
 		r.execContext.Workers = make([]Worker, r.execContext.NumWorker)
-		startID := rand.Intn(maxWorkerID)
-		r.execContext.StartWorkerID = startID
-		for i := range r.execContext.Workers {
-			r.execContext.Workers[i].id = (startID + i) % maxWorkerID
-			r.execContext.Workers[i].minSize = r.execContext.MinSize
-			r.execContext.Workers[i].maxSize = r.execContext.MaxSize
+		r.execContext.StartWorkerID = rand.Intn(maxWorkerID)
+	}
+	for i := range r.execContext.Workers {
+		r.execContext.Workers[i].id = (r.execContext.StartWorkerID + i) % maxWorkerID
+		r.execContext.Workers[i].minSize = r.execContext.MinSize
+		r.execContext.Workers[i].maxSize = r.execContext.MaxSize
+		if r.loadFileName == "" {
 			r.execContext.Workers[i].BucketsWithObject = make([]*BucketWithObject, len(r.execContext.BucketNames))
 			for j, bucketName := range r.execContext.BucketNames {
 				r.execContext.Workers[i].BucketsWithObject[j] = &BucketWithObject{
@@ -116,22 +117,14 @@ func (r *Runner) init() {
 						(int64(r.processID)<<32)+(int64(i)<<24)),
 				}
 			}
-			r.execContext.Workers[i].client = r.client
-			r.execContext.Workers[i].st = &r.st
-			r.execContext.Workers[i].ShowInfo()
-		}
-	} else {
-		for i := range r.execContext.Workers {
-			r.execContext.Workers[i].id = (r.execContext.StartWorkerID + i) % maxWorkerID
-			r.execContext.Workers[i].minSize = r.execContext.MinSize
-			r.execContext.Workers[i].maxSize = r.execContext.MaxSize
-			r.execContext.Workers[i].client = r.client
-			r.execContext.Workers[i].st = &r.st
+		} else {
 			for j := range r.execContext.Workers[i].BucketsWithObject {
 				r.execContext.Workers[i].BucketsWithObject[j].ObjectMeta.TidyUp()
 			}
-			r.execContext.Workers[i].ShowInfo()
 		}
+		r.execContext.Workers[i].client = r.client
+		r.execContext.Workers[i].st = &r.st
+		r.execContext.Workers[i].ShowInfo()
 	}
 }