diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 7ba47af..f53cf9f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: "1.20" + go-version: "1.21" - name: Lint run: make lint diff --git a/.github/workflows/local-run.yml b/.github/workflows/local-run.yml index 600140b..223c48e 100644 --- a/.github/workflows/local-run.yml +++ b/.github/workflows/local-run.yml @@ -22,7 +22,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: "1.20" + go-version: "1.21" - name: Locally run oval with MinIO env: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 540625b..8cbdc77 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -20,7 +20,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: "1.20" + go-version: "1.21" - name: Build run: make diff --git a/Dockerfile b/Dockerfile index 37e9603..3669709 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Stage 1 -FROM golang:1.20 AS builder +FROM golang:1.21 AS builder WORKDIR /go/src/github.com/ COPY . oval diff --git a/go.mod b/go.mod index 31f3124..945e17b 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,11 @@ module github.com/peng225/oval -go 1.20 +go 1.21 require ( github.com/aws/aws-sdk-go-v2 v1.22.1 github.com/aws/aws-sdk-go-v2/config v1.22.0 github.com/aws/aws-sdk-go-v2/service/s3 v1.42.0 - github.com/dsnet/golib/memfile v1.0.0 github.com/pkg/profile v1.7.0 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.4 diff --git a/go.sum b/go.sum index 1fde89f..7e67a87 100644 --- a/go.sum +++ b/go.sum @@ -41,11 +41,10 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dsnet/golib/memfile v1.0.0 h1:J9pUspY2bDCbF9o+YGwcf3uG6MdyITfh/Fk3/CaEiFs= -github.com/dsnet/golib/memfile v1.0.0/go.mod h1:tXGNW9q3RwvWt1VV2qrRKlSSz0npnh12yftCSCy2T64= github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/pprof v0.0.0-20221219190121-3cb0bae90811 h1:wORs2YN3R3ona/CXYuTvLM31QlgoNKHvlCNuArCDDCU= github.com/google/pprof v0.0.0-20221219190121-3cb0bae90811/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= diff --git a/internal/argparser/argparser_test.go b/internal/argparser/argparser_test.go index 7ca9f86..a31e41b 100644 --- a/internal/argparser/argparser_test.go +++ b/internal/argparser/argparser_test.go @@ -35,6 +35,11 @@ func TestParseMultipartThresh(t *testing.T) { }, { multipartThreshStr: "8g", + expectedMultipartThresh: 8 * 1024 * 1024 * 1024, + expectedErr: false, + }, + { + multipartThreshStr: "5t", expectedMultipartThresh: 0, expectedErr: true, }, diff --git a/internal/argparser/follower_list.go b/internal/argparser/follower_list.go index 661c8d4..dc911e9 100644 --- a/internal/argparser/follower_list.go +++ b/internal/argparser/follower_list.go @@ -2,13 +2,12 @@ package argparser import ( "fmt" - "strings" + "slices" ) -func ParseFollowerList(followerListStr string) ([]string, error) { - followerList := strings.Split(followerListStr, ",") - if len(followerList) == 0 || followerList[0] == "" { - return nil, fmt.Errorf("invalid follower list format %v", followerList) +func ValidateFollowerList(followerList []string) error { + if len(followerList) == 0 || slices.Contains(followerList, "") { + return fmt.Errorf("invalid follower list format %v", followerList) } - return followerList, nil + return nil } diff --git a/internal/argparser/size.go b/internal/argparser/size.go index 03ebeac..b919476 100644 --- a/internal/argparser/size.go +++ b/internal/argparser/size.go @@ -38,8 +38,9 @@ func parseSizeUnit(s string) (int, error) { unit := map[string]int{ "k": 1024, "m": 1024 * 1024, + "g": 1024 * 1024 * 1024, } - r := regexp.MustCompile("^[1-9][0-9]*[km]*$") + r := regexp.MustCompile("^[1-9][0-9]*[kmg]*$") if r.MatchString(s) { if size, err := strconv.Atoi(s); err == nil { return size, nil @@ -51,7 +52,7 @@ func parseSizeUnit(s string) (int, error) { return baseNum * unit[s[len(s)-1:]], nil } } - return 0, fmt.Errorf("Illegal size format: %v\n", s) + return 0, fmt.Errorf("illegal size format: %v\n", s) } func ParseMultipartThresh(s string) (int, error) { diff --git a/internal/cmd/leader.go b/internal/cmd/leader.go index 3adeca4..1547a6c 100644 --- a/internal/cmd/leader.go +++ b/internal/cmd/leader.go @@ -6,6 +6,7 @@ import ( "log" "os" + "github.com/peng225/oval/internal/argparser" "github.com/peng225/oval/internal/multiprocess" "github.com/spf13/cobra" ) @@ -49,16 +50,23 @@ var leaderCmd = &cobra.Command{ if err != nil { log.Fatal(err) } - if len(followerList) == 0 { - log.Fatalf("Invalid config file: %s", configFileName) - } } - err := multiprocess.StartFollower(followerList, execContext, - opeRatio, execTime.Milliseconds(), multipartThresh) + err := argparser.ValidateFollowerList(followerList) if err != nil { log.Fatal(err) } + + err = multiprocess.StartFollower(followerList, execContext, + opeRatio, execTime.Milliseconds(), multipartThresh) + if err != nil { + log.Printf("StartFollower failed. err: %v", err) + cancelErr := multiprocess.CancelFollowerWorkload(followerList) + if cancelErr != nil { + log.Printf("Failed to cancel followers' workload. err: %v\n", cancelErr) + } + os.Exit(1) + } log.Println("Sent start requests to all followers.") successAll, report, err := multiprocess.GetResultFromAllFollower(followerList) diff --git a/internal/cmd/root.go b/internal/cmd/root.go index 74fb372..9fba67e 100644 --- a/internal/cmd/root.go +++ b/internal/cmd/root.go @@ -162,10 +162,10 @@ func handleCommonFlags() { func defineCommonFlags(cmd *cobra.Command) { cmd.Flags().IntVar(&numObj, "num_obj", 10, "The maximum number of objects per process.") cmd.Flags().IntVar(&numWorker, "num_worker", 1, "The number of workers per process.") - cmd.Flags().StringVar(&sizePattern, "size", "4k", "The size of object. Should be in the form like \"8k\" or \"4k-2m\". Only \"k\" and \"m\" is allowed as an unit.") + cmd.Flags().StringVar(&sizePattern, "size", "4k", `The size of object. Should be in the form like "8k" or "4k-2m". Only "k", "m" and "g" is allowed as an unit.`) cmd.Flags().DurationVar(&execTime, "time", time.Second*3, "Time duration for run the workload.") cmd.Flags().StringSliceVar(&bucketNames, "bucket", nil, "The name list of the buckets. e.g. \"bucket1,bucket2\"") cmd.Flags().StringVar(&opeRatioStr, "ope_ratio", "1,1,1,0", "The ration of put, get, delete and list operations. e.g. \"2,3,1,1\"") cmd.Flags().StringVar(&endpoint, "endpoint", "", "The endpoint URL and TCP port number. e.g. \"http://127.0.0.1:9000\"") - cmd.Flags().StringVar(&multipartThreshStr, "multipart_thresh", "100m", "The threshold of the object size to switch to the multipart upload. Only \"k\" and \"m\" is allowed as an unit.") + cmd.Flags().StringVar(&multipartThreshStr, "multipart_thresh", "100m", `The threshold of the object size to switch to the multipart upload. Only "k", "m" and "g" is allowed as an unit.`) } diff --git a/internal/multiprocess/leader.go b/internal/multiprocess/leader.go index 533f22e..29eff00 100644 --- a/internal/multiprocess/leader.go +++ b/internal/multiprocess/leader.go @@ -91,7 +91,7 @@ func GetResultFromAllFollower(followerList []string) (bool, string, error) { canceled = true returnedErr = err successAll = false - cancelErr := cancelFollowerWorkload(followerList) + cancelErr := CancelFollowerWorkload(followerList) if cancelErr != nil { log.Printf("Failed to cancel followers' workload. err: %v\n", cancelErr) } @@ -102,7 +102,7 @@ func GetResultFromAllFollower(followerList []string) (bool, string, error) { if !canceled { canceled = true successAll = false - cancelErr := cancelFollowerWorkload(followerList) + cancelErr := CancelFollowerWorkload(followerList) if cancelErr != nil { log.Printf("Failed to cancel followers' workload. err: %v\n", cancelErr) } @@ -151,7 +151,7 @@ func getResultFromFollower(follower string) (bool, string, error) { return (string(body) == successMessage), report, nil } -func cancelFollowerWorkload(followerList []string) error { +func CancelFollowerWorkload(followerList []string) error { var returnedErr error for _, follower := range followerList { path, err := url.JoinPath(follower, "cancel") diff --git a/internal/object/object.go b/internal/object/object.go index c60c375..b3d0f42 100644 --- a/internal/object/object.go +++ b/internal/object/object.go @@ -23,8 +23,8 @@ type Object struct { } type ObjectMeta struct { - ObjectList []Object `json:"objectList"` - ExistingObjectIDs []int64 `json:"existingObjectIDs"` + ObjectList []*Object `json:"objectList"` + ExistingObjectIDs []int64 `json:"existingObjectIDs"` existingObjectIDMap map[int64]struct{} KeyIDOffset int64 `json:"keyIDOffset"` KeyPrefix string @@ -47,11 +47,16 @@ func generateKey(objID int64) string { return fmt.Sprintf("%s%010x", KeyShortPrefix, objID) } +func getObjIDFromKey(key string) (int64, error) { + return strconv.ParseInt(key[KeyPrefixLength:], 16, 64) + +} + func NewObjectMeta(numObj int, keyIDOffset int64) *ObjectMeta { om := &ObjectMeta{} - om.ObjectList = make([]Object, numObj) + om.ObjectList = make([]*Object, numObj) for objID := 0; objID < numObj; objID++ { - om.ObjectList[objID] = *NewObject(keyIDOffset + int64(objID)) + om.ObjectList[objID] = NewObject(keyIDOffset + int64(objID)) } om.ExistingObjectIDs = make([]int64, 0, int(math.Sqrt(float64(numObj)))) om.existingObjectIDMap = make(map[int64]struct{}) @@ -63,11 +68,11 @@ func NewObjectMeta(numObj int, keyIDOffset int64) *ObjectMeta { func (om *ObjectMeta) GetRandomObject() *Object { objID := rand.Intn(len(om.ObjectList)) - return &om.ObjectList[objID] + return om.ObjectList[objID] } func (om *ObjectMeta) RegisterToExistingList(key string) { - objID, err := strconv.ParseInt(key[KeyPrefixLength:], 16, 64) + objID, err := getObjIDFromKey(key) if err != nil { log.Fatal(err) } @@ -96,7 +101,7 @@ func (om *ObjectMeta) PopExistingRandomObject() *Object { log.Fatalf("objID 0x%x found in ExistingObjectIDs, but not in existingObjectIDMap.", objID) } delete(om.existingObjectIDMap, objID) - return &om.ObjectList[objID] + return om.ObjectList[objID] } func (om *ObjectMeta) GetExistingRandomObject() *Object { @@ -106,11 +111,11 @@ func (om *ObjectMeta) GetExistingRandomObject() *Object { eoIDIndex := rand.Intn(len(om.ExistingObjectIDs)) objID := om.ExistingObjectIDs[eoIDIndex] - return &om.ObjectList[objID] + return om.ObjectList[objID] } func (om *ObjectMeta) Exist(key string) bool { - objID, err := strconv.ParseInt(key[KeyPrefixLength:], 16, 64) + objID, err := getObjIDFromKey(key) if err != nil { log.Fatal(err) } diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 44d54e0..5ed9e45 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -101,12 +101,13 @@ func (r *Runner) init() { } if r.loadFileName == "" { r.execContext.Workers = make([]Worker, r.execContext.NumWorker) - startID := rand.Intn(maxWorkerID) - r.execContext.StartWorkerID = startID - for i := range r.execContext.Workers { - r.execContext.Workers[i].id = (startID + i) % maxWorkerID - r.execContext.Workers[i].minSize = r.execContext.MinSize - r.execContext.Workers[i].maxSize = r.execContext.MaxSize + r.execContext.StartWorkerID = rand.Intn(maxWorkerID) + } + for i := range r.execContext.Workers { + r.execContext.Workers[i].id = (r.execContext.StartWorkerID + i) % maxWorkerID + r.execContext.Workers[i].minSize = r.execContext.MinSize + r.execContext.Workers[i].maxSize = r.execContext.MaxSize + if r.loadFileName == "" { r.execContext.Workers[i].BucketsWithObject = make([]*BucketWithObject, len(r.execContext.BucketNames)) for j, bucketName := range r.execContext.BucketNames { r.execContext.Workers[i].BucketsWithObject[j] = &BucketWithObject{ @@ -116,22 +117,14 @@ func (r *Runner) init() { (int64(r.processID)<<32)+(int64(i)<<24)), } } - r.execContext.Workers[i].client = r.client - r.execContext.Workers[i].st = &r.st - r.execContext.Workers[i].ShowInfo() - } - } else { - for i := range r.execContext.Workers { - r.execContext.Workers[i].id = (r.execContext.StartWorkerID + i) % maxWorkerID - r.execContext.Workers[i].minSize = r.execContext.MinSize - r.execContext.Workers[i].maxSize = r.execContext.MaxSize - r.execContext.Workers[i].client = r.client - r.execContext.Workers[i].st = &r.st + } else { for j := range r.execContext.Workers[i].BucketsWithObject { r.execContext.Workers[i].BucketsWithObject[j].ObjectMeta.TidyUp() } - r.execContext.Workers[i].ShowInfo() } + r.execContext.Workers[i].client = r.client + r.execContext.Workers[i].st = &r.st + r.execContext.Workers[i].ShowInfo() } }