Skip to content

Commit

Permalink
Merge pull request #30 from peng225/list-support
Browse files Browse the repository at this point in the history 
List support
  • Loading branch information
@peng225
peng225 authored Feb 12, 2023
2 parents eeb7ab0 + 0a4f696 commit 4c39c80
Show file tree
Hide file tree
Showing 10 changed files with 262 additions and 163 deletions.
4 changes: 2 additions & 2 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ test: $(OVAL)

.PHONY: run
run: $(OVAL)
$(OVAL) --size 4k-16k --time $(EXEC_TIME) --num_obj 1024 --num_worker 4 --bucket "test-bucket,test-bucket2" --endpoint http://localhost:9000 --save test.json
$(OVAL) --size 4k-16k --time $(EXEC_TIME) --num_obj 1024 --num_worker 4 --bucket "test-bucket,test-bucket2" --ope_ratio 8,8,8,1 --endpoint http://localhost:9000 --save test.json
$(OVAL) --time 3s --load test.json

.PHONY: run-multi-process
Expand All @@ -30,7 +30,7 @@ run-multi-process: $(OVAL)

.PHONY: run-leader
run-leader: $(OVAL)
$(OVAL) leader --follower_list "http://localhost:8080,http://localhost:8081,http://localhost:8082" --size 4k-16k --time $(EXEC_TIME) --num_obj 1024 --num_worker 4 --bucket "test-bucket,test-bucket2" --endpoint http://localhost:9000
$(OVAL) leader --follower_list "http://localhost:8080,http://localhost:8081,http://localhost:8082" --size 4k-16k --time $(EXEC_TIME) --num_obj 1024 --num_worker 4 --bucket "test-bucket,test-bucket2" --ope_ratio 8,8,8,1 --endpoint http://localhost:9000

.PHONY: run-followers
run-followers: $(OVAL)
Expand Down
152 changes: 86 additions & 66 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,38 +65,47 @@ Just build and run Oval.
#### Example 1: Success case

```
$ ./oval --size 4k-16k --time 5s --num_obj 1024 --num_worker 4 --bucket "test-bucket,test-bucket2" --endpoint http://localhost:9000 --save test.json
2022/12/31 21:31:04 worker.go:33: Worker ID = 0x1e77, Key = [ov0000000000, ov00000000ff]
2022/12/31 21:31:04 worker.go:33: Worker ID = 0x1e78, Key = [ov0000000100, ov00000001ff]
2022/12/31 21:31:04 worker.go:33: Worker ID = 0x1e79, Key = [ov0000000200, ov00000002ff]
2022/12/31 21:31:04 worker.go:33: Worker ID = 0x1e7a, Key = [ov0000000300, ov00000003ff]
2022/12/31 21:31:04 runner.go:151: Validation start.
2022/12/31 21:31:09 runner.go:191: Validation finished.
2022/12/31 21:31:09 stat.go:32: Statistics report.
2022/12/31 21:31:09 stat.go:33: put count: 670
2022/12/31 21:31:09 stat.go:34: get count: 609
2022/12/31 21:31:09 stat.go:35: get (for validation) count: 1257
2022/12/31 21:31:09 stat.go:36: delete count: 575
$ ./oval --size 4k-16k --time 5s --num_obj 1024 --num_worker 4 --bucket "test-bucket,test-bucket2" --endpoint http://localhost:9000
2023/02/12 14:32:01 runner.go:108: Clearing bucket 'test-bucket'.
2023/02/12 14:32:02 runner.go:113: Bucket cleared successfully.
2023/02/12 14:32:02 runner.go:108: Clearing bucket 'test-bucket2'.
2023/02/12 14:32:03 runner.go:113: Bucket cleared successfully.
2023/02/12 14:32:03 worker.go:33: Worker ID = 0x3595, Key = [ov0000000000, ov00000000ff]
2023/02/12 14:32:03 worker.go:33: Worker ID = 0x3596, Key = [ov0001000000, ov00010000ff]
2023/02/12 14:32:03 worker.go:33: Worker ID = 0x3597, Key = [ov0002000000, ov00020000ff]
2023/02/12 14:32:03 worker.go:33: Worker ID = 0x3598, Key = [ov0003000000, ov00030000ff]
2023/02/12 14:32:03 runner.go:153: Validation start.
2023/02/12 14:32:08 runner.go:197: Validation finished.
2023/02/12 14:32:08 stat.go:37: Statistics report.
2023/02/12 14:32:08 stat.go:38: put count: 618
2023/02/12 14:32:08 stat.go:39: get count: 483
2023/02/12 14:32:08 stat.go:40: get (for validation) count: 1140
2023/02/12 14:32:08 stat.go:41: list count: 0
2023/02/12 14:32:08 stat.go:42: delete count: 50
```

#### Example 2: Data corruption case

```
$ ./oval --size 4k-16k --time 5s --num_obj 1024 --num_worker 4 --bucket "test-bucket,test-bucket2" --endpoint http://localhost:9000 --save test.json
2023/01/01 14:54:41 worker.go:33: Worker ID = 0x2009, Key = [ov0000000000, ov00000000ff]
2023/01/01 14:54:41 worker.go:33: Worker ID = 0x200a, Key = [ov0000000100, ov00000001ff]
2023/01/01 14:54:41 worker.go:33: Worker ID = 0x200b, Key = [ov0000000200, ov00000002ff]
2023/01/01 14:54:41 worker.go:33: Worker ID = 0x200c, Key = [ov0000000300, ov00000003ff]
2023/01/01 14:54:41 runner.go:151: Validation start.
2023/01/01 14:54:41 worker.go:101: Data validation error occurred after put.
$ ./oval --size 4k-16k --time 5s --num_obj 1024 --num_worker 4 --bucket "test-bucket,test-bucket2" --endpoint http://localhost:9000
2023/02/12 14:43:46 runner.go:108: Clearing bucket 'test-bucket'.
2023/02/12 14:43:47 runner.go:113: Bucket cleared successfully.
2023/02/12 14:43:47 runner.go:108: Clearing bucket 'test-bucket2'.
2023/02/12 14:43:47 runner.go:113: Bucket cleared successfully.
2023/02/12 14:43:47 worker.go:33: Worker ID = 0x8e14, Key = [ov0000000000, ov00000000ff]
2023/02/12 14:43:47 worker.go:33: Worker ID = 0x8e15, Key = [ov0001000000, ov00010000ff]
2023/02/12 14:43:47 worker.go:33: Worker ID = 0x8e16, Key = [ov0002000000, ov00020000ff]
2023/02/12 14:43:47 worker.go:33: Worker ID = 0x8e17, Key = [ov0003000000, ov00030000ff]
2023/02/12 14:43:47 runner.go:153: Validation start.
2023/02/12 14:43:47 worker.go:101: Data validation error occurred after put.
- WriteCount is wrong. (expected = "2", actual = "1")
- OffsetInObject is wrong. (expected = "0", actual = "256")
00000000 74 65 73 74 2d 62 75 63 6b 65 74 20 20 20 20 20 |test-bucket |
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ bucket name
00000010 6f 76 30 30 30 30 30 30 30 31 66 34 01 00 00 00 |ov00000001f4....|
00000010 6f 76 30 30 30 32 30 30 30 30 32 35 01 00 00 00 |ov0002000025....|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ key name
^^^^^^^^^^^ write count
00000020 00 01 00 00 0a 20 00 00 18 a1 8d 78 2d f1 05 00 |..... .....x-...|
00000020 00 01 00 00 16 8e 00 00 67 ea e0 36 7a f4 05 00 |........g..6z...|
^^^^^^^^^^^ byte offset in this object
^^^^^^^^^^^ worker ID
^^^^^^^^^^^^^^^^^^^^^^^ unix time (micro sec)
Expand All @@ -114,13 +123,14 @@ $ ./oval --size 4k-16k --time 5s --num_obj 1024 --num_worker 4 --bucket "test-bu
000000e0 e0 e1 e2 e3 e4 e5 e6 e7 e8 e9 ea eb ec ed ee ef |................|
000000f0 f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb fc fd fe ff |................|
2023/01/01 14:54:41 runner.go:193: Validation finished.
2023/01/01 14:54:41 stat.go:32: Statistics report.
2023/01/01 14:54:41 stat.go:33: put count: 93
2023/01/01 14:54:41 stat.go:34: get count: 66
2023/01/01 14:54:41 stat.go:35: get (for validation) count: 157
2023/01/01 14:54:41 stat.go:36: delete count: 64
2023/01/01 14:54:41 root.go:52: r.Run() failed.
2023/02/12 14:43:47 runner.go:197: Validation finished.
2023/02/12 14:43:47 stat.go:37: Statistics report.
2023/02/12 14:43:47 stat.go:38: put count: 23
2023/02/12 14:43:47 stat.go:39: get count: 11
2023/02/12 14:43:47 stat.go:40: get (for validation) count: 30
2023/02/12 14:43:47 stat.go:41: list count: 0
2023/02/12 14:43:47 stat.go:42: delete count: 7
2023/02/12 14:43:47 root.go:66: r.Run() failed.
```

### The multi-process mode
Expand All @@ -135,56 +145,66 @@ $ ./oval --size 4k-16k --time 5s --num_obj 1024 --num_worker 4 --bucket "test-bu

```
$ ./oval follower --follower_port 8080
2022/12/31 21:31:47 follower.go:41: Start server. port = 8080
2022/12/31 21:32:04 follower.go:46: Received a init request.
2022/12/31 21:32:04 follower.go:62: Received a start request.
2022/12/31 21:32:04 follower.go:119: ID: 0
2022/12/31 21:32:04 follower.go:120: Context: {http://localhost:9000 [test-bucket test-bucket2] 1024 4 4096 16384 0 []}
2022/12/31 21:32:04 follower.go:121: OpeRatio: [0.3333333333333333 0.3333333333333333 0.3333333333333333]
2022/12/31 21:32:04 follower.go:122: TimeInMs: 5000
2022/12/31 21:32:05 worker.go:33: Worker ID = 0xd681, Key = [ov0000000000, ov00000000ff]
2022/12/31 21:32:05 worker.go:33: Worker ID = 0xd682, Key = [ov0000000100, ov00000001ff]
2022/12/31 21:32:05 worker.go:33: Worker ID = 0xd683, Key = [ov0000000200, ov00000002ff]
2022/12/31 21:32:05 worker.go:33: Worker ID = 0xd684, Key = [ov0000000300, ov00000003ff]
2022/12/31 21:32:05 runner.go:151: Validation start.
2022/12/31 21:32:10 runner.go:191: Validation finished.
2022/12/31 21:32:10 stat.go:32: Statistics report.
2022/12/31 21:32:10 stat.go:33: put count: 479
2022/12/31 21:32:10 stat.go:34: get count: 393
2022/12/31 21:32:10 stat.go:35: get (for validation) count: 892
2022/12/31 21:32:10 stat.go:36: delete count: 399
2023/02/12 14:41:40 follower.go:41: Start server. port = 8080
2023/02/12 14:41:53 follower.go:46: Received a init request.
2023/02/12 14:41:53 follower.go:62: Received a start request.
2023/02/12 14:41:53 follower.go:119: ID: 0
2023/02/12 14:41:53 follower.go:120: Context: {http://localhost:9000 [test-bucket test-bucket2] 1024 4 4096 16384 0 []}
2023/02/12 14:41:53 follower.go:121: OpeRatio: [0.3333333333333333 0.3333333333333333 0.3333333333333333 0]
2023/02/12 14:41:53 follower.go:122: TimeInMs: 5000
2023/02/12 14:41:53 runner.go:108: Clearing bucket 'test-bucket'.
2023/02/12 14:41:53 runner.go:113: Bucket cleared successfully.
2023/02/12 14:41:53 runner.go:108: Clearing bucket 'test-bucket2'.
2023/02/12 14:41:53 runner.go:113: Bucket cleared successfully.
2023/02/12 14:41:53 worker.go:33: Worker ID = 0x7ea7, Key = [ov0000000000, ov00000000ff]
2023/02/12 14:41:53 worker.go:33: Worker ID = 0x7ea8, Key = [ov0001000000, ov00010000ff]
2023/02/12 14:41:53 worker.go:33: Worker ID = 0x7ea9, Key = [ov0002000000, ov00020000ff]
2023/02/12 14:41:53 worker.go:33: Worker ID = 0x7eaa, Key = [ov0003000000, ov00030000ff]
2023/02/12 14:41:53 runner.go:153: Validation start.
2023/02/12 14:41:58 runner.go:197: Validation finished.
2023/02/12 14:41:58 stat.go:37: Statistics report.
2023/02/12 14:41:58 stat.go:38: put count: 479
2023/02/12 14:41:58 stat.go:39: get count: 402
2023/02/12 14:41:58 stat.go:40: get (for validation) count: 872
2023/02/12 14:41:58 stat.go:41: list count: 0
2023/02/12 14:41:58 stat.go:42: delete count: 383
```

##### follower2

```
$ ./oval follower --follower_port 8081
2022/12/31 21:31:53 follower.go:41: Start server. port = 8081
2022/12/31 21:32:04 follower.go:46: Received a init request.
2022/12/31 21:32:04 follower.go:62: Received a start request.
2022/12/31 21:32:04 follower.go:119: ID: 1
2022/12/31 21:32:04 follower.go:120: Context: {http://localhost:9000 [test-bucket test-bucket2] 1024 4 4096 16384 0 []}
2022/12/31 21:32:04 follower.go:121: OpeRatio: [0.3333333333333333 0.3333333333333333 0.3333333333333333]
2022/12/31 21:32:04 follower.go:122: TimeInMs: 5000
2022/12/31 21:32:05 worker.go:33: Worker ID = 0x6518, Key = [ov0100000000, ov01000000ff]
2022/12/31 21:32:05 worker.go:33: Worker ID = 0x6519, Key = [ov0100000100, ov01000001ff]
2022/12/31 21:32:05 worker.go:33: Worker ID = 0x651a, Key = [ov0100000200, ov01000002ff]
2022/12/31 21:32:05 worker.go:33: Worker ID = 0x651b, Key = [ov0100000300, ov01000003ff]
2022/12/31 21:32:05 runner.go:151: Validation start.
2022/12/31 21:32:10 runner.go:191: Validation finished.
2022/12/31 21:32:10 stat.go:32: Statistics report.
2022/12/31 21:32:10 stat.go:33: put count: 466
2022/12/31 21:32:10 stat.go:34: get count: 430
2022/12/31 21:32:10 stat.go:35: get (for validation) count: 886
2022/12/31 21:32:10 stat.go:36: delete count: 416
2023/02/12 14:41:45 follower.go:41: Start server. port = 8081
2023/02/12 14:41:53 follower.go:46: Received a init request.
2023/02/12 14:41:53 follower.go:62: Received a start request.
2023/02/12 14:41:53 follower.go:119: ID: 1
2023/02/12 14:41:53 follower.go:120: Context: {http://localhost:9000 [test-bucket test-bucket2] 1024 4 4096 16384 0 []}
2023/02/12 14:41:53 follower.go:121: OpeRatio: [0.3333333333333333 0.3333333333333333 0.3333333333333333 0]
2023/02/12 14:41:53 follower.go:122: TimeInMs: 5000
2023/02/12 14:41:53 runner.go:108: Clearing bucket 'test-bucket'.
2023/02/12 14:41:53 runner.go:113: Bucket cleared successfully.
2023/02/12 14:41:53 runner.go:108: Clearing bucket 'test-bucket2'.
2023/02/12 14:41:53 runner.go:113: Bucket cleared successfully.
2023/02/12 14:41:53 worker.go:33: Worker ID = 0x5803, Key = [ov0100000000, ov01000000ff]
2023/02/12 14:41:53 worker.go:33: Worker ID = 0x5804, Key = [ov0101000000, ov01010000ff]
2023/02/12 14:41:53 worker.go:33: Worker ID = 0x5805, Key = [ov0102000000, ov01020000ff]
2023/02/12 14:41:53 worker.go:33: Worker ID = 0x5806, Key = [ov0103000000, ov01030000ff]
2023/02/12 14:41:53 runner.go:153: Validation start.
2023/02/12 14:41:58 runner.go:197: Validation finished.
2023/02/12 14:41:58 stat.go:37: Statistics report.
2023/02/12 14:41:58 stat.go:38: put count: 488
2023/02/12 14:41:58 stat.go:39: get count: 430
2023/02/12 14:41:58 stat.go:40: get (for validation) count: 892
2023/02/12 14:41:58 stat.go:41: list count: 0
2023/02/12 14:41:58 stat.go:42: delete count: 392
```

##### leader

```
$ ./oval leader --follower_list "http://localhost:8080,http://localhost:8081" --size 4k-16k --time 5s --num_obj 1024 --num_worker 4 --bucket "test-bucket,test-bucket2" --endpoint http://localhost:9000
2022/12/31 21:32:04 leader.go:34: Sent start requests to all followers.
2022/12/31 21:32:10 leader.go:40: The report from followers:
2023/02/12 14:41:53 leader.go:31: Sent start requests to all followers.
2023/02/12 14:41:58 leader.go:37: The report from followers:
follower: http://localhost:8080
OK
follower: http://localhost:8081
Expand Down
22 changes: 17 additions & 5 deletions cmd/root.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
)

var (
numObj int64
numObj int
numWorker int
sizePattern string
execTime time.Duration
Expand Down Expand Up @@ -114,9 +114,21 @@ func handleCommonFlags() {
log.Fatal(err)
}

if numObj%int64(numWorker) != 0 {
if numWorker >= 256 {
log.Fatal("The number of workers must be less than 256.")
}

if numObj > 0x1000000 {
log.Fatal("The number of objects must be less than 16777216.")
}

if numObj < numWorker {
log.Fatal("The number of objects must be larger than or equal to the number of workers.")
}

if numObj%numWorker != 0 {
log.Printf("warning: The number of objects (%d) is not divisible by the number of workers (%d). Only %d objects will be used.\n",
numObj, numWorker, numObj/int64(numWorker*numWorker))
numObj, numWorker, numObj/numWorker*numWorker)
}

execContext = &runner.ExecutionContext{
Expand All @@ -130,11 +142,11 @@ func handleCommonFlags() {
}

func defineCommonFlags(cmd *cobra.Command) {
cmd.Flags().Int64Var(&numObj, "num_obj", 10, "The maximum number of objects.")
cmd.Flags().IntVar(&numObj, "num_obj", 10, "The maximum number of objects per process.")
cmd.Flags().IntVar(&numWorker, "num_worker", 1, "The number of workers per process.")
cmd.Flags().StringVar(&sizePattern, "size", "4k", "The size of object. Should be in the form like \"8k\" or \"4k-2m\". The unit \"g\" or \"G\" is not allowed.")
cmd.Flags().DurationVar(&execTime, "time", time.Second*3, "Time duration for run the workload.")
cmd.Flags().StringSliceVar(&bucketNames, "bucket", nil, "The name list of the buckets. e.g. \"bucket1,bucket2\"")
cmd.Flags().StringVar(&opeRatioStr, "ope_ratio", "1,1,1", "The ration of put, get and delete operations. e.g. \"2,3,1\"")
cmd.Flags().StringVar(&opeRatioStr, "ope_ratio", "1,1,1,0", "The ration of put, get, delete and list operations. e.g. \"2,3,1,1\"")
cmd.Flags().StringVar(&endpoint, "endpoint", "", "The endpoint URL and TCP port number. e.g. \"http://127.0.0.1:9000\"")
}
30 changes: 15 additions & 15 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ module github.com/peng225/oval
go 1.19

require (
github.com/aws/aws-sdk-go-v2 v1.17.3
github.com/aws/aws-sdk-go-v2/config v1.18.8
github.com/aws/aws-sdk-go-v2/service/s3 v1.30.0
github.com/aws/aws-sdk-go-v2 v1.17.4
github.com/aws/aws-sdk-go-v2/config v1.18.12
github.com/aws/aws-sdk-go-v2/service/s3 v1.30.2
github.com/dsnet/golib/memfile v1.0.0
github.com/pkg/profile v1.7.0
github.com/spf13/cobra v1.6.1
Expand All @@ -14,19 +14,19 @@ require (

require (
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.8 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.18 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.12 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.19 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.22 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.21 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.12.0 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.18.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.23 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.22 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.12.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.18.3 // indirect
github.com/aws/smithy-go v1.13.5 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/felixge/fgprof v0.9.3 // indirect
Expand Down
Loading

0 comments on commit 4c39c80

Please sign in to comment.