From bf3f6c36f37cf367b53e76816210b68881ada8f4 Mon Sep 17 00:00:00 2001 From: xgzlucario <912156837@qq.com> Date: Sat, 13 Jul 2024 02:16:14 +0800 Subject: [PATCH] feat: quicklist iter support start end param --- command.go | 4 +- command_test.go | 9 ++-- internal/list/bench_test.go | 18 +++++++- internal/list/list.go | 54 +++++++++++++++++----- internal/list/list_test.go | 21 ++++++++- internal/list/listpack.go | 7 +-- internal/sstring/sstring.go | 38 ---------------- internal/sstring/sstring_test.go | 77 -------------------------------- rotom.go | 10 ++--- 9 files changed, 91 insertions(+), 147 deletions(-) delete mode 100644 internal/sstring/sstring.go delete mode 100644 internal/sstring/sstring_test.go diff --git a/command.go b/command.go index ff722c7..0dc7481 100644 --- a/command.go +++ b/command.go @@ -60,8 +60,8 @@ func equalCommand(str, lowerText string) bool { return false } const s = 'a' - 'A' - for i, lo := range lowerText { - delta := lo - rune(str[i]) + for i, lt := range lowerText { + delta := lt - rune(str[i]) if delta != 0 && delta != s { return false } diff --git a/command_test.go b/command_test.go index 9c84c96..1db5cc3 100644 --- a/command_test.go +++ b/command_test.go @@ -97,10 +97,8 @@ func TestCommand(t *testing.T) { assert.Equal(resm, map[string]string{"k1": "v1", "k2": "v2", "k3": "v3", "k4": "v4", "k5": "v5"}) // hdel - { - res, _ := rdb.HDel(ctx, "map", "k1", "k2", "k3", "k99").Result() - assert.Equal(res, int64(3)) - } + res, _ = rdb.HDel(ctx, "map", "k1", "k2", "k3", "k99").Result() + assert.Equal(res, int64(3)) // error _, err := rdb.HSet(ctx, "map").Result() @@ -123,6 +121,9 @@ func TestCommand(t *testing.T) { res, _ := rdb.LRange(ctx, "list", 0, -1).Result() assert.Equal(res, []string{"c", "b", "a", "d", "e", "f"}) + res, _ = rdb.LRange(ctx, "list", 1, 3).Result() + assert.Equal(res, []string{"b", "a"}) + // lpop val, _ := rdb.LPop(ctx, "list").Result() assert.Equal(val, "c") diff --git a/internal/list/bench_test.go b/internal/list/bench_test.go index 68e388c..03f10fc 100644 --- a/internal/list/bench_test.go +++ b/internal/list/bench_test.go @@ -31,20 +31,34 @@ func BenchmarkList(b *testing.B) { ls.RPop() } }) - b.Run("range", func(b *testing.B) { + b.Run("range_all", func(b *testing.B) { ls := genList(0, 1000) b.ResetTimer() for i := 0; i < b.N; i++ { ls.Range(0, -1, func([]byte) {}) } }) - b.Run("revrange", func(b *testing.B) { + b.Run("range_100", func(b *testing.B) { + ls := genList(0, 1000) + b.ResetTimer() + for i := 0; i < b.N; i++ { + ls.Range(0, 100, func([]byte) {}) + } + }) + b.Run("revrange_all", func(b *testing.B) { ls := genList(0, 1000) b.ResetTimer() for i := 0; i < b.N; i++ { ls.RevRange(0, -1, func([]byte) {}) } }) + b.Run("revrange_100", func(b *testing.B) { + ls := genList(0, 1000) + b.ResetTimer() + for i := 0; i < b.N; i++ { + ls.RevRange(0, 100, func([]byte) {}) + } + }) } func BenchmarkListPack(b *testing.B) { diff --git a/internal/list/list.go b/internal/list/list.go index 2891d7a..7dbead5 100644 --- a/internal/list/list.go +++ b/internal/list/list.go @@ -18,10 +18,6 @@ type Node struct { prev, next *Node } -func SetMaxListPackSize(s int) { - maxListPackSize = s -} - // New create a quicklist instance. func New() *QuickList { n := newNode() @@ -97,11 +93,28 @@ func (ls *QuickList) Range(start, end int, f func(data []byte)) { if end == -1 { end = math.MaxInt } - for lp := ls.head; lp != nil; lp = lp.next { - it := lp.Iterator().SeekBegin() - for !it.IsEnd() { - f(it.Next()) + count := end - start + + lp := ls.head + for lp != nil && start > lp.Size() { + start -= lp.Size() + lp = lp.next + } + + it := lp.Iterator().SeekBegin() + for range start { + it.Next() + } + + for range count { + if it.IsEnd() { + if lp.next == nil { + return + } + lp = lp.next + it = lp.Iterator().SeekBegin() } + f(it.Next()) } } @@ -109,10 +122,27 @@ func (ls *QuickList) RevRange(start, end int, f func(data []byte)) { if end == -1 { end = math.MaxInt } - for lp := ls.tail; lp != nil; lp = lp.prev { - it := lp.Iterator().SeekEnd() - for !it.IsBegin() { - f(it.Prev()) + count := end - start + + lp := ls.tail + for lp != nil && start > lp.Size() { + start -= lp.Size() + lp = lp.prev + } + + it := lp.Iterator().SeekEnd() + for range start { + it.Prev() + } + + for range count { + if it.IsBegin() { + if lp.prev == nil { + return + } + lp = lp.prev + it = lp.Iterator().SeekEnd() } + f(it.Prev()) } } diff --git a/internal/list/list_test.go b/internal/list/list_test.go index 17e1d56..34ffebd 100644 --- a/internal/list/list_test.go +++ b/internal/list/list_test.go @@ -23,9 +23,8 @@ func list2slice(ls *QuickList) (res []string) { } func TestList(t *testing.T) { - const N = 1000 + const N = 10000 assert := assert.New(t) - SetMaxListPackSize(128) t.Run("lpush", func(t *testing.T) { ls := New() @@ -87,6 +86,15 @@ func TestList(t *testing.T) { i++ }) assert.Equal(i, N) + + for _, start := range []int{100, 1000, 5000} { + i = 0 + ls.Range(start, start+100, func(data []byte) { + assert.Equal(string(data), genKey(start+i)) + i++ + }) + assert.Equal(i, 100) + } }) t.Run("revrange", func(t *testing.T) { @@ -97,5 +105,14 @@ func TestList(t *testing.T) { i++ }) assert.Equal(i, N) + + for _, start := range []int{100, 1000, 5000} { + i = 0 + ls.RevRange(start, start+100, func(data []byte) { + assert.Equal(string(data), genKey(N-start-i-1)) + i++ + }) + assert.Equal(i, 100) + } }) } diff --git a/internal/list/listpack.go b/internal/list/listpack.go index 440fd24..674f70a 100644 --- a/internal/list/listpack.go +++ b/internal/list/listpack.go @@ -8,11 +8,12 @@ import ( "github.com/xgzlucario/rotom/internal/pkg" ) -var ( +const ( maxListPackSize = 8 * 1024 +) - bpool = pkg.NewBufferPool() - +var ( + bpool = pkg.NewBufferPool() encoder, _ = zstd.NewWriter(nil) decoder, _ = zstd.NewReader(nil) ) diff --git a/internal/sstring/sstring.go b/internal/sstring/sstring.go deleted file mode 100644 index ccc0b31..0000000 --- a/internal/sstring/sstring.go +++ /dev/null @@ -1,38 +0,0 @@ -// package sstring is shared strings. - -package sstring - -import ( - "strings" - - "github.com/cockroachdb/swiss" -) - -var nums = swiss.New[string, int](8) -var strs []string - -// Load a shared string from its number. -func Load(num int) (str string) { - if num >= 0 && num < len(strs) { - str = strs[num] - return str - } - panic("string not found") -} - -// Store a shared string. -func Store(str string) int { - num, ok := nums.Get(str) - if !ok { - str = strings.Clone(str) - num = len(strs) - strs = append(strs, str) - nums.Put(str, num) - } - return num -} - -// Len returns the number of shared strings. -func Len() int { - return len(strs) -} diff --git a/internal/sstring/sstring_test.go b/internal/sstring/sstring_test.go deleted file mode 100644 index aa742b1..0000000 --- a/internal/sstring/sstring_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package sstring - -import ( - "crypto/rand" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestShared(t *testing.T) { - assert := assert.New(t) - - for i := -1; i < 10; i++ { - var str string - func() { - defer func() { - assert.True(recover().(string) == "string not found") - }() - str = Load(i) - }() - assert.Equal(str, "") - } - assert.Equal(Store("hello"), 0) - assert.Equal(Store(""), 1) - assert.Equal(Store("jello"), 2) - assert.Equal(Store("hello"), 0) - assert.Equal(Store(""), 1) - assert.Equal(Store("jello"), 2) - assert.Equal(Load(0), "hello") - assert.Equal(Load(1), "") - assert.Equal(Load(2), "jello") - assert.Equal(Len(), 3) - -} - -func randStr(n int) string { - b := make([]byte, n) - rand.Read(b) - for i := 0; i < n; i++ { - b[i] = 'a' + b[i]%26 - } - return string(b) -} - -func BenchmarkStore(b *testing.B) { - wmap := make(map[string]bool, b.N) - for len(wmap) < b.N { - wmap[randStr(10)] = true - } - words := make([]string, 0, b.N) - for word := range wmap { - words = append(words, word) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - Store(words[i]) - } -} - -func BenchmarkLoad(b *testing.B) { - wmap := make(map[string]bool, b.N) - for len(wmap) < b.N { - wmap[randStr(10)] = true - } - words := make([]string, 0, b.N) - for word := range wmap { - words = append(words, word) - } - var nums []int - for i := 0; i < b.N; i++ { - nums = append(nums, Store(words[i])) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - Load(nums[i]) - } -} diff --git a/rotom.go b/rotom.go index 3c51018..2fa7b80 100644 --- a/rotom.go +++ b/rotom.go @@ -155,7 +155,7 @@ func ProcessQueryBuf(client *Client) { } else { err := ErrUnknownCommand(command) client.replyWriter.WriteError(err) - log.Warn().Msgf("ERR %v", err) + log.Error().Msg(err.Error()) } } @@ -197,12 +197,8 @@ func initServer(config *Config) (err error) { } func SyncAOF(loop *AeLoop, id int, extra interface{}) { - if db.aof == nil { - return - } - err := db.aof.Flush() - if err != nil { - log.Error().Msgf("flush aof buffer error: %v", err) + if err := db.aof.Flush(); err != nil { + log.Error().Msgf("sync aof error: %v", err) } }