feat: listpack use lz4 compress algprithm

go.mod

@@ -5,7 +5,7 @@ go 1.22
 require (
 	github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964
 	github.com/deckarep/golang-set/v2 v2.6.0
-	github.com/klauspost/compress v1.17.9
+	github.com/pierrec/lz4/v4 v4.1.21
 	github.com/redis/go-redis/v9 v9.5.2
 	github.com/rs/zerolog v1.33.0
 	github.com/sakeven/RbTree v1.1.1
@@ -24,7 +24,7 @@ require (
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rogpeppe/go-internal v1.9.0 // indirect
-	golang.org/x/exp v0.0.0-20240716160929-1d5bc16f04a8 // indirect
+	golang.org/x/exp v0.0.0-20240716175740-e3f259677ff7 // indirect
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -19,8 +19,6 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cu
 github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ=
 github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
-github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
@@ -34,6 +32,8 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
+github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -51,8 +51,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tidwall/mmap v0.3.0 h1:XXt1YsiXCF5/UAu3pLbu6g7iulJ9jsbs6vt7UpiV0sY=
 github.com/tidwall/mmap v0.3.0/go.mod h1:2/dNzF5zA+te/JVHfrqNLcRkb8LjdH3c80vYHFQEZRk=
-golang.org/x/exp v0.0.0-20240716160929-1d5bc16f04a8 h1:Z+vTUQyBb738QmIhbJx3z4htsxDeI+rd0EHvNm8jHkg=
-golang.org/x/exp v0.0.0-20240716160929-1d5bc16f04a8/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
+golang.org/x/exp v0.0.0-20240716175740-e3f259677ff7 h1:wDLEX9a7YQoKdKNQt88rtydkqDxeGaBUTnIYc3iG/mA=
+golang.org/x/exp v0.0.0-20240716175740-e3f259677ff7/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

internal/list/bench_test.go

@@ -46,3 +46,17 @@ func BenchmarkList(b *testing.B) {
 		}
 	})
 }
+
+func BenchmarkCompress(b *testing.B) {
+	b.Run("compress", func(b *testing.B) {
+		lp := NewListPack()
+		for i := 0; i < 256; i++ {
+			lp.RPush("Hello", "World")
+		}
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			lp.Compress()
+			lp.Decompress()
+		}
+	})
+}

internal/list/listpack.go

@@ -4,7 +4,7 @@ import (
 	"encoding/binary"
 	"slices"
 
-	"github.com/klauspost/compress/zstd"
+	"github.com/pierrec/lz4/v4"
 	"github.com/xgzlucario/rotom/internal/pkg"
 )
 
@@ -13,9 +13,9 @@ const (
 )
 
 var (
-	bpool      = pkg.NewBufferPool()
-	encoder, _ = zstd.NewWriter(nil)
-	decoder, _ = zstd.NewReader(nil)
+	bpool = pkg.NewBufferPool()
+
+	c lz4.Compressor
 )
 
 // ListPack is a lists of strings serialization format on Redis.
@@ -35,9 +35,9 @@ var (
 	Using this structure, it is fast to iterate from both sides.
 */
 type ListPack struct {
-	compress bool
-	size     uint32
-	data     []byte
+	srcLen uint32 // srcLen is data size before compressed.
+	size   uint32
+	data   []byte
 }
 
 func NewListPack() *ListPack {
@@ -79,20 +79,31 @@ func (lp *ListPack) RPop() (val string, ok bool) {
 }
 
 func (lp *ListPack) Compress() {
-	if lp.compress {
+	if lp.srcLen > 0 {
+		return
+	}
+	if len(lp.data) == 0 {
 		return
 	}
-	lp.data = encoder.EncodeAll(lp.data, make([]byte, 0, len(lp.data)/3))
-	lp.data = slices.Clip(lp.data)
-	lp.compress = true
+	lp.srcLen = uint32(len(lp.data))
+
+	dst := bpool.Get(lz4.CompressBlockBound(len(lp.data)))
+	n, _ := c.CompressBlock(lp.data, dst)
+
+	bpool.Put(lp.data)
+	lp.data = dst[:n]
 }
 
 func (lp *ListPack) Decompress() {
-	if !lp.compress {
+	if lp.srcLen == 0 {
 		return
 	}
-	lp.data, _ = decoder.DecodeAll(lp.data, nil)
-	lp.compress = false
+	dst := bpool.Get(int(lp.srcLen))
+	n, _ := lz4.UncompressBlock(lp.data, dst)
+
+	bpool.Put(lp.data)
+	lp.data = dst[:n]
+	lp.srcLen = 0
 }
 
 type lpIterator struct {
@@ -101,6 +112,9 @@ type lpIterator struct {
 }
 
 func (lp *ListPack) Iterator() *lpIterator {
+	if lp.srcLen > 0 {
+		lp.Decompress()
+	}
 	return &lpIterator{ListPack: lp}
 }
 

internal/list/listpack_test.go

@@ -132,11 +132,20 @@ func TestListpack(t *testing.T) {
 
 	t.Run("compress", func(t *testing.T) {
 		lp := NewListPack()
-		lp.RPush("A", "B", "C")
+		for i := 0; i < 100; i++ {
+			lp.RPush("A1", "B2", "C3")
+		}
+
+		before := len(lp.data)
 		lp.Compress()
 		lp.Compress()
+		afterC := len(lp.data)
+		assert.Less(afterC, before)
+
 		lp.Decompress()
 		lp.Decompress()
-		assert.Equal(lp2list(lp), []string{"A", "B", "C"})
+		afterD := len(lp.data)
+		assert.Less(afterC, afterD)
+		assert.Equal(afterD, before)
 	})
 }