-
Notifications
You must be signed in to change notification settings - Fork 2.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
16 changed files
with
692 additions
and
12 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
package geecache | ||
|
||
// A ByteView holds an immutable view of bytes. | ||
type ByteView struct { | ||
b []byte | ||
} | ||
|
||
// Len returns the view's length | ||
func (v ByteView) Len() int { | ||
return len(v.b) | ||
} | ||
|
||
// ByteSlice returns a copy of the data as a byte slice. | ||
func (v ByteView) ByteSlice() []byte { | ||
return cloneBytes(v.b) | ||
} | ||
|
||
// String returns the data as a string, making a copy if necessary. | ||
func (v ByteView) String() string { | ||
return string(v.b) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,35 @@ | ||
package geecache | ||
|
||
import ( | ||
"geecache/lru" | ||
"sync" | ||
) | ||
|
||
type cache struct { | ||
mu sync.RWMutex | ||
lru *lru.Cache | ||
cacheBytes int64 | ||
} | ||
|
||
func (c *cache) add(key string, value ByteView) { | ||
c.mu.RLock() | ||
defer c.mu.RUnlock() | ||
if c.lru == nil { | ||
c.lru = lru.New(c.cacheBytes, nil) | ||
} | ||
c.lru.Add(key, value) | ||
} | ||
|
||
func (c *cache) get(key string) (value ByteView, ok bool) { | ||
c.mu.RLock() | ||
defer c.mu.RUnlock() | ||
if c.lru == nil { | ||
return | ||
} | ||
|
||
if v, ok := c.lru.Get(key); ok { | ||
return v.(ByteView), ok | ||
} | ||
|
||
return | ||
} |
58 changes: 58 additions & 0 deletions
58
gee-cache/day5-multi-nodes/geecache/consistenthash/consistenthash.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,58 @@ | ||
package consistenthash | ||
|
||
import ( | ||
"hash/crc32" | ||
"sort" | ||
"strconv" | ||
) | ||
|
||
// Hash maps bytes to uint32 | ||
type Hash func(data []byte) uint32 | ||
|
||
// Map constains all hashed keys | ||
type Map struct { | ||
hash Hash | ||
replicas int | ||
keys []int // Sorted | ||
hashMap map[int]string | ||
} | ||
|
||
// New creates a Map instance | ||
func New(replicas int, fn Hash) *Map { | ||
m := &Map{ | ||
replicas: replicas, | ||
hash: fn, | ||
hashMap: make(map[int]string), | ||
} | ||
if m.hash == nil { | ||
m.hash = crc32.ChecksumIEEE | ||
} | ||
return m | ||
} | ||
|
||
// Add adds some keys to the hash. | ||
func (m *Map) Add(keys ...string) { | ||
for _, key := range keys { | ||
for i := 0; i < m.replicas; i++ { | ||
hash := int(m.hash([]byte(strconv.Itoa(i) + key))) | ||
m.keys = append(m.keys, hash) | ||
m.hashMap[hash] = key | ||
} | ||
} | ||
sort.Ints(m.keys) | ||
} | ||
|
||
// Get gets the closest item in the hash to the provided key. | ||
func (m *Map) Get(key string) string { | ||
if len(m.keys) == 0 { | ||
return "" | ||
} | ||
|
||
hash := int(m.hash([]byte(key))) | ||
// Binary search for appropriate replica. | ||
idx := sort.Search(len(m.keys), func(i int) bool { | ||
return m.keys[i] >= hash | ||
}) | ||
|
||
return m.hashMap[m.keys[idx%len(m.keys)]] | ||
} |
43 changes: 43 additions & 0 deletions
43
gee-cache/day5-multi-nodes/geecache/consistenthash/consistenthash_test.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
package consistenthash | ||
|
||
import ( | ||
"strconv" | ||
"testing" | ||
) | ||
|
||
func TestHashing(t *testing.T) { | ||
hash := New(3, func(key []byte) uint32 { | ||
i, _ := strconv.Atoi(string(key)) | ||
return uint32(i) | ||
}) | ||
|
||
// Given the above hash function, this will give replicas with "hashes": | ||
// 2, 4, 6, 12, 14, 16, 22, 24, 26 | ||
hash.Add("6", "4", "2") | ||
|
||
testCases := map[string]string{ | ||
"2": "2", | ||
"11": "2", | ||
"23": "4", | ||
"27": "2", | ||
} | ||
|
||
for k, v := range testCases { | ||
if hash.Get(k) != v { | ||
t.Errorf("Asking for %s, should have yielded %s", k, v) | ||
} | ||
} | ||
|
||
// Adds 8, 18, 28 | ||
hash.Add("8") | ||
|
||
// 27 should now map to 8. | ||
testCases["27"] = "8" | ||
|
||
for k, v := range testCases { | ||
if hash.Get(k) != v { | ||
t.Errorf("Asking for %s, should have yielded %s", k, v) | ||
} | ||
} | ||
|
||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,109 @@ | ||
package geecache | ||
|
||
import ( | ||
"sync" | ||
) | ||
|
||
// A Group is a cache namespace and associated data loaded spread over | ||
type Group struct { | ||
name string | ||
getter Getter | ||
mainCache cache | ||
peers PeerPicker | ||
peersOnce sync.Once | ||
} | ||
|
||
// A Getter loads data for a key. | ||
type Getter interface { | ||
Get(key string) ([]byte, error) | ||
} | ||
|
||
// A GetterFunc implements Getter with a function. | ||
type GetterFunc func(key string) ([]byte, error) | ||
|
||
// Get implements Getter interface function | ||
func (f GetterFunc) Get(key string) ([]byte, error) { | ||
return f(key) | ||
} | ||
|
||
var ( | ||
mu sync.RWMutex | ||
groups = make(map[string]*Group) | ||
) | ||
|
||
// NewGroup create a new instance of Group | ||
func NewGroup(name string, cacheBytes int64, getter Getter) *Group { | ||
if getter == nil { | ||
panic("nil Getter") | ||
} | ||
mu.Lock() | ||
defer mu.Unlock() | ||
g := &Group{ | ||
name: name, | ||
getter: getter, | ||
mainCache: cache{cacheBytes: cacheBytes}, | ||
} | ||
groups[name] = g | ||
return g | ||
} | ||
|
||
// GetGroup returns the named group previously created with NewGroup, or | ||
// nil if there's no such group. | ||
func GetGroup(name string) *Group { | ||
mu.RLock() | ||
g := groups[name] | ||
mu.RUnlock() | ||
return g | ||
} | ||
|
||
// Get value for a key from cache | ||
func (g *Group) Get(key string) (ByteView, error) { | ||
g.peersOnce.Do(func() { | ||
g.peers = getPeers() | ||
}) | ||
if v, ok := g.mainCache.get(key); ok { | ||
return v, nil | ||
} | ||
|
||
return g.load(key) | ||
} | ||
|
||
func cloneBytes(b []byte) []byte { | ||
c := make([]byte, len(b)) | ||
copy(c, b) | ||
return c | ||
} | ||
|
||
func (g *Group) load(key string) (value ByteView, err error) { | ||
if peer, ok := g.peers.PickPeer(key); ok { | ||
value, err = g.getFromPeer(peer, key) | ||
if err == nil { | ||
return value, nil | ||
} | ||
} | ||
|
||
return g.getLocally(key) | ||
} | ||
|
||
func (g *Group) populateCache(key string, value ByteView) { | ||
g.mainCache.add(key, value) | ||
} | ||
|
||
func (g *Group) getLocally(key string) (ByteView, error) { | ||
bytes, err := g.getter.Get(key) | ||
if err != nil { | ||
return ByteView{}, err | ||
|
||
} | ||
value := ByteView{b: cloneBytes(bytes)} | ||
g.populateCache(key, value) | ||
return value, nil | ||
} | ||
|
||
func (g *Group) getFromPeer(peer PeerGetter, key string) (ByteView, error) { | ||
bytes, err := peer.Get(g.name, key) | ||
if err != nil { | ||
return ByteView{}, err | ||
} | ||
return ByteView{b: bytes}, nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
package geecache | ||
|
||
import ( | ||
"fmt" | ||
"log" | ||
"testing" | ||
) | ||
|
||
var db = map[string]string{ | ||
"Tom": "630", | ||
"Jack": "589", | ||
"Sam": "567", | ||
} | ||
|
||
func TestGet(t *testing.T) { | ||
gee := NewGroup("scores", 2<<10, GetterFunc( | ||
func(key string) ([]byte, error) { | ||
log.Println("[group scores] search key", key) | ||
if v, ok := db[key]; ok { | ||
return []byte(v), nil | ||
} | ||
return nil, fmt.Errorf("%s not exist", key) | ||
})) | ||
|
||
for k, v := range db { | ||
view, err := gee.Get(k) | ||
if err != nil || view.String() != v { | ||
t.Fatal("failed to get value of Tom") | ||
} | ||
} | ||
|
||
if view, err := gee.Get("unknown"); err == nil { | ||
t.Fatalf("the value of unknow should be empty, but %s got", view) | ||
} | ||
} | ||
|
||
func TestGetGroup(t *testing.T) { | ||
groupName := "scores" | ||
NewGroup(groupName, 2<<10, GetterFunc( | ||
func(key string) (bytes []byte, err error) { return })) | ||
if group := GetGroup(groupName); group == nil || group.name != groupName { | ||
t.Fatalf("group %s not exist", groupName) | ||
} | ||
|
||
if group := GetGroup(groupName + "111"); group != nil { | ||
t.Fatalf("expect nil, but %s got", group.name) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
module geecache | ||
|
||
go 1.13 |
Oops, something went wrong.