Skip to content

Commit

Permalink
chore: generalize persisted lists in ledger (algorand#5741)
Browse files Browse the repository at this point in the history
  • Loading branch information
algorandskiy authored Sep 18, 2023
1 parent ea20873 commit 3ff5722
Show file tree
Hide file tree
Showing 13 changed files with 344 additions and 1,279 deletions.
17 changes: 9 additions & 8 deletions ledger/lruaccts.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/algorand/go-algorand/data/basics"
"github.com/algorand/go-algorand/ledger/store/trackerdb"
"github.com/algorand/go-algorand/logging"
"github.com/algorand/go-algorand/util"
)

// lruAccounts provides a storage class for the most recently used accounts data.
Expand All @@ -28,10 +29,10 @@ import (
type lruAccounts struct {
// accountsList contain the list of persistedAccountData, where the front ones are the most "fresh"
// and the ones on the back are the oldest.
accountsList *persistedAccountDataList
accountsList *util.List[*trackerdb.PersistedAccountData]
// accounts provides fast access to the various elements in the list by using the account address
// if lruAccounts is set with pendingWrites 0, then accounts is nil
accounts map[basics.Address]*persistedAccountDataListNode
accounts map[basics.Address]*util.ListNode[*trackerdb.PersistedAccountData]
// pendingAccounts are used as a way to avoid taking a write-lock. When the caller needs to "materialize" these,
// it would call flushPendingWrites and these would be merged into the accounts/accountsList
// if lruAccounts is set with pendingWrites 0, then pendingAccounts is nil
Expand All @@ -50,8 +51,8 @@ type lruAccounts struct {
// thread locking semantics : write lock
func (m *lruAccounts) init(log logging.Logger, pendingWrites int, pendingWritesWarnThreshold int) {
if pendingWrites > 0 {
m.accountsList = newPersistedAccountList().allocateFreeNodes(pendingWrites)
m.accounts = make(map[basics.Address]*persistedAccountDataListNode, pendingWrites)
m.accountsList = util.NewList[*trackerdb.PersistedAccountData]().AllocateFreeNodes(pendingWrites)
m.accounts = make(map[basics.Address]*util.ListNode[*trackerdb.PersistedAccountData], pendingWrites)
m.pendingAccounts = make(chan trackerdb.PersistedAccountData, pendingWrites)
m.notFound = make(map[basics.Address]struct{}, pendingWrites)
m.pendingNotFound = make(chan basics.Address, pendingWrites)
Expand Down Expand Up @@ -141,10 +142,10 @@ func (m *lruAccounts) write(acctData trackerdb.PersistedAccountData) {
// we update with a newer version.
el.Value = &acctData
}
m.accountsList.moveToFront(el)
m.accountsList.MoveToFront(el)
} else {
// new entry.
m.accounts[acctData.Addr] = m.accountsList.pushFront(&acctData)
m.accounts[acctData.Addr] = m.accountsList.PushFront(&acctData)
}
}

Expand All @@ -159,9 +160,9 @@ func (m *lruAccounts) prune(newSize int) (removed int) {
if len(m.accounts) <= newSize {
break
}
back := m.accountsList.back()
back := m.accountsList.Back()
delete(m.accounts, back.Value.Addr)
m.accountsList.remove(back)
m.accountsList.Remove(back)
removed++
}

Expand Down
17 changes: 9 additions & 8 deletions ledger/lrukv.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package ledger
import (
"github.com/algorand/go-algorand/ledger/store/trackerdb"
"github.com/algorand/go-algorand/logging"
"github.com/algorand/go-algorand/util"
)

//msgp:ignore cachedKVData
Expand All @@ -35,11 +36,11 @@ type cachedKVData struct {
type lruKV struct {
// kvList contain the list of persistedKVData, where the front ones are the most "fresh"
// and the ones on the back are the oldest.
kvList *persistedKVDataList
kvList *util.List[*cachedKVData]

// kvs provides fast access to the various elements in the list by using the key
// if lruKV is set with pendingWrites 0, then kvs is nil
kvs map[string]*persistedKVDataListNode
kvs map[string]*util.ListNode[*cachedKVData]

// pendingKVs are used as a way to avoid taking a write-lock. When the caller needs to "materialize" these,
// it would call flushPendingWrites and these would be merged into the kvs/kvList
Expand All @@ -57,8 +58,8 @@ type lruKV struct {
// thread locking semantics : write lock
func (m *lruKV) init(log logging.Logger, pendingWrites int, pendingWritesWarnThreshold int) {
if pendingWrites > 0 {
m.kvList = newPersistedKVList().allocateFreeNodes(pendingWrites)
m.kvs = make(map[string]*persistedKVDataListNode, pendingWrites)
m.kvList = util.NewList[*cachedKVData]().AllocateFreeNodes(pendingWrites)
m.kvs = make(map[string]*util.ListNode[*cachedKVData], pendingWrites)
m.pendingKVs = make(chan cachedKVData, pendingWrites)
}
m.log = log
Expand Down Expand Up @@ -116,10 +117,10 @@ func (m *lruKV) write(kvData trackerdb.PersistedKVData, key string) {
// we update with a newer version.
el.Value = &cachedKVData{PersistedKVData: kvData, key: key}
}
m.kvList.moveToFront(el)
m.kvList.MoveToFront(el)
} else {
// new entry.
m.kvs[key] = m.kvList.pushFront(&cachedKVData{PersistedKVData: kvData, key: key})
m.kvs[key] = m.kvList.PushFront(&cachedKVData{PersistedKVData: kvData, key: key})
}
}

Expand All @@ -134,9 +135,9 @@ func (m *lruKV) prune(newSize int) (removed int) {
if len(m.kvs) <= newSize {
break
}
back := m.kvList.back()
back := m.kvList.Back()
delete(m.kvs, back.Value.key)
m.kvList.remove(back)
m.kvList.Remove(back)
removed++
}
return
Expand Down
17 changes: 9 additions & 8 deletions ledger/lruonlineaccts.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/algorand/go-algorand/data/basics"
"github.com/algorand/go-algorand/ledger/store/trackerdb"
"github.com/algorand/go-algorand/logging"
"github.com/algorand/go-algorand/util"
)

// lruAccounts provides a storage class for the most recently used accounts data.
Expand All @@ -28,10 +29,10 @@ import (
type lruOnlineAccounts struct {
// accountsList contain the list of persistedAccountData, where the front ones are the most "fresh"
// and the ones on the back are the oldest.
accountsList *persistedOnlineAccountDataList
accountsList *util.List[*trackerdb.PersistedOnlineAccountData]
// accounts provides fast access to the various elements in the list by using the account address
// if lruOnlineAccounts is set with pendingWrites 0, then accounts is nil
accounts map[basics.Address]*persistedOnlineAccountDataListNode
accounts map[basics.Address]*util.ListNode[*trackerdb.PersistedOnlineAccountData]
// pendingAccounts are used as a way to avoid taking a write-lock. When the caller needs to "materialize" these,
// it would call flushPendingWrites and these would be merged into the accounts/accountsList
// if lruOnlineAccounts is set with pendingWrites 0, then pendingAccounts is nil
Expand All @@ -46,8 +47,8 @@ type lruOnlineAccounts struct {
// thread locking semantics : write lock
func (m *lruOnlineAccounts) init(log logging.Logger, pendingWrites int, pendingWritesWarnThreshold int) {
if pendingWrites > 0 {
m.accountsList = newPersistedOnlineAccountList().allocateFreeNodes(pendingWrites)
m.accounts = make(map[basics.Address]*persistedOnlineAccountDataListNode, pendingWrites)
m.accountsList = util.NewList[*trackerdb.PersistedOnlineAccountData]().AllocateFreeNodes(pendingWrites)
m.accounts = make(map[basics.Address]*util.ListNode[*trackerdb.PersistedOnlineAccountData], pendingWrites)
m.pendingAccounts = make(chan trackerdb.PersistedOnlineAccountData, pendingWrites)
}
m.log = log
Expand Down Expand Up @@ -105,10 +106,10 @@ func (m *lruOnlineAccounts) write(acctData trackerdb.PersistedOnlineAccountData)
// we update with a newer version.
el.Value = &acctData
}
m.accountsList.moveToFront(el)
m.accountsList.MoveToFront(el)
} else {
// new entry.
m.accounts[acctData.Addr] = m.accountsList.pushFront(&acctData)
m.accounts[acctData.Addr] = m.accountsList.PushFront(&acctData)
}
}

Expand All @@ -123,9 +124,9 @@ func (m *lruOnlineAccounts) prune(newSize int) (removed int) {
if len(m.accounts) <= newSize {
break
}
back := m.accountsList.back()
back := m.accountsList.Back()
delete(m.accounts, back.Value.Addr)
m.accountsList.remove(back)
m.accountsList.Remove(back)
removed++
}
return
Expand Down
17 changes: 9 additions & 8 deletions ledger/lruresources.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/algorand/go-algorand/data/basics"
"github.com/algorand/go-algorand/ledger/store/trackerdb"
"github.com/algorand/go-algorand/logging"
"github.com/algorand/go-algorand/util"
)

//msgp:ignore cachedResourceData
Expand All @@ -35,11 +36,11 @@ type cachedResourceData struct {
type lruResources struct {
// resourcesList contain the list of persistedResourceData, where the front ones are the most "fresh"
// and the ones on the back are the oldest.
resourcesList *persistedResourcesDataList
resourcesList *util.List[*cachedResourceData]

// resources provides fast access to the various elements in the list by using the account address
// if lruResources is set with pendingWrites 0, then resources is nil
resources map[accountCreatable]*persistedResourcesDataListNode
resources map[accountCreatable]*util.ListNode[*cachedResourceData]

// pendingResources are used as a way to avoid taking a write-lock. When the caller needs to "materialize" these,
// it would call flushPendingWrites and these would be merged into the resources/resourcesList
Expand All @@ -61,8 +62,8 @@ type lruResources struct {
// thread locking semantics : write lock
func (m *lruResources) init(log logging.Logger, pendingWrites int, pendingWritesWarnThreshold int) {
if pendingWrites > 0 {
m.resourcesList = newPersistedResourcesList().allocateFreeNodes(pendingWrites)
m.resources = make(map[accountCreatable]*persistedResourcesDataListNode, pendingWrites)
m.resourcesList = util.NewList[*cachedResourceData]().AllocateFreeNodes(pendingWrites)
m.resources = make(map[accountCreatable]*util.ListNode[*cachedResourceData], pendingWrites)
m.pendingResources = make(chan cachedResourceData, pendingWrites)
m.notFound = make(map[accountCreatable]struct{}, pendingWrites)
m.pendingNotFound = make(chan accountCreatable, pendingWrites)
Expand Down Expand Up @@ -163,10 +164,10 @@ func (m *lruResources) write(resData trackerdb.PersistedResourcesData, addr basi
// we update with a newer version.
el.Value = &cachedResourceData{PersistedResourcesData: resData, address: addr}
}
m.resourcesList.moveToFront(el)
m.resourcesList.MoveToFront(el)
} else {
// new entry.
m.resources[accountCreatable{address: addr, index: resData.Aidx}] = m.resourcesList.pushFront(&cachedResourceData{PersistedResourcesData: resData, address: addr})
m.resources[accountCreatable{address: addr, index: resData.Aidx}] = m.resourcesList.PushFront(&cachedResourceData{PersistedResourcesData: resData, address: addr})
}
}

Expand All @@ -181,9 +182,9 @@ func (m *lruResources) prune(newSize int) (removed int) {
if len(m.resources) <= newSize {
break
}
back := m.resourcesList.back()
back := m.resourcesList.Back()
delete(m.resources, accountCreatable{address: back.Value.address, index: back.Value.Aidx})
m.resourcesList.remove(back)
m.resourcesList.Remove(back)
removed++
}

Expand Down
146 changes: 0 additions & 146 deletions ledger/persistedaccts_list.go

This file was deleted.

Loading

0 comments on commit 3ff5722

Please sign in to comment.