Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

bloom: don't calc checksum at startup #12967

Merged
merged 4 commits into from
Dec 3, 2024
Merged

bloom: don't calc checksum at startup #12967

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d1bab1f
save
AskAlexSharov Dec 3, 2024
a09be2f
save
AskAlexSharov Dec 3, 2024
9e796a8
save
AskAlexSharov Dec 3, 2024
21e94db
save
AskAlexSharov Dec 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions erigon-lib/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@ module github.com/erigontech/erigon-lib

go 1.22.0

replace (
github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-38
github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.9
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So we still need your fork of bloomfilter ?

)

require (
github.com/erigontech/erigon-snapshot v1.3.1-0.20241023024258-f64407a77e8e
github.com/erigontech/interfaces v0.0.0-20241116035842-5d396f10468e
Expand Down Expand Up @@ -154,8 +159,3 @@ require (
rsc.io/tmplfunc v0.0.3 // indirect
zombiezen.com/go/sqlite v0.13.1 // indirect
)

replace (
github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-38
github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.8
)
4 changes: 2 additions & 2 deletions erigon-lib/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797/go.mod h1:sXBiorCo8c46JlQV3oX
crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4=
filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmGfU=
filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns=
github.com/AskAlexSharov/bloomfilter/v2 v2.0.8 h1:eRExAhnCcGHKC4/s8bpbYHJTQfOtn/urU/CYXNx2Q+8=
github.com/AskAlexSharov/bloomfilter/v2 v2.0.8/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA=
github.com/AskAlexSharov/bloomfilter/v2 v2.0.9 h1:BuZqNjRlYmcXJIsI7nrIkejYMz9mgFi7ZsNFCbSPpaI=
github.com/AskAlexSharov/bloomfilter/v2 v2.0.9/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
Expand Down
36 changes: 19 additions & 17 deletions erigon-lib/state/existence_filter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package state

import (
"bufio"
"fmt"
"hash"
"os"
Expand Down Expand Up @@ -144,26 +145,27 @@ func OpenExistenceFilter(filePath string) (exFilder *ExistenceFilter, err error)
if !exists {
return nil, fmt.Errorf("file doesn't exists: %s", fileName)
}
{
ff, err := os.Open(filePath)
if err != nil {
return nil, err
}
defer ff.Close()
stat, err := ff.Stat()
if err != nil {
return nil, err
}
idx.empty = stat.Size() == 0

f, err := os.Open(filePath)
if err != nil {
return nil, err
}
defer f.Close()
stat, err := f.Stat()
if err != nil {
return nil, err
}
idx.empty = stat.Size() == 0
if idx.empty {
return idx, nil
}

if !idx.empty {
var err error
idx.filter, _, err = bloomfilter.ReadFile(filePath)
if err != nil {
return nil, fmt.Errorf("OpenExistenceFilter: %w, %s", err, fileName)
}
filter := new(bloomfilter.Filter)
_, err = filter.UnmarshalFromReaderNoVerify(bufio.NewReaderSize(f, 1*1024*1024))
if err != nil {
return nil, fmt.Errorf("OpenExistenceFilter: %w, %s", err, fileName)
}
idx.filter = filter
return idx, nil
}
func (b *ExistenceFilter) Close() {
Expand Down
14 changes: 7 additions & 7 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,20 @@ module github.com/erigontech/erigon

go 1.22.5

replace github.com/erigontech/erigon-lib => ./erigon-lib

replace (
github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-38
github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.9
)

require (
github.com/erigontech/erigonwatch v0.0.0-20240718131902-b6576bde1116
github.com/erigontech/mdbx-go v0.38.4
github.com/erigontech/secp256k1 v1.1.0
github.com/erigontech/silkworm-go v0.18.0
)

replace github.com/erigontech/erigon-lib => ./erigon-lib

require (
gfx.cafe/util/go/generic v0.0.0-20230721185457-c559e86c829c
github.com/99designs/gqlgen v0.17.56
Expand Down Expand Up @@ -287,8 +292,3 @@ require (
rsc.io/tmplfunc v0.0.3 // indirect
zombiezen.com/go/sqlite v0.13.1 // indirect
)

replace (
github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-38
github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.8
)
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ gfx.cafe/util/go/generic v0.0.0-20230721185457-c559e86c829c/go.mod h1:WvSX4JsCRB
git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
github.com/99designs/gqlgen v0.17.56 h1:+J42ARAHvnysH6klO9Wq+tCsGF32cpAgU3SyF0VRJtI=
github.com/99designs/gqlgen v0.17.56/go.mod h1:rmB6vLvtL8uf9F9w0/irJ5alBkD8DJvj35ET31BKbtY=
github.com/AskAlexSharov/bloomfilter/v2 v2.0.8 h1:eRExAhnCcGHKC4/s8bpbYHJTQfOtn/urU/CYXNx2Q+8=
github.com/AskAlexSharov/bloomfilter/v2 v2.0.8/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA=
github.com/AskAlexSharov/bloomfilter/v2 v2.0.9 h1:BuZqNjRlYmcXJIsI7nrIkejYMz9mgFi7ZsNFCbSPpaI=
github.com/AskAlexSharov/bloomfilter/v2 v2.0.9/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/Giulio2002/bls v0.0.0-20241116091023-2ddcc8954ec0 h1:6DVEDL29nd7f2GoHZIA9rjpW90gYeNE3x5aUadOgTB4=
Expand Down
Loading