Skip to content

Commit

Permalink
add test
Browse files Browse the repository at this point in the history
  • Loading branch information
@chris124567 @ChrisSchinnerl
chris124567 authored and ChrisSchinnerl committed Nov 19, 2024
1 parent e50554e commit e28f25f
Show file tree
Hide file tree
Showing 4 changed files with 144 additions and 29 deletions.
12 changes: 8 additions & 4 deletions explorer/scan.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,8 @@ func (e *Explorer) scanV2Host(locator geoip.Locator, host Host) (HostScan, error
ctx, cancel := context.WithTimeout(e.ctx, e.scanCfg.Timeout)
defer cancel()

transport, err := crhpv4.DialSiaMux(ctx, host.V2NetAddresses[0].Address, host.PublicKey)
addr := host.V2NetAddresses[0].Address
transport, err := crhpv4.DialSiaMux(ctx, addr, host.PublicKey)
if err != nil {
return HostScan{}, fmt.Errorf("failed to dial host: %w", err)
}
Expand All @@ -129,7 +130,7 @@ func (e *Explorer) scanV2Host(locator geoip.Locator, host Host) (HostScan, error
return HostScan{}, fmt.Errorf("failed to get host settings: %w", err)
}

hostIP, _, err := net.SplitHostPort(host.NetAddress)
hostIP, _, err := net.SplitHostPort(addr)
if err != nil {
return HostScan{}, fmt.Errorf("scanHost: failed to parse net address: %w", err)
}
Expand Down Expand Up @@ -173,10 +174,13 @@ func (e *Explorer) addHostScans(hosts chan Host) {
}

var scan HostScan
var addr string
var err error
if len(host.V2NetAddresses) == 0 {
addr = host.NetAddress
scan, err = e.scanV1Host(locator, host)
} else {
addr = host.V2NetAddresses[0].Address
scan, err = e.scanV2Host(locator, host)
}
if err != nil {
Expand All @@ -185,11 +189,11 @@ func (e *Explorer) addHostScans(hosts chan Host) {
Success: false,
Timestamp: types.CurrentTimestamp(),
})
e.log.Debug("Scanning host failed", zap.String("addr", host.NetAddress), zap.Stringer("pk", host.PublicKey), zap.Error(err))
e.log.Debug("Scanning host failed", zap.String("addr", addr), zap.Stringer("pk", host.PublicKey), zap.Error(err))
continue
}

e.log.Debug("Scanning host succeeded", zap.String("addr", host.NetAddress), zap.Stringer("pk", host.PublicKey))
e.log.Debug("Scanning host succeeded", zap.String("addr", addr), zap.Stringer("pk", host.PublicKey))
scans = append(scans, scan)
}

Expand Down
8 changes: 6 additions & 2 deletions persist/sqlite/addresses.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ func (st *Store) Hosts(pks []types.PublicKey) (result []explorer.Host, err error
encoded = append(encoded, encode(pk))
}

rows, err := tx.Query(`SELECT public_key,net_address,country_code,known_since,last_scan,last_scan_successful,last_announcement,total_scans,successful_interactions,failed_interactions,settings_accepting_contracts,settings_max_download_batch_size,settings_max_duration,settings_max_revise_batch_size,settings_net_address,settings_remaining_storage,settings_sector_size,settings_total_storage,settings_address,settings_window_size,settings_collateral,settings_max_collateral,settings_base_rpc_price,settings_contract_price,settings_download_bandwidth_price,settings_sector_access_price,settings_storage_price,settings_upload_bandwidth_price,settings_ephemeral_account_expiry,settings_max_ephemeral_account_balance,settings_revision_number,settings_version,settings_release,settings_sia_mux_port,price_table_uid,price_table_validity,price_table_host_block_height,price_table_update_price_table_cost,price_table_account_balance_cost,price_table_fund_account_cost,price_table_latest_revision_cost,price_table_subscription_memory_cost,price_table_subscription_notification_cost,price_table_init_base_cost,price_table_memory_time_cost,price_table_download_bandwidth_cost,price_table_upload_bandwidth_cost,price_table_drop_sectors_base_cost,price_table_drop_sectors_unit_cost,price_table_has_sector_base_cost,price_table_read_base_cost,price_table_read_length_cost,price_table_renew_contract_cost,price_table_revision_base_cost,price_table_swap_sector_base_cost,price_table_write_base_cost,price_table_write_length_cost,price_table_write_store_cost,price_table_txn_fee_min_recommended,price_table_txn_fee_max_recommended,price_table_contract_price,price_table_collateral_cost,price_table_max_collateral,price_table_max_duration,price_table_window_size,price_table_registry_entries_left,price_table_registry_entries_total FROM host_info WHERE public_key IN (`+queryPlaceHolders(len(pks))+`)`, encoded...)
rows, err := tx.Query(`SELECT public_key,net_address,country_code,known_since,last_scan,last_scan_successful,last_announcement,total_scans,successful_interactions,failed_interactions,settings_accepting_contracts,settings_max_download_batch_size,settings_max_duration,settings_max_revise_batch_size,settings_net_address,settings_remaining_storage,settings_sector_size,settings_total_storage,settings_address,settings_window_size,settings_collateral,settings_max_collateral,settings_base_rpc_price,settings_contract_price,settings_download_bandwidth_price,settings_sector_access_price,settings_storage_price,settings_upload_bandwidth_price,settings_ephemeral_account_expiry,settings_max_ephemeral_account_balance,settings_revision_number,settings_version,settings_release,settings_sia_mux_port,price_table_uid,price_table_validity,price_table_host_block_height,price_table_update_price_table_cost,price_table_account_balance_cost,price_table_fund_account_cost,price_table_latest_revision_cost,price_table_subscription_memory_cost,price_table_subscription_notification_cost,price_table_init_base_cost,price_table_memory_time_cost,price_table_download_bandwidth_cost,price_table_upload_bandwidth_cost,price_table_drop_sectors_base_cost,price_table_drop_sectors_unit_cost,price_table_has_sector_base_cost,price_table_read_base_cost,price_table_read_length_cost,price_table_renew_contract_cost,price_table_revision_base_cost,price_table_swap_sector_base_cost,price_table_write_base_cost,price_table_write_length_cost,price_table_write_store_cost,price_table_txn_fee_min_recommended,price_table_txn_fee_max_recommended,price_table_contract_price,price_table_collateral_cost,price_table_max_collateral,price_table_max_duration,price_table_window_size,price_table_registry_entries_left,price_table_registry_entries_total,rhp4_settings_protocol_version,rhp4_settings_release,rhp4_settings_wallet_address,rhp4_settings_accepting_contracts,rhp4_settings_max_collateral,rhp4_settings_max_collateral_duration,rhp4_settings_max_sector_duration,rhp4_settings_max_sector_batch_size,rhp4_settings_remaining_storage,rhp4_settings_total_storage,rhp4_prices_contract_price,rhp4_prices_collateral_price,rhp4_prices_storage_price,rhp4_prices_ingress_price,rhp4_prices_egress_price,rhp4_prices_free_sector_price,rhp4_prices_tip_height,rhp4_prices_valid_until,rhp4_prices_signature FROM host_info WHERE public_key IN (`+queryPlaceHolders(len(pks))+`)`, encoded...)
if err != nil {
return err
}
Expand All @@ -104,10 +104,14 @@ func (st *Store) Hosts(pks []types.PublicKey) (result []explorer.Host, err error
for rows.Next() {
if err := func() error {
var host explorer.Host
var protocolVersion []uint8

s, p := &host.Settings, &host.PriceTable
if err := rows.Scan(decode(&host.PublicKey), &host.NetAddress, &host.CountryCode, decode(&host.KnownSince), decode(&host.LastScan), &host.LastScanSuccessful, decode(&host.LastAnnouncement), &host.TotalScans, &host.SuccessfulInteractions, &host.FailedInteractions, &s.AcceptingContracts, decode(&s.MaxDownloadBatchSize), decode(&s.MaxDuration), decode(&s.MaxReviseBatchSize), &s.NetAddress, decode(&s.RemainingStorage), decode(&s.SectorSize), decode(&s.TotalStorage), decode(&s.Address), decode(&s.WindowSize), decode(&s.Collateral), decode(&s.MaxCollateral), decode(&s.BaseRPCPrice), decode(&s.ContractPrice), decode(&s.DownloadBandwidthPrice), decode(&s.SectorAccessPrice), decode(&s.StoragePrice), decode(&s.UploadBandwidthPrice), &s.EphemeralAccountExpiry, decode(&s.MaxEphemeralAccountBalance), decode(&s.RevisionNumber), &s.Version, &s.Release, &s.SiaMuxPort, decode(&p.UID), &p.Validity, decode(&p.HostBlockHeight), decode(&p.UpdatePriceTableCost), decode(&p.AccountBalanceCost), decode(&p.FundAccountCost), decode(&p.LatestRevisionCost), decode(&p.SubscriptionMemoryCost), decode(&p.SubscriptionNotificationCost), decode(&p.InitBaseCost), decode(&p.MemoryTimeCost), decode(&p.DownloadBandwidthCost), decode(&p.UploadBandwidthCost), decode(&p.DropSectorsBaseCost), decode(&p.DropSectorsUnitCost), decode(&p.HasSectorBaseCost), decode(&p.ReadBaseCost), decode(&p.ReadLengthCost), decode(&p.RenewContractCost), decode(&p.RevisionBaseCost), decode(&p.SwapSectorBaseCost), decode(&p.WriteBaseCost), decode(&p.WriteLengthCost), decode(&p.WriteStoreCost), decode(&p.TxnFeeMinRecommended), decode(&p.TxnFeeMaxRecommended), decode(&p.ContractPrice), decode(&p.CollateralCost), decode(&p.MaxCollateral), decode(&p.MaxDuration), decode(&p.WindowSize), decode(&p.RegistryEntriesLeft), decode(&p.RegistryEntriesTotal)); err != nil {
sV4, pV4 := &host.RHPV4Settings, &host.RHPV4Settings.Prices
if err := rows.Scan(decode(&host.PublicKey), &host.NetAddress, &host.CountryCode, decode(&host.KnownSince), decode(&host.LastScan), &host.LastScanSuccessful, decode(&host.LastAnnouncement), &host.TotalScans, &host.SuccessfulInteractions, &host.FailedInteractions, &s.AcceptingContracts, decode(&s.MaxDownloadBatchSize), decode(&s.MaxDuration), decode(&s.MaxReviseBatchSize), &s.NetAddress, decode(&s.RemainingStorage), decode(&s.SectorSize), decode(&s.TotalStorage), decode(&s.Address), decode(&s.WindowSize), decode(&s.Collateral), decode(&s.MaxCollateral), decode(&s.BaseRPCPrice), decode(&s.ContractPrice), decode(&s.DownloadBandwidthPrice), decode(&s.SectorAccessPrice), decode(&s.StoragePrice), decode(&s.UploadBandwidthPrice), &s.EphemeralAccountExpiry, decode(&s.MaxEphemeralAccountBalance), decode(&s.RevisionNumber), &s.Version, &s.Release, &s.SiaMuxPort, decode(&p.UID), &p.Validity, decode(&p.HostBlockHeight), decode(&p.UpdatePriceTableCost), decode(&p.AccountBalanceCost), decode(&p.FundAccountCost), decode(&p.LatestRevisionCost), decode(&p.SubscriptionMemoryCost), decode(&p.SubscriptionNotificationCost), decode(&p.InitBaseCost), decode(&p.MemoryTimeCost), decode(&p.DownloadBandwidthCost), decode(&p.UploadBandwidthCost), decode(&p.DropSectorsBaseCost), decode(&p.DropSectorsUnitCost), decode(&p.HasSectorBaseCost), decode(&p.ReadBaseCost), decode(&p.ReadLengthCost), decode(&p.RenewContractCost), decode(&p.RevisionBaseCost), decode(&p.SwapSectorBaseCost), decode(&p.WriteBaseCost), decode(&p.WriteLengthCost), decode(&p.WriteStoreCost), decode(&p.TxnFeeMinRecommended), decode(&p.TxnFeeMaxRecommended), decode(&p.ContractPrice), decode(&p.CollateralCost), decode(&p.MaxCollateral), decode(&p.MaxDuration), decode(&p.WindowSize), decode(&p.RegistryEntriesLeft), decode(&p.RegistryEntriesTotal), &protocolVersion, &sV4.Release, decode(&sV4.WalletAddress), &sV4.AcceptingContracts, decode(&sV4.MaxCollateral), decode(&sV4.MaxContractDuration), decode(&sV4.MaxSectorDuration), decode(&sV4.MaxSectorBatchSize), decode(&sV4.RemainingStorage), decode(&sV4.TotalStorage), decode(&pV4.ContractPrice), decode(&pV4.Collateral), decode(&pV4.StoragePrice), decode(&pV4.IngressPrice), decode(&pV4.EgressPrice), decode(&pV4.FreeSectorPrice), decode(&pV4.TipHeight), decode(&pV4.ValidUntil), decode(&pV4.Signature)); err != nil {
return err
}
sV4.ProtocolVersion = [3]uint8(protocolVersion)

v2AddrRows, err := v2AddrStmt.Query(encode(host.PublicKey))
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion persist/sqlite/init.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -565,7 +565,7 @@ CREATE TABLE host_info_v2_netaddresses(
protocol TEXT NOT NULL,
address TEXT NOT NULL,

UNIQUE(public_key, netaddress_order)
PRIMARY KEY(public_key, netaddress_order)
);

CREATE INDEX host_info_v2_netaddresses_public_key ON host_info_v2_netaddresses(public_key);
Expand Down
151 changes: 129 additions & 22 deletions persist/sqlite/scan_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,27 @@ package sqlite
import (
"context"
"errors"
"net"
"path/filepath"
"sort"
"testing"
"time"

"go.sia.tech/core/consensus"
"go.sia.tech/core/gateway"
proto4 "go.sia.tech/core/rhp/v4"
"go.sia.tech/core/types"
"go.sia.tech/coreutils"
"go.sia.tech/coreutils/chain"
crhpv4 "go.sia.tech/coreutils/rhp/v4"
"go.sia.tech/coreutils/syncer"
ctestutil "go.sia.tech/coreutils/testutil"
"go.sia.tech/coreutils/wallet"
"go.sia.tech/explored/config"
"go.sia.tech/explored/explorer"
"go.sia.tech/explored/internal/testutil"
"lukechampine.com/frand"

"go.uber.org/zap"
"go.uber.org/zap/zaptest"
)

Expand Down Expand Up @@ -43,33 +51,114 @@ func syncDB(t *testing.T, db *Store, cm *chain.Manager) {
}
}

func startTestNode(tb testing.TB, n *consensus.Network, genesis types.Block) (*chain.Manager, *syncer.Syncer, *wallet.SingleAddressWallet) {
db, tipstate, err := chain.NewDBStore(chain.NewMemDB(), n, genesis)
if err != nil {
tb.Fatal(err)
}
cm := chain.NewManager(db, tipstate)

syncerListener, err := net.Listen("tcp", ":0")
if err != nil {
tb.Fatal(err)
}
tb.Cleanup(func() { syncerListener.Close() })

s := syncer.New(syncerListener, cm, ctestutil.NewMemPeerStore(), gateway.Header{
GenesisID: genesis.ID(),
UniqueID: gateway.GenerateUniqueID(),
NetAddress: "localhost:1234",
})
go s.Run(context.Background())
tb.Cleanup(func() { s.Close() })

ws := ctestutil.NewEphemeralWalletStore()
w, err := wallet.NewSingleAddressWallet(types.GeneratePrivateKey(), cm, ws)
if err != nil {
tb.Fatal(err)
}
tb.Cleanup(func() { w.Close() })

reorgCh := make(chan struct{}, 1)
tb.Cleanup(func() { close(reorgCh) })

go func() {
for range reorgCh {
reverted, applied, err := cm.UpdatesSince(w.Tip(), 1000)
if err != nil {
tb.Error(err)
}

err = ws.UpdateChainState(func(tx wallet.UpdateTx) error {
return w.UpdateChainState(tx, reverted, applied)
})
if err != nil {
tb.Error(err)
}
}
}()

stop := cm.OnReorg(func(index types.ChainIndex) {
select {
case reorgCh <- struct{}{}:
default:
}
})
tb.Cleanup(stop)

return cm, s, w
}

func testRenterHostPair(tb testing.TB, hostKey types.PrivateKey, cm crhpv4.ChainManager, s crhpv4.Syncer, w crhpv4.Wallet, c crhpv4.Contractor, sr crhpv4.Settings, ss crhpv4.Sectors, log *zap.Logger) string {
rs := crhpv4.NewServer(hostKey, cm, s, c, w, sr, ss, crhpv4.WithContractProofWindowBuffer(10), crhpv4.WithPriceTableValidity(2*time.Minute))
return ctestutil.ServeSiaMux(tb, rs, log.Named("siamux"))
}

func TestScan(t *testing.T) {
if testing.Short() {
t.Skip()
}

log := zaptest.NewLogger(t)
dir := t.TempDir()

db, err := OpenDatabase(filepath.Join(dir, "explored.sqlite3"), log.Named("sqlite3"))
if err != nil {
t.Fatal(err)
}
defer db.Close()

bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db"))
if err != nil {
t.Fatal(err)
}
defer bdb.Close()

network, genesisBlock := ctestutil.Network()

store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock)
if err != nil {
t.Fatal(err)
}
cm, s, w := startTestNode(t, network, genesisBlock)

sr := ctestutil.NewEphemeralSettingsReporter()
sr.Update(proto4.HostSettings{
ProtocolVersion: [3]uint8{1, 2, 3},
Release: "test",
AcceptingContracts: true,
WalletAddress: w.Address(),
MaxCollateral: types.Siacoins(10000),
MaxContractDuration: 1000,
MaxSectorDuration: 3 * 144,
MaxSectorBatchSize: 100,
RemainingStorage: 100 * proto4.SectorSize,
TotalStorage: 100 * proto4.SectorSize,
Prices: proto4.HostPrices{
ContractPrice: types.Siacoins(uint32(frand.Uint64n(10000))),
StoragePrice: types.Siacoins(uint32(frand.Uint64n(10000))),
IngressPrice: types.Siacoins(uint32(frand.Uint64n(10000))),
EgressPrice: types.Siacoins(uint32(frand.Uint64n(10000))),
Collateral: types.Siacoins(uint32(frand.Uint64n(10000))),
},
})
ss := ctestutil.NewEphemeralSectorStore()
c := ctestutil.NewEphemeralContractor(cm)

pk0 := types.GeneratePrivateKey()
pubkey0 := pk0.PublicKey()

cm := chain.NewManager(store, genesisState)
v4Addr := testRenterHostPair(t, pk0, cm, s, w, c, sr, ss, zap.NewNop())

cfg := config.Scanner{
Threads: 10,
Expand All @@ -94,6 +183,12 @@ func TestScan(t *testing.T) {

ts := types.CurrentTimestamp()
hosts := []explorer.Host{
{
PublicKey: pubkey0,
V2NetAddresses: []chain.NetAddress{{Protocol: crhpv4.ProtocolTCPSiaMux, Address: v4Addr}},
LastAnnouncement: ts,
KnownSince: ts,
},
{
PublicKey: pubkey1,
NetAddress: "sia1.siahost.ca:9982",
Expand All @@ -115,36 +210,48 @@ func TestScan(t *testing.T) {
}

// explorer won't start scanning till a recent block is mined
b := testutil.MineBlock(genesisState, nil, types.VoidAddress)
b := testutil.MineBlock(cm.TipState(), nil, types.VoidAddress)
if err := cm.AddBlocks([]types.Block{b}); err != nil {
t.Fatal(err)
}

time.Sleep(3 * cfg.Timeout)
time.Sleep(4 * cfg.Timeout)

dbHosts, err := e.Hosts([]types.PublicKey{pubkey1, pubkey2})
dbHosts, err := e.Hosts([]types.PublicKey{pubkey0, pubkey1, pubkey2})
if err != nil {
t.Fatal(err)
}
testutil.Equal(t, "len(dbHosts)", 2, len(dbHosts))
testutil.Equal(t, "len(dbHosts)", 3, len(dbHosts))

sort.Slice(hosts, func(i, j int) bool {
return hosts[i].NetAddress < hosts[j].NetAddress
})
sort.Slice(dbHosts, func(i, j int) bool {
return dbHosts[i].NetAddress < dbHosts[j].NetAddress
})
host1 := dbHosts[0]
testutil.Equal(t, "host1.NetAddress", hosts[0].NetAddress, host1.NetAddress)
testutil.Equal(t, "host1.PublicKey", hosts[0].PublicKey, host1.PublicKey)

host0 := dbHosts[0]
testutil.Equal(t, "host0.NetAddress", hosts[0].NetAddress, host0.NetAddress)
testutil.Equal(t, "host0.PublicKey", hosts[0].PublicKey, host0.PublicKey)
testutil.Equal(t, "host0.TotalScans", 1, host0.TotalScans)
testutil.Equal(t, "host0.SuccessfulInteractions", 1, host0.SuccessfulInteractions)
testutil.Equal(t, "host0.FailedInteractions", 0, host0.FailedInteractions)
testutil.Equal(t, "host0.LastScanSuccessful", true, host0.LastScanSuccessful)
if !host0.RHPV4Settings.AcceptingContracts {
log.Fatal("AcceptingContracts = false on host that's supposed to be active")
}

host1 := dbHosts[1]
testutil.Equal(t, "host1.NetAddress", hosts[1].NetAddress, host1.NetAddress)
testutil.Equal(t, "host1.PublicKey", hosts[1].PublicKey, host1.PublicKey)
testutil.Equal(t, "host1.TotalScans", 1, host1.TotalScans)
testutil.Equal(t, "host1.SuccessfulInteractions", 0, host1.SuccessfulInteractions)
testutil.Equal(t, "host1.FailedInteractions", 1, host1.FailedInteractions)
testutil.Equal(t, "host1.LastScanSuccessful", false, host1.LastScanSuccessful)

host2 := dbHosts[1]
testutil.Equal(t, "host2.NetAddress", hosts[1].NetAddress, host2.NetAddress)
testutil.Equal(t, "host2.PublicKey", hosts[1].PublicKey, host2.PublicKey)
host2 := dbHosts[2]
testutil.Equal(t, "host2.NetAddress", hosts[2].NetAddress, host2.NetAddress)
testutil.Equal(t, "host2.PublicKey", hosts[2].PublicKey, host2.PublicKey)
testutil.Equal(t, "host2.CountryCode", "CA", host2.CountryCode)
testutil.Equal(t, "host2.TotalScans", 1, host2.TotalScans)
testutil.Equal(t, "host2.SuccessfulInteractions", 1, host2.SuccessfulInteractions)
Expand Down

0 comments on commit e28f25f

Please sign in to comment.