diff --git a/explorer/scan.go b/explorer/scan.go index 313537a0..ede7e2ac 100644 --- a/explorer/scan.go +++ b/explorer/scan.go @@ -118,7 +118,8 @@ func (e *Explorer) scanV2Host(locator geoip.Locator, host Host) (HostScan, error ctx, cancel := context.WithTimeout(e.ctx, e.scanCfg.Timeout) defer cancel() - transport, err := crhpv4.DialSiaMux(ctx, host.V2NetAddresses[0].Address, host.PublicKey) + addr := host.V2NetAddresses[0].Address + transport, err := crhpv4.DialSiaMux(ctx, addr, host.PublicKey) if err != nil { return HostScan{}, fmt.Errorf("failed to dial host: %w", err) } @@ -129,7 +130,7 @@ func (e *Explorer) scanV2Host(locator geoip.Locator, host Host) (HostScan, error return HostScan{}, fmt.Errorf("failed to get host settings: %w", err) } - hostIP, _, err := net.SplitHostPort(host.NetAddress) + hostIP, _, err := net.SplitHostPort(addr) if err != nil { return HostScan{}, fmt.Errorf("scanHost: failed to parse net address: %w", err) } @@ -173,10 +174,13 @@ func (e *Explorer) addHostScans(hosts chan Host) { } var scan HostScan + var addr string var err error if len(host.V2NetAddresses) == 0 { + addr = host.NetAddress scan, err = e.scanV1Host(locator, host) } else { + addr = host.V2NetAddresses[0].Address scan, err = e.scanV2Host(locator, host) } if err != nil { @@ -185,11 +189,11 @@ func (e *Explorer) addHostScans(hosts chan Host) { Success: false, Timestamp: types.CurrentTimestamp(), }) - e.log.Debug("Scanning host failed", zap.String("addr", host.NetAddress), zap.Stringer("pk", host.PublicKey), zap.Error(err)) + e.log.Debug("Scanning host failed", zap.String("addr", addr), zap.Stringer("pk", host.PublicKey), zap.Error(err)) continue } - e.log.Debug("Scanning host succeeded", zap.String("addr", host.NetAddress), zap.Stringer("pk", host.PublicKey)) + e.log.Debug("Scanning host succeeded", zap.String("addr", addr), zap.Stringer("pk", host.PublicKey)) scans = append(scans, scan) } diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index f63a47c0..772a3308 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -89,7 +89,7 @@ func (st *Store) Hosts(pks []types.PublicKey) (result []explorer.Host, err error encoded = append(encoded, encode(pk)) } - rows, err := tx.Query(`SELECT public_key,net_address,country_code,known_since,last_scan,last_scan_successful,last_announcement,total_scans,successful_interactions,failed_interactions,settings_accepting_contracts,settings_max_download_batch_size,settings_max_duration,settings_max_revise_batch_size,settings_net_address,settings_remaining_storage,settings_sector_size,settings_total_storage,settings_address,settings_window_size,settings_collateral,settings_max_collateral,settings_base_rpc_price,settings_contract_price,settings_download_bandwidth_price,settings_sector_access_price,settings_storage_price,settings_upload_bandwidth_price,settings_ephemeral_account_expiry,settings_max_ephemeral_account_balance,settings_revision_number,settings_version,settings_release,settings_sia_mux_port,price_table_uid,price_table_validity,price_table_host_block_height,price_table_update_price_table_cost,price_table_account_balance_cost,price_table_fund_account_cost,price_table_latest_revision_cost,price_table_subscription_memory_cost,price_table_subscription_notification_cost,price_table_init_base_cost,price_table_memory_time_cost,price_table_download_bandwidth_cost,price_table_upload_bandwidth_cost,price_table_drop_sectors_base_cost,price_table_drop_sectors_unit_cost,price_table_has_sector_base_cost,price_table_read_base_cost,price_table_read_length_cost,price_table_renew_contract_cost,price_table_revision_base_cost,price_table_swap_sector_base_cost,price_table_write_base_cost,price_table_write_length_cost,price_table_write_store_cost,price_table_txn_fee_min_recommended,price_table_txn_fee_max_recommended,price_table_contract_price,price_table_collateral_cost,price_table_max_collateral,price_table_max_duration,price_table_window_size,price_table_registry_entries_left,price_table_registry_entries_total FROM host_info WHERE public_key IN (`+queryPlaceHolders(len(pks))+`)`, encoded...) + rows, err := tx.Query(`SELECT public_key,net_address,country_code,known_since,last_scan,last_scan_successful,last_announcement,total_scans,successful_interactions,failed_interactions,settings_accepting_contracts,settings_max_download_batch_size,settings_max_duration,settings_max_revise_batch_size,settings_net_address,settings_remaining_storage,settings_sector_size,settings_total_storage,settings_address,settings_window_size,settings_collateral,settings_max_collateral,settings_base_rpc_price,settings_contract_price,settings_download_bandwidth_price,settings_sector_access_price,settings_storage_price,settings_upload_bandwidth_price,settings_ephemeral_account_expiry,settings_max_ephemeral_account_balance,settings_revision_number,settings_version,settings_release,settings_sia_mux_port,price_table_uid,price_table_validity,price_table_host_block_height,price_table_update_price_table_cost,price_table_account_balance_cost,price_table_fund_account_cost,price_table_latest_revision_cost,price_table_subscription_memory_cost,price_table_subscription_notification_cost,price_table_init_base_cost,price_table_memory_time_cost,price_table_download_bandwidth_cost,price_table_upload_bandwidth_cost,price_table_drop_sectors_base_cost,price_table_drop_sectors_unit_cost,price_table_has_sector_base_cost,price_table_read_base_cost,price_table_read_length_cost,price_table_renew_contract_cost,price_table_revision_base_cost,price_table_swap_sector_base_cost,price_table_write_base_cost,price_table_write_length_cost,price_table_write_store_cost,price_table_txn_fee_min_recommended,price_table_txn_fee_max_recommended,price_table_contract_price,price_table_collateral_cost,price_table_max_collateral,price_table_max_duration,price_table_window_size,price_table_registry_entries_left,price_table_registry_entries_total,rhp4_settings_protocol_version,rhp4_settings_release,rhp4_settings_wallet_address,rhp4_settings_accepting_contracts,rhp4_settings_max_collateral,rhp4_settings_max_collateral_duration,rhp4_settings_max_sector_duration,rhp4_settings_max_sector_batch_size,rhp4_settings_remaining_storage,rhp4_settings_total_storage,rhp4_prices_contract_price,rhp4_prices_collateral_price,rhp4_prices_storage_price,rhp4_prices_ingress_price,rhp4_prices_egress_price,rhp4_prices_free_sector_price,rhp4_prices_tip_height,rhp4_prices_valid_until,rhp4_prices_signature FROM host_info WHERE public_key IN (`+queryPlaceHolders(len(pks))+`)`, encoded...) if err != nil { return err } @@ -104,10 +104,14 @@ func (st *Store) Hosts(pks []types.PublicKey) (result []explorer.Host, err error for rows.Next() { if err := func() error { var host explorer.Host + var protocolVersion []uint8 + s, p := &host.Settings, &host.PriceTable - if err := rows.Scan(decode(&host.PublicKey), &host.NetAddress, &host.CountryCode, decode(&host.KnownSince), decode(&host.LastScan), &host.LastScanSuccessful, decode(&host.LastAnnouncement), &host.TotalScans, &host.SuccessfulInteractions, &host.FailedInteractions, &s.AcceptingContracts, decode(&s.MaxDownloadBatchSize), decode(&s.MaxDuration), decode(&s.MaxReviseBatchSize), &s.NetAddress, decode(&s.RemainingStorage), decode(&s.SectorSize), decode(&s.TotalStorage), decode(&s.Address), decode(&s.WindowSize), decode(&s.Collateral), decode(&s.MaxCollateral), decode(&s.BaseRPCPrice), decode(&s.ContractPrice), decode(&s.DownloadBandwidthPrice), decode(&s.SectorAccessPrice), decode(&s.StoragePrice), decode(&s.UploadBandwidthPrice), &s.EphemeralAccountExpiry, decode(&s.MaxEphemeralAccountBalance), decode(&s.RevisionNumber), &s.Version, &s.Release, &s.SiaMuxPort, decode(&p.UID), &p.Validity, decode(&p.HostBlockHeight), decode(&p.UpdatePriceTableCost), decode(&p.AccountBalanceCost), decode(&p.FundAccountCost), decode(&p.LatestRevisionCost), decode(&p.SubscriptionMemoryCost), decode(&p.SubscriptionNotificationCost), decode(&p.InitBaseCost), decode(&p.MemoryTimeCost), decode(&p.DownloadBandwidthCost), decode(&p.UploadBandwidthCost), decode(&p.DropSectorsBaseCost), decode(&p.DropSectorsUnitCost), decode(&p.HasSectorBaseCost), decode(&p.ReadBaseCost), decode(&p.ReadLengthCost), decode(&p.RenewContractCost), decode(&p.RevisionBaseCost), decode(&p.SwapSectorBaseCost), decode(&p.WriteBaseCost), decode(&p.WriteLengthCost), decode(&p.WriteStoreCost), decode(&p.TxnFeeMinRecommended), decode(&p.TxnFeeMaxRecommended), decode(&p.ContractPrice), decode(&p.CollateralCost), decode(&p.MaxCollateral), decode(&p.MaxDuration), decode(&p.WindowSize), decode(&p.RegistryEntriesLeft), decode(&p.RegistryEntriesTotal)); err != nil { + sV4, pV4 := &host.RHPV4Settings, &host.RHPV4Settings.Prices + if err := rows.Scan(decode(&host.PublicKey), &host.NetAddress, &host.CountryCode, decode(&host.KnownSince), decode(&host.LastScan), &host.LastScanSuccessful, decode(&host.LastAnnouncement), &host.TotalScans, &host.SuccessfulInteractions, &host.FailedInteractions, &s.AcceptingContracts, decode(&s.MaxDownloadBatchSize), decode(&s.MaxDuration), decode(&s.MaxReviseBatchSize), &s.NetAddress, decode(&s.RemainingStorage), decode(&s.SectorSize), decode(&s.TotalStorage), decode(&s.Address), decode(&s.WindowSize), decode(&s.Collateral), decode(&s.MaxCollateral), decode(&s.BaseRPCPrice), decode(&s.ContractPrice), decode(&s.DownloadBandwidthPrice), decode(&s.SectorAccessPrice), decode(&s.StoragePrice), decode(&s.UploadBandwidthPrice), &s.EphemeralAccountExpiry, decode(&s.MaxEphemeralAccountBalance), decode(&s.RevisionNumber), &s.Version, &s.Release, &s.SiaMuxPort, decode(&p.UID), &p.Validity, decode(&p.HostBlockHeight), decode(&p.UpdatePriceTableCost), decode(&p.AccountBalanceCost), decode(&p.FundAccountCost), decode(&p.LatestRevisionCost), decode(&p.SubscriptionMemoryCost), decode(&p.SubscriptionNotificationCost), decode(&p.InitBaseCost), decode(&p.MemoryTimeCost), decode(&p.DownloadBandwidthCost), decode(&p.UploadBandwidthCost), decode(&p.DropSectorsBaseCost), decode(&p.DropSectorsUnitCost), decode(&p.HasSectorBaseCost), decode(&p.ReadBaseCost), decode(&p.ReadLengthCost), decode(&p.RenewContractCost), decode(&p.RevisionBaseCost), decode(&p.SwapSectorBaseCost), decode(&p.WriteBaseCost), decode(&p.WriteLengthCost), decode(&p.WriteStoreCost), decode(&p.TxnFeeMinRecommended), decode(&p.TxnFeeMaxRecommended), decode(&p.ContractPrice), decode(&p.CollateralCost), decode(&p.MaxCollateral), decode(&p.MaxDuration), decode(&p.WindowSize), decode(&p.RegistryEntriesLeft), decode(&p.RegistryEntriesTotal), &protocolVersion, &sV4.Release, decode(&sV4.WalletAddress), &sV4.AcceptingContracts, decode(&sV4.MaxCollateral), decode(&sV4.MaxContractDuration), decode(&sV4.MaxSectorDuration), decode(&sV4.MaxSectorBatchSize), decode(&sV4.RemainingStorage), decode(&sV4.TotalStorage), decode(&pV4.ContractPrice), decode(&pV4.Collateral), decode(&pV4.StoragePrice), decode(&pV4.IngressPrice), decode(&pV4.EgressPrice), decode(&pV4.FreeSectorPrice), decode(&pV4.TipHeight), decode(&pV4.ValidUntil), decode(&pV4.Signature)); err != nil { return err } + sV4.ProtocolVersion = [3]uint8(protocolVersion) v2AddrRows, err := v2AddrStmt.Query(encode(host.PublicKey)) if err != nil { diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index 1eef93a6..a3fbfb48 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -565,7 +565,7 @@ CREATE TABLE host_info_v2_netaddresses( protocol TEXT NOT NULL, address TEXT NOT NULL, - UNIQUE(public_key, netaddress_order) + PRIMARY KEY(public_key, netaddress_order) ); CREATE INDEX host_info_v2_netaddresses_public_key ON host_info_v2_netaddresses(public_key); diff --git a/persist/sqlite/scan_test.go b/persist/sqlite/scan_test.go index 8b0e4642..36e02f7d 100644 --- a/persist/sqlite/scan_test.go +++ b/persist/sqlite/scan_test.go @@ -3,19 +3,27 @@ package sqlite import ( "context" "errors" + "net" "path/filepath" "sort" "testing" "time" + "go.sia.tech/core/consensus" + "go.sia.tech/core/gateway" + proto4 "go.sia.tech/core/rhp/v4" "go.sia.tech/core/types" - "go.sia.tech/coreutils" "go.sia.tech/coreutils/chain" + crhpv4 "go.sia.tech/coreutils/rhp/v4" + "go.sia.tech/coreutils/syncer" ctestutil "go.sia.tech/coreutils/testutil" + "go.sia.tech/coreutils/wallet" "go.sia.tech/explored/config" "go.sia.tech/explored/explorer" "go.sia.tech/explored/internal/testutil" + "lukechampine.com/frand" + "go.uber.org/zap" "go.uber.org/zap/zaptest" ) @@ -43,6 +51,69 @@ func syncDB(t *testing.T, db *Store, cm *chain.Manager) { } } +func startTestNode(tb testing.TB, n *consensus.Network, genesis types.Block) (*chain.Manager, *syncer.Syncer, *wallet.SingleAddressWallet) { + db, tipstate, err := chain.NewDBStore(chain.NewMemDB(), n, genesis) + if err != nil { + tb.Fatal(err) + } + cm := chain.NewManager(db, tipstate) + + syncerListener, err := net.Listen("tcp", ":0") + if err != nil { + tb.Fatal(err) + } + tb.Cleanup(func() { syncerListener.Close() }) + + s := syncer.New(syncerListener, cm, ctestutil.NewMemPeerStore(), gateway.Header{ + GenesisID: genesis.ID(), + UniqueID: gateway.GenerateUniqueID(), + NetAddress: "localhost:1234", + }) + go s.Run(context.Background()) + tb.Cleanup(func() { s.Close() }) + + ws := ctestutil.NewEphemeralWalletStore() + w, err := wallet.NewSingleAddressWallet(types.GeneratePrivateKey(), cm, ws) + if err != nil { + tb.Fatal(err) + } + tb.Cleanup(func() { w.Close() }) + + reorgCh := make(chan struct{}, 1) + tb.Cleanup(func() { close(reorgCh) }) + + go func() { + for range reorgCh { + reverted, applied, err := cm.UpdatesSince(w.Tip(), 1000) + if err != nil { + tb.Error(err) + } + + err = ws.UpdateChainState(func(tx wallet.UpdateTx) error { + return w.UpdateChainState(tx, reverted, applied) + }) + if err != nil { + tb.Error(err) + } + } + }() + + stop := cm.OnReorg(func(index types.ChainIndex) { + select { + case reorgCh <- struct{}{}: + default: + } + }) + tb.Cleanup(stop) + + return cm, s, w +} + +func testRenterHostPair(tb testing.TB, hostKey types.PrivateKey, cm crhpv4.ChainManager, s crhpv4.Syncer, w crhpv4.Wallet, c crhpv4.Contractor, sr crhpv4.Settings, ss crhpv4.Sectors, log *zap.Logger) string { + rs := crhpv4.NewServer(hostKey, cm, s, c, w, sr, ss, crhpv4.WithContractProofWindowBuffer(10), crhpv4.WithPriceTableValidity(2*time.Minute)) + return ctestutil.ServeSiaMux(tb, rs, log.Named("siamux")) +} + func TestScan(t *testing.T) { if testing.Short() { t.Skip() @@ -50,26 +121,44 @@ func TestScan(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() + db, err := OpenDatabase(filepath.Join(dir, "explored.sqlite3"), log.Named("sqlite3")) if err != nil { t.Fatal(err) } defer db.Close() - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - network, genesisBlock := ctestutil.Network() - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) - if err != nil { - t.Fatal(err) - } + cm, s, w := startTestNode(t, network, genesisBlock) + + sr := ctestutil.NewEphemeralSettingsReporter() + sr.Update(proto4.HostSettings{ + ProtocolVersion: [3]uint8{1, 2, 3}, + Release: "test", + AcceptingContracts: true, + WalletAddress: w.Address(), + MaxCollateral: types.Siacoins(10000), + MaxContractDuration: 1000, + MaxSectorDuration: 3 * 144, + MaxSectorBatchSize: 100, + RemainingStorage: 100 * proto4.SectorSize, + TotalStorage: 100 * proto4.SectorSize, + Prices: proto4.HostPrices{ + ContractPrice: types.Siacoins(uint32(frand.Uint64n(10000))), + StoragePrice: types.Siacoins(uint32(frand.Uint64n(10000))), + IngressPrice: types.Siacoins(uint32(frand.Uint64n(10000))), + EgressPrice: types.Siacoins(uint32(frand.Uint64n(10000))), + Collateral: types.Siacoins(uint32(frand.Uint64n(10000))), + }, + }) + ss := ctestutil.NewEphemeralSectorStore() + c := ctestutil.NewEphemeralContractor(cm) + + pk0 := types.GeneratePrivateKey() + pubkey0 := pk0.PublicKey() - cm := chain.NewManager(store, genesisState) + v4Addr := testRenterHostPair(t, pk0, cm, s, w, c, sr, ss, zap.NewNop()) cfg := config.Scanner{ Threads: 10, @@ -94,6 +183,12 @@ func TestScan(t *testing.T) { ts := types.CurrentTimestamp() hosts := []explorer.Host{ + { + PublicKey: pubkey0, + V2NetAddresses: []chain.NetAddress{{Protocol: crhpv4.ProtocolTCPSiaMux, Address: v4Addr}}, + LastAnnouncement: ts, + KnownSince: ts, + }, { PublicKey: pubkey1, NetAddress: "sia1.siahost.ca:9982", @@ -115,18 +210,18 @@ func TestScan(t *testing.T) { } // explorer won't start scanning till a recent block is mined - b := testutil.MineBlock(genesisState, nil, types.VoidAddress) + b := testutil.MineBlock(cm.TipState(), nil, types.VoidAddress) if err := cm.AddBlocks([]types.Block{b}); err != nil { t.Fatal(err) } - time.Sleep(3 * cfg.Timeout) + time.Sleep(4 * cfg.Timeout) - dbHosts, err := e.Hosts([]types.PublicKey{pubkey1, pubkey2}) + dbHosts, err := e.Hosts([]types.PublicKey{pubkey0, pubkey1, pubkey2}) if err != nil { t.Fatal(err) } - testutil.Equal(t, "len(dbHosts)", 2, len(dbHosts)) + testutil.Equal(t, "len(dbHosts)", 3, len(dbHosts)) sort.Slice(hosts, func(i, j int) bool { return hosts[i].NetAddress < hosts[j].NetAddress @@ -134,17 +229,29 @@ func TestScan(t *testing.T) { sort.Slice(dbHosts, func(i, j int) bool { return dbHosts[i].NetAddress < dbHosts[j].NetAddress }) - host1 := dbHosts[0] - testutil.Equal(t, "host1.NetAddress", hosts[0].NetAddress, host1.NetAddress) - testutil.Equal(t, "host1.PublicKey", hosts[0].PublicKey, host1.PublicKey) + + host0 := dbHosts[0] + testutil.Equal(t, "host0.NetAddress", hosts[0].NetAddress, host0.NetAddress) + testutil.Equal(t, "host0.PublicKey", hosts[0].PublicKey, host0.PublicKey) + testutil.Equal(t, "host0.TotalScans", 1, host0.TotalScans) + testutil.Equal(t, "host0.SuccessfulInteractions", 1, host0.SuccessfulInteractions) + testutil.Equal(t, "host0.FailedInteractions", 0, host0.FailedInteractions) + testutil.Equal(t, "host0.LastScanSuccessful", true, host0.LastScanSuccessful) + if !host0.RHPV4Settings.AcceptingContracts { + log.Fatal("AcceptingContracts = false on host that's supposed to be active") + } + + host1 := dbHosts[1] + testutil.Equal(t, "host1.NetAddress", hosts[1].NetAddress, host1.NetAddress) + testutil.Equal(t, "host1.PublicKey", hosts[1].PublicKey, host1.PublicKey) testutil.Equal(t, "host1.TotalScans", 1, host1.TotalScans) testutil.Equal(t, "host1.SuccessfulInteractions", 0, host1.SuccessfulInteractions) testutil.Equal(t, "host1.FailedInteractions", 1, host1.FailedInteractions) testutil.Equal(t, "host1.LastScanSuccessful", false, host1.LastScanSuccessful) - host2 := dbHosts[1] - testutil.Equal(t, "host2.NetAddress", hosts[1].NetAddress, host2.NetAddress) - testutil.Equal(t, "host2.PublicKey", hosts[1].PublicKey, host2.PublicKey) + host2 := dbHosts[2] + testutil.Equal(t, "host2.NetAddress", hosts[2].NetAddress, host2.NetAddress) + testutil.Equal(t, "host2.PublicKey", hosts[2].PublicKey, host2.PublicKey) testutil.Equal(t, "host2.CountryCode", "CA", host2.CountryCode) testutil.Equal(t, "host2.TotalScans", 1, host2.TotalScans) testutil.Equal(t, "host2.SuccessfulInteractions", 1, host2.SuccessfulInteractions)