diff --git a/explorer/explorer.go b/explorer/explorer.go index 0667dc29..672d2b73 100644 --- a/explorer/explorer.go +++ b/explorer/explorer.go @@ -67,7 +67,7 @@ type Store interface { SiafundElements(ids []types.SiafundOutputID) (result []SiafundOutput, err error) Hosts(pks []types.PublicKey) ([]Host, error) - HostsForScanning(maxLastScan, minLastAnnouncement time.Time, offset, limit uint64) ([]chain.HostAnnouncement, error) + HostsForScanning(maxLastScan, minLastAnnouncement time.Time, offset, limit uint64) ([]Host, error) } // Explorer implements a Sia explorer. diff --git a/explorer/scan.go b/explorer/scan.go index 5fe4b8f9..36288a87 100644 --- a/explorer/scan.go +++ b/explorer/scan.go @@ -9,7 +9,7 @@ import ( crhpv2 "go.sia.tech/core/rhp/v2" "go.sia.tech/core/types" - "go.sia.tech/coreutils/chain" + crhpv4 "go.sia.tech/coreutils/rhp/v4" "go.sia.tech/explored/internal/geoip" rhpv2 "go.sia.tech/explored/internal/rhp/v2" rhpv3 "go.sia.tech/explored/internal/rhp/v3" @@ -52,11 +52,12 @@ func (e *Explorer) waitForSync() error { return nil } -func (e *Explorer) scanHost(locator geoip.Locator, host chain.HostAnnouncement) (HostScan, error) { +func (e *Explorer) scanV1Host(locator geoip.Locator, host Host) (HostScan, error) { ctx, cancel := context.WithTimeout(e.ctx, e.scanCfg.Timeout) defer cancel() dialer := (&net.Dialer{}) + conn, err := dialer.DialContext(ctx, "tcp", host.NetAddress) if err != nil { return HostScan{}, fmt.Errorf("scanHost: failed to connect to host: %w", err) @@ -80,7 +81,6 @@ func (e *Explorer) scanHost(locator geoip.Locator, host chain.HostAnnouncement) } resolved, err := net.ResolveIPAddr("ip", hostIP) - // if we can resolve the address if err != nil { return HostScan{}, fmt.Errorf("scanHost: failed to resolve host address: %w", err) } @@ -113,7 +113,53 @@ func (e *Explorer) scanHost(locator geoip.Locator, host chain.HostAnnouncement) }, nil } -func (e *Explorer) addHostScans(hosts chan chain.HostAnnouncement) { +func (e *Explorer) scanV2Host(locator geoip.Locator, host Host) (HostScan, error) { + ctx, cancel := context.WithTimeout(e.ctx, e.scanCfg.Timeout) + defer cancel() + + addr, ok := host.V2SiamuxAddr() + if !ok { + return HostScan{}, fmt.Errorf("host has no v2 siamux address") + } + + transport, err := crhpv4.DialSiaMux(ctx, addr, host.PublicKey) + if err != nil { + return HostScan{}, fmt.Errorf("failed to dial host: %w", err) + } + defer transport.Close() + + settings, err := crhpv4.RPCSettings(ctx, transport) + if err != nil { + return HostScan{}, fmt.Errorf("failed to get host settings: %w", err) + } + + hostIP, _, err := net.SplitHostPort(addr) + if err != nil { + return HostScan{}, fmt.Errorf("scanHost: failed to parse net address: %w", err) + } + + resolved, err := net.ResolveIPAddr("ip", hostIP) + if err != nil { + return HostScan{}, fmt.Errorf("scanHost: failed to resolve host address: %w", err) + } + + countryCode, err := locator.CountryCode(resolved) + if err != nil { + e.log.Debug("Failed to resolve IP geolocation, not setting country code", zap.String("addr", host.NetAddress)) + countryCode = "" + } + + return HostScan{ + PublicKey: host.PublicKey, + CountryCode: countryCode, + Success: true, + Timestamp: types.CurrentTimestamp(), + + RHPV4Settings: settings, + }, nil +} + +func (e *Explorer) addHostScans(hosts chan Host) { // use default included ip2location database locator, err := geoip.NewIP2LocationLocator("") if err != nil { @@ -129,18 +175,32 @@ func (e *Explorer) addHostScans(hosts chan chain.HostAnnouncement) { break } - scan, err := e.scanHost(locator, host) + var scan HostScan + var addr string + var ok bool + var err error + + if host.IsV2() { + addr, ok = host.V2SiamuxAddr() + if !ok { + e.log.Debug("Host did not have any v2 siamux net addresses in its announcement, unable to scan", zap.Stringer("pk", host.PublicKey)) + continue + } + scan, err = e.scanV2Host(locator, host) + } else { + scan, err = e.scanV1Host(locator, host) + } if err != nil { scans = append(scans, HostScan{ PublicKey: host.PublicKey, Success: false, Timestamp: types.CurrentTimestamp(), }) - e.log.Debug("Scanning host failed", zap.String("addr", host.NetAddress), zap.Stringer("pk", host.PublicKey), zap.Error(err)) + e.log.Debug("Scanning host failed", zap.String("addr", addr), zap.Stringer("pk", host.PublicKey), zap.Error(err)) continue } - e.log.Debug("Scanning host succeeded", zap.String("addr", host.NetAddress), zap.Stringer("pk", host.PublicKey)) + e.log.Debug("Scanning host succeeded", zap.String("addr", addr), zap.Stringer("pk", host.PublicKey)) scans = append(scans, scan) } @@ -172,7 +232,7 @@ func (e *Explorer) isClosed() bool { } } -func (e *Explorer) fetchHosts(hosts chan chain.HostAnnouncement) { +func (e *Explorer) fetchHosts(hosts chan Host) { var exhausted bool offset := 0 @@ -210,7 +270,7 @@ func (e *Explorer) scanHosts() { for !e.isClosed() { // fetch hosts - hosts := make(chan chain.HostAnnouncement, scanBatchSize) + hosts := make(chan Host, scanBatchSize) e.wg.Add(1) go func() { defer e.wg.Done() diff --git a/explorer/types.go b/explorer/types.go index 2212d85b..49aeabea 100644 --- a/explorer/types.go +++ b/explorer/types.go @@ -8,8 +8,10 @@ import ( "go.sia.tech/core/consensus" rhpv2 "go.sia.tech/core/rhp/v2" rhpv3 "go.sia.tech/core/rhp/v3" + rhpv4 "go.sia.tech/core/rhp/v4" "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" + crhpv4 "go.sia.tech/coreutils/rhp/v4" ) // A Source represents where a siacoin output came from. @@ -287,6 +289,8 @@ type HostScan struct { Settings rhpv2.HostSettings `json:"settings"` PriceTable rhpv3.HostPriceTable `json:"priceTable"` + + RHPV4Settings rhpv4.HostSettings `json:"rhpV4Settings"` } // Host represents a host and the information gathered from scanning it. @@ -307,6 +311,25 @@ type Host struct { Settings rhpv2.HostSettings `json:"settings"` PriceTable rhpv3.HostPriceTable `json:"priceTable"` + + RHPV4Settings rhpv4.HostSettings `json:"rhpV4Settings"` +} + +// V2SiamuxAddr returns the `Address` of the first TCP siamux `NetAddress` it +// finds in the host's list of net addresses. The protocol for this address is +// ProtocolTCPSiaMux. +func (h Host) V2SiamuxAddr() (string, bool) { + for _, netAddr := range h.V2NetAddresses { + if netAddr.Protocol == crhpv4.ProtocolTCPSiaMux { + return netAddr.Address, true + } + } + return "", false +} + +// IsV2 returns whether a host supports V2 or not. +func (h Host) IsV2() bool { + return len(h.V2NetAddresses) > 0 } // HostMetrics represents averages of scanned information from hosts. diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index c7fd9a58..772a3308 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -89,7 +89,7 @@ func (st *Store) Hosts(pks []types.PublicKey) (result []explorer.Host, err error encoded = append(encoded, encode(pk)) } - rows, err := tx.Query(`SELECT public_key,net_address,country_code,known_since,last_scan,last_scan_successful,last_announcement,total_scans,successful_interactions,failed_interactions,settings_accepting_contracts,settings_max_download_batch_size,settings_max_duration,settings_max_revise_batch_size,settings_net_address,settings_remaining_storage,settings_sector_size,settings_total_storage,settings_address,settings_window_size,settings_collateral,settings_max_collateral,settings_base_rpc_price,settings_contract_price,settings_download_bandwidth_price,settings_sector_access_price,settings_storage_price,settings_upload_bandwidth_price,settings_ephemeral_account_expiry,settings_max_ephemeral_account_balance,settings_revision_number,settings_version,settings_release,settings_sia_mux_port,price_table_uid,price_table_validity,price_table_host_block_height,price_table_update_price_table_cost,price_table_account_balance_cost,price_table_fund_account_cost,price_table_latest_revision_cost,price_table_subscription_memory_cost,price_table_subscription_notification_cost,price_table_init_base_cost,price_table_memory_time_cost,price_table_download_bandwidth_cost,price_table_upload_bandwidth_cost,price_table_drop_sectors_base_cost,price_table_drop_sectors_unit_cost,price_table_has_sector_base_cost,price_table_read_base_cost,price_table_read_length_cost,price_table_renew_contract_cost,price_table_revision_base_cost,price_table_swap_sector_base_cost,price_table_write_base_cost,price_table_write_length_cost,price_table_write_store_cost,price_table_txn_fee_min_recommended,price_table_txn_fee_max_recommended,price_table_contract_price,price_table_collateral_cost,price_table_max_collateral,price_table_max_duration,price_table_window_size,price_table_registry_entries_left,price_table_registry_entries_total FROM host_info WHERE public_key IN (`+queryPlaceHolders(len(pks))+`)`, encoded...) + rows, err := tx.Query(`SELECT public_key,net_address,country_code,known_since,last_scan,last_scan_successful,last_announcement,total_scans,successful_interactions,failed_interactions,settings_accepting_contracts,settings_max_download_batch_size,settings_max_duration,settings_max_revise_batch_size,settings_net_address,settings_remaining_storage,settings_sector_size,settings_total_storage,settings_address,settings_window_size,settings_collateral,settings_max_collateral,settings_base_rpc_price,settings_contract_price,settings_download_bandwidth_price,settings_sector_access_price,settings_storage_price,settings_upload_bandwidth_price,settings_ephemeral_account_expiry,settings_max_ephemeral_account_balance,settings_revision_number,settings_version,settings_release,settings_sia_mux_port,price_table_uid,price_table_validity,price_table_host_block_height,price_table_update_price_table_cost,price_table_account_balance_cost,price_table_fund_account_cost,price_table_latest_revision_cost,price_table_subscription_memory_cost,price_table_subscription_notification_cost,price_table_init_base_cost,price_table_memory_time_cost,price_table_download_bandwidth_cost,price_table_upload_bandwidth_cost,price_table_drop_sectors_base_cost,price_table_drop_sectors_unit_cost,price_table_has_sector_base_cost,price_table_read_base_cost,price_table_read_length_cost,price_table_renew_contract_cost,price_table_revision_base_cost,price_table_swap_sector_base_cost,price_table_write_base_cost,price_table_write_length_cost,price_table_write_store_cost,price_table_txn_fee_min_recommended,price_table_txn_fee_max_recommended,price_table_contract_price,price_table_collateral_cost,price_table_max_collateral,price_table_max_duration,price_table_window_size,price_table_registry_entries_left,price_table_registry_entries_total,rhp4_settings_protocol_version,rhp4_settings_release,rhp4_settings_wallet_address,rhp4_settings_accepting_contracts,rhp4_settings_max_collateral,rhp4_settings_max_collateral_duration,rhp4_settings_max_sector_duration,rhp4_settings_max_sector_batch_size,rhp4_settings_remaining_storage,rhp4_settings_total_storage,rhp4_prices_contract_price,rhp4_prices_collateral_price,rhp4_prices_storage_price,rhp4_prices_ingress_price,rhp4_prices_egress_price,rhp4_prices_free_sector_price,rhp4_prices_tip_height,rhp4_prices_valid_until,rhp4_prices_signature FROM host_info WHERE public_key IN (`+queryPlaceHolders(len(pks))+`)`, encoded...) if err != nil { return err } @@ -104,10 +104,14 @@ func (st *Store) Hosts(pks []types.PublicKey) (result []explorer.Host, err error for rows.Next() { if err := func() error { var host explorer.Host + var protocolVersion []uint8 + s, p := &host.Settings, &host.PriceTable - if err := rows.Scan(decode(&host.PublicKey), &host.NetAddress, &host.CountryCode, decode(&host.KnownSince), decode(&host.LastScan), &host.LastScanSuccessful, decode(&host.LastAnnouncement), &host.TotalScans, &host.SuccessfulInteractions, &host.FailedInteractions, &s.AcceptingContracts, decode(&s.MaxDownloadBatchSize), decode(&s.MaxDuration), decode(&s.MaxReviseBatchSize), &s.NetAddress, decode(&s.RemainingStorage), decode(&s.SectorSize), decode(&s.TotalStorage), decode(&s.Address), decode(&s.WindowSize), decode(&s.Collateral), decode(&s.MaxCollateral), decode(&s.BaseRPCPrice), decode(&s.ContractPrice), decode(&s.DownloadBandwidthPrice), decode(&s.SectorAccessPrice), decode(&s.StoragePrice), decode(&s.UploadBandwidthPrice), &s.EphemeralAccountExpiry, decode(&s.MaxEphemeralAccountBalance), decode(&s.RevisionNumber), &s.Version, &s.Release, &s.SiaMuxPort, decode(&p.UID), &p.Validity, decode(&p.HostBlockHeight), decode(&p.UpdatePriceTableCost), decode(&p.AccountBalanceCost), decode(&p.FundAccountCost), decode(&p.LatestRevisionCost), decode(&p.SubscriptionMemoryCost), decode(&p.SubscriptionNotificationCost), decode(&p.InitBaseCost), decode(&p.MemoryTimeCost), decode(&p.DownloadBandwidthCost), decode(&p.UploadBandwidthCost), decode(&p.DropSectorsBaseCost), decode(&p.DropSectorsUnitCost), decode(&p.HasSectorBaseCost), decode(&p.ReadBaseCost), decode(&p.ReadLengthCost), decode(&p.RenewContractCost), decode(&p.RevisionBaseCost), decode(&p.SwapSectorBaseCost), decode(&p.WriteBaseCost), decode(&p.WriteLengthCost), decode(&p.WriteStoreCost), decode(&p.TxnFeeMinRecommended), decode(&p.TxnFeeMaxRecommended), decode(&p.ContractPrice), decode(&p.CollateralCost), decode(&p.MaxCollateral), decode(&p.MaxDuration), decode(&p.WindowSize), decode(&p.RegistryEntriesLeft), decode(&p.RegistryEntriesTotal)); err != nil { + sV4, pV4 := &host.RHPV4Settings, &host.RHPV4Settings.Prices + if err := rows.Scan(decode(&host.PublicKey), &host.NetAddress, &host.CountryCode, decode(&host.KnownSince), decode(&host.LastScan), &host.LastScanSuccessful, decode(&host.LastAnnouncement), &host.TotalScans, &host.SuccessfulInteractions, &host.FailedInteractions, &s.AcceptingContracts, decode(&s.MaxDownloadBatchSize), decode(&s.MaxDuration), decode(&s.MaxReviseBatchSize), &s.NetAddress, decode(&s.RemainingStorage), decode(&s.SectorSize), decode(&s.TotalStorage), decode(&s.Address), decode(&s.WindowSize), decode(&s.Collateral), decode(&s.MaxCollateral), decode(&s.BaseRPCPrice), decode(&s.ContractPrice), decode(&s.DownloadBandwidthPrice), decode(&s.SectorAccessPrice), decode(&s.StoragePrice), decode(&s.UploadBandwidthPrice), &s.EphemeralAccountExpiry, decode(&s.MaxEphemeralAccountBalance), decode(&s.RevisionNumber), &s.Version, &s.Release, &s.SiaMuxPort, decode(&p.UID), &p.Validity, decode(&p.HostBlockHeight), decode(&p.UpdatePriceTableCost), decode(&p.AccountBalanceCost), decode(&p.FundAccountCost), decode(&p.LatestRevisionCost), decode(&p.SubscriptionMemoryCost), decode(&p.SubscriptionNotificationCost), decode(&p.InitBaseCost), decode(&p.MemoryTimeCost), decode(&p.DownloadBandwidthCost), decode(&p.UploadBandwidthCost), decode(&p.DropSectorsBaseCost), decode(&p.DropSectorsUnitCost), decode(&p.HasSectorBaseCost), decode(&p.ReadBaseCost), decode(&p.ReadLengthCost), decode(&p.RenewContractCost), decode(&p.RevisionBaseCost), decode(&p.SwapSectorBaseCost), decode(&p.WriteBaseCost), decode(&p.WriteLengthCost), decode(&p.WriteStoreCost), decode(&p.TxnFeeMinRecommended), decode(&p.TxnFeeMaxRecommended), decode(&p.ContractPrice), decode(&p.CollateralCost), decode(&p.MaxCollateral), decode(&p.MaxDuration), decode(&p.WindowSize), decode(&p.RegistryEntriesLeft), decode(&p.RegistryEntriesTotal), &protocolVersion, &sV4.Release, decode(&sV4.WalletAddress), &sV4.AcceptingContracts, decode(&sV4.MaxCollateral), decode(&sV4.MaxContractDuration), decode(&sV4.MaxSectorDuration), decode(&sV4.MaxSectorBatchSize), decode(&sV4.RemainingStorage), decode(&sV4.TotalStorage), decode(&pV4.ContractPrice), decode(&pV4.Collateral), decode(&pV4.StoragePrice), decode(&pV4.IngressPrice), decode(&pV4.EgressPrice), decode(&pV4.FreeSectorPrice), decode(&pV4.TipHeight), decode(&pV4.ValidUntil), decode(&pV4.Signature)); err != nil { return err } + sV4.ProtocolVersion = [3]uint8(protocolVersion) v2AddrRows, err := v2AddrStmt.Query(encode(host.PublicKey)) if err != nil { @@ -134,7 +138,9 @@ func (st *Store) Hosts(pks []types.PublicKey) (result []explorer.Host, err error } // HostsForScanning returns hosts ordered by the transaction they were created in. -func (s *Store) HostsForScanning(maxLastScan, minLastAnnouncement time.Time, offset, limit uint64) (result []chain.HostAnnouncement, err error) { +// Note that only the PublicKey, NetAddress, and V2NetAddresses fields are +// populated. +func (s *Store) HostsForScanning(maxLastScan, minLastAnnouncement time.Time, offset, limit uint64) (result []explorer.Host, err error) { err = s.transaction(func(tx *txn) error { rows, err := tx.Query(`SELECT public_key, net_address FROM host_info WHERE last_scan <= ? AND last_announcement >= ? ORDER BY last_scan ASC LIMIT ? OFFSET ?`, encode(maxLastScan), encode(minLastAnnouncement), limit, offset) if err != nil { @@ -142,11 +148,37 @@ func (s *Store) HostsForScanning(maxLastScan, minLastAnnouncement time.Time, off } defer rows.Close() + v2AddrStmt, err := tx.Prepare(`SELECT protocol,address FROM host_info_v2_netaddresses WHERE public_key = ? ORDER BY netaddress_order`) + if err != nil { + return err + } + defer v2AddrStmt.Close() + for rows.Next() { - var host chain.HostAnnouncement + var host explorer.Host if err := rows.Scan(decode(&host.PublicKey), &host.NetAddress); err != nil { return err } + + err := func() error { + v2AddrRows, err := v2AddrStmt.Query(encode(host.PublicKey)) + if err != nil { + return err + } + defer v2AddrRows.Close() + for v2AddrRows.Next() { + var netAddr chain.NetAddress + if err := v2AddrRows.Scan(&netAddr.Protocol, &netAddr.Address); err != nil { + return err + } + host.V2NetAddresses = append(host.V2NetAddresses, netAddr) + } + return nil + }() + if err != nil { + return err + } + result = append(result, host) } return nil diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 1a43cd43..3b3576bc 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -1120,7 +1120,7 @@ func (ut *updateTx) RevertIndex(state explorer.UpdateState) error { } func addHosts(tx *txn, scans []explorer.Host) error { - stmt, err := tx.Prepare(`INSERT INTO host_info(public_key, net_address, country_code, known_since, last_scan, last_scan_successful, last_announcement, total_scans, successful_interactions, failed_interactions, settings_accepting_contracts, settings_max_download_batch_size, settings_max_duration, settings_max_revise_batch_size, settings_net_address, settings_remaining_storage, settings_sector_size, settings_total_storage, settings_address, settings_window_size, settings_collateral, settings_max_collateral, settings_base_rpc_price, settings_contract_price, settings_download_bandwidth_price, settings_sector_access_price, settings_storage_price, settings_upload_bandwidth_price, settings_ephemeral_account_expiry, settings_max_ephemeral_account_balance, settings_revision_number, settings_version, settings_release, settings_sia_mux_port, price_table_uid, price_table_validity, price_table_host_block_height, price_table_update_price_table_cost, price_table_account_balance_cost, price_table_fund_account_cost, price_table_latest_revision_cost, price_table_subscription_memory_cost, price_table_subscription_notification_cost, price_table_init_base_cost, price_table_memory_time_cost, price_table_download_bandwidth_cost, price_table_upload_bandwidth_cost, price_table_drop_sectors_base_cost, price_table_drop_sectors_unit_cost, price_table_has_sector_base_cost, price_table_read_base_cost, price_table_read_length_cost, price_table_renew_contract_cost, price_table_revision_base_cost, price_table_swap_sector_base_cost, price_table_write_base_cost, price_table_write_length_cost, price_table_write_store_cost, price_table_txn_fee_min_recommended, price_table_txn_fee_max_recommended, price_table_contract_price, price_table_collateral_cost, price_table_max_collateral, price_table_max_duration, price_table_window_size, price_table_registry_entries_left, price_table_registry_entries_total) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26,$27,$28,$29,$30,$31,$32,$33,$34,$35,$36,$37,$38,$39,$40,$41,$42,$43,$44,$45,$46,$47,$48,$49,$50,$51,$52,$53,$54,$55,$56,$57,$58,$59,$60,$61,$62,$63,$64,$65,$66,$67) ON CONFLICT (public_key) DO UPDATE SET net_address = $2, country_code = $3, last_scan = $5, last_scan_successful = $6, last_announcement = CASE WHEN $7 > 0 THEN last_announcement ELSE $7 END, total_scans = $8, successful_interactions = $9, failed_interactions = failed_interactions + $10, settings_accepting_contracts = $11, settings_max_download_batch_size = $12, settings_max_duration = $13, settings_max_revise_batch_size = $14 , settings_net_address = $15, settings_remaining_storage = $16, settings_sector_size = $17, settings_total_storage = $18, settings_address = $19, settings_window_size = $20, settings_collateral = $21, settings_max_collateral = $22, settings_base_rpc_price = $23, settings_contract_price = $24, settings_download_bandwidth_price = $25, settings_sector_access_price = $26, settings_storage_price = $27, settings_upload_bandwidth_price = $28, settings_ephemeral_account_expiry = $29, settings_max_ephemeral_account_balance = $30, settings_revision_number = $31, settings_version = $32, settings_release = $33, settings_sia_mux_port = $34, price_table_uid = $35, price_table_validity = $36, price_table_host_block_height = $37, price_table_update_price_table_cost = $38, price_table_account_balance_cost = $39, price_table_fund_account_cost = $40, price_table_latest_revision_cost = $41, price_table_subscription_memory_cost = $42, price_table_subscription_notification_cost = $43, price_table_init_base_cost = $44, price_table_memory_time_cost = $45, price_table_download_bandwidth_cost = $46, price_table_upload_bandwidth_cost = $47, price_table_drop_sectors_base_cost = $48, price_table_drop_sectors_unit_cost = $49, price_table_has_sector_base_cost = $50, price_table_read_base_cost = $51, price_table_read_length_cost = $52, price_table_renew_contract_cost = $53, price_table_revision_base_cost = $54, price_table_swap_sector_base_cost = $55, price_table_write_base_cost = $56, price_table_write_length_cost = $57, price_table_write_store_cost = $58, price_table_txn_fee_min_recommended = $59, price_table_txn_fee_max_recommended = $60, price_table_contract_price = $61, price_table_collateral_cost = $62, price_table_max_collateral = $63, price_table_max_duration = $64, price_table_window_size = $65, price_table_registry_entries_left = $66, price_table_registry_entries_total = $67`) + stmt, err := tx.Prepare(`INSERT INTO host_info(public_key, net_address, country_code, known_since, last_scan, last_scan_successful, last_announcement, total_scans, successful_interactions, failed_interactions, settings_accepting_contracts, settings_max_download_batch_size, settings_max_duration, settings_max_revise_batch_size, settings_net_address, settings_remaining_storage, settings_sector_size, settings_total_storage, settings_address, settings_window_size, settings_collateral, settings_max_collateral, settings_base_rpc_price, settings_contract_price, settings_download_bandwidth_price, settings_sector_access_price, settings_storage_price, settings_upload_bandwidth_price, settings_ephemeral_account_expiry, settings_max_ephemeral_account_balance, settings_revision_number, settings_version, settings_release, settings_sia_mux_port, price_table_uid, price_table_validity, price_table_host_block_height, price_table_update_price_table_cost, price_table_account_balance_cost, price_table_fund_account_cost, price_table_latest_revision_cost, price_table_subscription_memory_cost, price_table_subscription_notification_cost, price_table_init_base_cost, price_table_memory_time_cost, price_table_download_bandwidth_cost, price_table_upload_bandwidth_cost, price_table_drop_sectors_base_cost, price_table_drop_sectors_unit_cost, price_table_has_sector_base_cost, price_table_read_base_cost, price_table_read_length_cost, price_table_renew_contract_cost, price_table_revision_base_cost, price_table_swap_sector_base_cost, price_table_write_base_cost, price_table_write_length_cost, price_table_write_store_cost, price_table_txn_fee_min_recommended, price_table_txn_fee_max_recommended, price_table_contract_price, price_table_collateral_cost, price_table_max_collateral, price_table_max_duration, price_table_window_size, price_table_registry_entries_left, price_table_registry_entries_total, rhp4_settings_protocol_version, rhp4_settings_release, rhp4_settings_wallet_address, rhp4_settings_accepting_contracts, rhp4_settings_max_collateral, rhp4_settings_max_collateral_duration, rhp4_settings_max_sector_duration, rhp4_settings_max_sector_batch_size, rhp4_settings_remaining_storage, rhp4_settings_total_storage, rhp4_prices_contract_price, rhp4_prices_collateral_price, rhp4_prices_storage_price, rhp4_prices_ingress_price, rhp4_prices_egress_price, rhp4_prices_free_sector_price, rhp4_prices_tip_height, rhp4_prices_valid_until, rhp4_prices_signature) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26,$27,$28,$29,$30,$31,$32,$33,$34,$35,$36,$37,$38,$39,$40,$41,$42,$43,$44,$45,$46,$47,$48,$49,$50,$51,$52,$53,$54,$55,$56,$57,$58,$59,$60,$61,$62,$63,$64,$65,$66,$67,$68,$69,$70,$71,$72,$73,$74,$75,$76,$77,$78,$79,$80,$81,$82,$83,$84,$85,$86) ON CONFLICT (public_key) DO UPDATE SET net_address = $2, country_code = $3, last_scan = $5, last_scan_successful = $6, last_announcement = CASE WHEN $7 > 0 THEN last_announcement ELSE $7 END, total_scans = $8, successful_interactions = $9, failed_interactions = failed_interactions + $10, settings_accepting_contracts = $11, settings_max_download_batch_size = $12, settings_max_duration = $13, settings_max_revise_batch_size = $14 , settings_net_address = $15, settings_remaining_storage = $16, settings_sector_size = $17, settings_total_storage = $18, settings_address = $19, settings_window_size = $20, settings_collateral = $21, settings_max_collateral = $22, settings_base_rpc_price = $23, settings_contract_price = $24, settings_download_bandwidth_price = $25, settings_sector_access_price = $26, settings_storage_price = $27, settings_upload_bandwidth_price = $28, settings_ephemeral_account_expiry = $29, settings_max_ephemeral_account_balance = $30, settings_revision_number = $31, settings_version = $32, settings_release = $33, settings_sia_mux_port = $34, price_table_uid = $35, price_table_validity = $36, price_table_host_block_height = $37, price_table_update_price_table_cost = $38, price_table_account_balance_cost = $39, price_table_fund_account_cost = $40, price_table_latest_revision_cost = $41, price_table_subscription_memory_cost = $42, price_table_subscription_notification_cost = $43, price_table_init_base_cost = $44, price_table_memory_time_cost = $45, price_table_download_bandwidth_cost = $46, price_table_upload_bandwidth_cost = $47, price_table_drop_sectors_base_cost = $48, price_table_drop_sectors_unit_cost = $49, price_table_has_sector_base_cost = $50, price_table_read_base_cost = $51, price_table_read_length_cost = $52, price_table_renew_contract_cost = $53, price_table_revision_base_cost = $54, price_table_swap_sector_base_cost = $55, price_table_write_base_cost = $56, price_table_write_length_cost = $57, price_table_write_store_cost = $58, price_table_txn_fee_min_recommended = $59, price_table_txn_fee_max_recommended = $60, price_table_contract_price = $61, price_table_collateral_cost = $62, price_table_max_collateral = $63, price_table_max_duration = $64, price_table_window_size = $65, price_table_registry_entries_left = $66, price_table_registry_entries_total = $67, rhp4_settings_protocol_version = $68, rhp4_settings_release = $69, rhp4_settings_wallet_address = $70, rhp4_settings_accepting_contracts = $71, rhp4_settings_max_collateral = $72, rhp4_settings_max_collateral_duration = $73, rhp4_settings_max_sector_duration = $74, rhp4_settings_max_sector_batch_size = $75, rhp4_settings_remaining_storage = $76, rhp4_settings_total_storage = $77, rhp4_prices_contract_price = $78, rhp4_prices_collateral_price = $79, rhp4_prices_storage_price = $80, rhp4_prices_ingress_price = $81, rhp4_prices_egress_price = $82, rhp4_prices_free_sector_price = $83, rhp4_prices_tip_height = $84, rhp4_prices_valid_until = $85, rhp4_prices_signature = $86`) if err != nil { return fmt.Errorf("failed to prepare host_info stmt: %w", err) } @@ -1140,7 +1140,8 @@ func addHosts(tx *txn, scans []explorer.Host) error { for _, scan := range scans { s, p := scan.Settings, scan.PriceTable - if _, err := stmt.Exec(encode(scan.PublicKey), scan.NetAddress, scan.CountryCode, encode(scan.KnownSince), encode(scan.LastScan), scan.LastScanSuccessful, encode(scan.LastAnnouncement), scan.TotalScans, scan.SuccessfulInteractions, scan.FailedInteractions, s.AcceptingContracts, encode(s.MaxDownloadBatchSize), encode(s.MaxDuration), encode(s.MaxReviseBatchSize), s.NetAddress, encode(s.RemainingStorage), encode(s.SectorSize), encode(s.TotalStorage), encode(s.Address), encode(s.WindowSize), encode(s.Collateral), encode(s.MaxCollateral), encode(s.BaseRPCPrice), encode(s.ContractPrice), encode(s.DownloadBandwidthPrice), encode(s.SectorAccessPrice), encode(s.StoragePrice), encode(s.UploadBandwidthPrice), s.EphemeralAccountExpiry, encode(s.MaxEphemeralAccountBalance), encode(s.RevisionNumber), s.Version, s.Release, s.SiaMuxPort, encode(p.UID), p.Validity, encode(p.HostBlockHeight), encode(p.UpdatePriceTableCost), encode(p.AccountBalanceCost), encode(p.FundAccountCost), encode(p.LatestRevisionCost), encode(p.SubscriptionMemoryCost), encode(p.SubscriptionNotificationCost), encode(p.InitBaseCost), encode(p.MemoryTimeCost), encode(p.DownloadBandwidthCost), encode(p.UploadBandwidthCost), encode(p.DropSectorsBaseCost), encode(p.DropSectorsUnitCost), encode(p.HasSectorBaseCost), encode(p.ReadBaseCost), encode(p.ReadLengthCost), encode(p.RenewContractCost), encode(p.RevisionBaseCost), encode(p.SwapSectorBaseCost), encode(p.WriteBaseCost), encode(p.WriteLengthCost), encode(p.WriteStoreCost), encode(p.TxnFeeMinRecommended), encode(p.TxnFeeMaxRecommended), encode(p.ContractPrice), encode(p.CollateralCost), encode(p.MaxCollateral), encode(p.MaxDuration), encode(p.WindowSize), encode(p.RegistryEntriesLeft), encode(p.RegistryEntriesTotal)); err != nil { + sV4, pV4 := scan.RHPV4Settings, scan.RHPV4Settings.Prices + if _, err := stmt.Exec(encode(scan.PublicKey), scan.NetAddress, scan.CountryCode, encode(scan.KnownSince), encode(scan.LastScan), scan.LastScanSuccessful, encode(scan.LastAnnouncement), scan.TotalScans, scan.SuccessfulInteractions, scan.FailedInteractions, s.AcceptingContracts, encode(s.MaxDownloadBatchSize), encode(s.MaxDuration), encode(s.MaxReviseBatchSize), s.NetAddress, encode(s.RemainingStorage), encode(s.SectorSize), encode(s.TotalStorage), encode(s.Address), encode(s.WindowSize), encode(s.Collateral), encode(s.MaxCollateral), encode(s.BaseRPCPrice), encode(s.ContractPrice), encode(s.DownloadBandwidthPrice), encode(s.SectorAccessPrice), encode(s.StoragePrice), encode(s.UploadBandwidthPrice), s.EphemeralAccountExpiry, encode(s.MaxEphemeralAccountBalance), encode(s.RevisionNumber), s.Version, s.Release, s.SiaMuxPort, encode(p.UID), p.Validity, encode(p.HostBlockHeight), encode(p.UpdatePriceTableCost), encode(p.AccountBalanceCost), encode(p.FundAccountCost), encode(p.LatestRevisionCost), encode(p.SubscriptionMemoryCost), encode(p.SubscriptionNotificationCost), encode(p.InitBaseCost), encode(p.MemoryTimeCost), encode(p.DownloadBandwidthCost), encode(p.UploadBandwidthCost), encode(p.DropSectorsBaseCost), encode(p.DropSectorsUnitCost), encode(p.HasSectorBaseCost), encode(p.ReadBaseCost), encode(p.ReadLengthCost), encode(p.RenewContractCost), encode(p.RevisionBaseCost), encode(p.SwapSectorBaseCost), encode(p.WriteBaseCost), encode(p.WriteLengthCost), encode(p.WriteStoreCost), encode(p.TxnFeeMinRecommended), encode(p.TxnFeeMaxRecommended), encode(p.ContractPrice), encode(p.CollateralCost), encode(p.MaxCollateral), encode(p.MaxDuration), encode(p.WindowSize), encode(p.RegistryEntriesLeft), encode(p.RegistryEntriesTotal), sV4.ProtocolVersion[:], sV4.Release, encode(sV4.WalletAddress), sV4.AcceptingContracts, encode(sV4.MaxCollateral), encode(sV4.MaxContractDuration), encode(sV4.MaxSectorDuration), encode(sV4.MaxSectorBatchSize), encode(sV4.RemainingStorage), encode(sV4.TotalStorage), encode(pV4.ContractPrice), encode(pV4.Collateral), encode(pV4.StoragePrice), encode(pV4.IngressPrice), encode(pV4.EgressPrice), encode(pV4.FreeSectorPrice), encode(pV4.TipHeight), encode(pV4.ValidUntil), encode(pV4.Signature)); err != nil { return fmt.Errorf("failed to execute host_info stmt: %w", err) } if _, err := deleteV2AddrStmt.Exec(encode(scan.PublicKey)); err != nil { @@ -1156,7 +1157,7 @@ func addHosts(tx *txn, scans []explorer.Host) error { } func addHostScans(tx *txn, scans []explorer.HostScan) error { - stmt, err := tx.Prepare(`UPDATE host_info SET country_code = ?, last_scan = ?, last_scan_successful = ?, total_scans = total_scans + 1, successful_interactions = successful_interactions + ?, failed_interactions = failed_interactions + ?, settings_accepting_contracts = ?, settings_max_download_batch_size = ?, settings_max_duration = ?, settings_max_revise_batch_size = ?, settings_net_address = ?, settings_remaining_storage = ?, settings_sector_size = ?, settings_total_storage = ?, settings_address = ?, settings_window_size = ?, settings_collateral = ?, settings_max_collateral = ?, settings_base_rpc_price = ?, settings_contract_price = ?, settings_download_bandwidth_price = ?, settings_sector_access_price = ?, settings_storage_price = ?, settings_upload_bandwidth_price = ?, settings_ephemeral_account_expiry = ?, settings_max_ephemeral_account_balance = ?, settings_revision_number = ?, settings_version = ?, settings_release = ?, settings_sia_mux_port = ?, price_table_uid = ?, price_table_validity = ?, price_table_host_block_height = ?, price_table_update_price_table_cost = ?, price_table_account_balance_cost = ?, price_table_fund_account_cost = ?, price_table_latest_revision_cost = ?, price_table_subscription_memory_cost = ?, price_table_subscription_notification_cost = ?, price_table_init_base_cost = ?, price_table_memory_time_cost = ?, price_table_download_bandwidth_cost = ?, price_table_upload_bandwidth_cost = ?, price_table_drop_sectors_base_cost = ?, price_table_drop_sectors_unit_cost = ?, price_table_has_sector_base_cost = ?, price_table_read_base_cost = ?, price_table_read_length_cost = ?, price_table_renew_contract_cost = ?, price_table_revision_base_cost = ?, price_table_swap_sector_base_cost = ?, price_table_write_base_cost = ?, price_table_write_length_cost = ?, price_table_write_store_cost = ?, price_table_txn_fee_min_recommended = ?, price_table_txn_fee_max_recommended = ?, price_table_contract_price = ?, price_table_collateral_cost = ?, price_table_max_collateral = ?, price_table_max_duration = ?, price_table_window_size = ?, price_table_registry_entries_left = ?, price_table_registry_entries_total = ? WHERE public_key = ?`) + stmt, err := tx.Prepare(`UPDATE host_info SET country_code = ?, last_scan = ?, last_scan_successful = ?, total_scans = total_scans + 1, successful_interactions = successful_interactions + ?, failed_interactions = failed_interactions + ?, settings_accepting_contracts = ?, settings_max_download_batch_size = ?, settings_max_duration = ?, settings_max_revise_batch_size = ?, settings_net_address = ?, settings_remaining_storage = ?, settings_sector_size = ?, settings_total_storage = ?, settings_address = ?, settings_window_size = ?, settings_collateral = ?, settings_max_collateral = ?, settings_base_rpc_price = ?, settings_contract_price = ?, settings_download_bandwidth_price = ?, settings_sector_access_price = ?, settings_storage_price = ?, settings_upload_bandwidth_price = ?, settings_ephemeral_account_expiry = ?, settings_max_ephemeral_account_balance = ?, settings_revision_number = ?, settings_version = ?, settings_release = ?, settings_sia_mux_port = ?, price_table_uid = ?, price_table_validity = ?, price_table_host_block_height = ?, price_table_update_price_table_cost = ?, price_table_account_balance_cost = ?, price_table_fund_account_cost = ?, price_table_latest_revision_cost = ?, price_table_subscription_memory_cost = ?, price_table_subscription_notification_cost = ?, price_table_init_base_cost = ?, price_table_memory_time_cost = ?, price_table_download_bandwidth_cost = ?, price_table_upload_bandwidth_cost = ?, price_table_drop_sectors_base_cost = ?, price_table_drop_sectors_unit_cost = ?, price_table_has_sector_base_cost = ?, price_table_read_base_cost = ?, price_table_read_length_cost = ?, price_table_renew_contract_cost = ?, price_table_revision_base_cost = ?, price_table_swap_sector_base_cost = ?, price_table_write_base_cost = ?, price_table_write_length_cost = ?, price_table_write_store_cost = ?, price_table_txn_fee_min_recommended = ?, price_table_txn_fee_max_recommended = ?, price_table_contract_price = ?, price_table_collateral_cost = ?, price_table_max_collateral = ?, price_table_max_duration = ?, price_table_window_size = ?, price_table_registry_entries_left = ?, price_table_registry_entries_total = ?, rhp4_settings_protocol_version = ?, rhp4_settings_release = ?, rhp4_settings_wallet_address = ?, rhp4_settings_accepting_contracts = ?, rhp4_settings_max_collateral = ?, rhp4_settings_max_collateral_duration = ?, rhp4_settings_max_sector_duration = ?, rhp4_settings_max_sector_batch_size = ?, rhp4_settings_remaining_storage = ?, rhp4_settings_total_storage = ?, rhp4_prices_contract_price = ?, rhp4_prices_collateral_price = ?, rhp4_prices_storage_price = ?, rhp4_prices_ingress_price = ?, rhp4_prices_egress_price = ?, rhp4_prices_free_sector_price = ?, rhp4_prices_tip_height = ?, rhp4_prices_valid_until = ?, rhp4_prices_signature = ? WHERE public_key = ?`) if err != nil { return err } @@ -1169,7 +1170,8 @@ func addHostScans(tx *txn, scans []explorer.HostScan) error { } s, p := scan.Settings, scan.PriceTable - if _, err := stmt.Exec(scan.CountryCode, encode(scan.Timestamp), scan.Success, successful, failed, s.AcceptingContracts, encode(s.MaxDownloadBatchSize), encode(s.MaxDuration), encode(s.MaxReviseBatchSize), s.NetAddress, encode(s.RemainingStorage), encode(s.SectorSize), encode(s.TotalStorage), encode(s.Address), encode(s.WindowSize), encode(s.Collateral), encode(s.MaxCollateral), encode(s.BaseRPCPrice), encode(s.ContractPrice), encode(s.DownloadBandwidthPrice), encode(s.SectorAccessPrice), encode(s.StoragePrice), encode(s.UploadBandwidthPrice), s.EphemeralAccountExpiry, encode(s.MaxEphemeralAccountBalance), encode(s.RevisionNumber), s.Version, s.Release, s.SiaMuxPort, encode(p.UID), p.Validity, encode(p.HostBlockHeight), encode(p.UpdatePriceTableCost), encode(p.AccountBalanceCost), encode(p.FundAccountCost), encode(p.LatestRevisionCost), encode(p.SubscriptionMemoryCost), encode(p.SubscriptionNotificationCost), encode(p.InitBaseCost), encode(p.MemoryTimeCost), encode(p.DownloadBandwidthCost), encode(p.UploadBandwidthCost), encode(p.DropSectorsBaseCost), encode(p.DropSectorsUnitCost), encode(p.HasSectorBaseCost), encode(p.ReadBaseCost), encode(p.ReadLengthCost), encode(p.RenewContractCost), encode(p.RevisionBaseCost), encode(p.SwapSectorBaseCost), encode(p.WriteBaseCost), encode(p.WriteLengthCost), encode(p.WriteStoreCost), encode(p.TxnFeeMinRecommended), encode(p.TxnFeeMaxRecommended), encode(p.ContractPrice), encode(p.CollateralCost), encode(p.MaxCollateral), encode(p.MaxDuration), encode(p.WindowSize), encode(p.RegistryEntriesLeft), encode(p.RegistryEntriesTotal), encode(scan.PublicKey)); err != nil { + sV4, pV4 := scan.RHPV4Settings, scan.RHPV4Settings.Prices + if _, err := stmt.Exec(scan.CountryCode, encode(scan.Timestamp), scan.Success, successful, failed, s.AcceptingContracts, encode(s.MaxDownloadBatchSize), encode(s.MaxDuration), encode(s.MaxReviseBatchSize), s.NetAddress, encode(s.RemainingStorage), encode(s.SectorSize), encode(s.TotalStorage), encode(s.Address), encode(s.WindowSize), encode(s.Collateral), encode(s.MaxCollateral), encode(s.BaseRPCPrice), encode(s.ContractPrice), encode(s.DownloadBandwidthPrice), encode(s.SectorAccessPrice), encode(s.StoragePrice), encode(s.UploadBandwidthPrice), s.EphemeralAccountExpiry, encode(s.MaxEphemeralAccountBalance), encode(s.RevisionNumber), s.Version, s.Release, s.SiaMuxPort, encode(p.UID), p.Validity, encode(p.HostBlockHeight), encode(p.UpdatePriceTableCost), encode(p.AccountBalanceCost), encode(p.FundAccountCost), encode(p.LatestRevisionCost), encode(p.SubscriptionMemoryCost), encode(p.SubscriptionNotificationCost), encode(p.InitBaseCost), encode(p.MemoryTimeCost), encode(p.DownloadBandwidthCost), encode(p.UploadBandwidthCost), encode(p.DropSectorsBaseCost), encode(p.DropSectorsUnitCost), encode(p.HasSectorBaseCost), encode(p.ReadBaseCost), encode(p.ReadLengthCost), encode(p.RenewContractCost), encode(p.RevisionBaseCost), encode(p.SwapSectorBaseCost), encode(p.WriteBaseCost), encode(p.WriteLengthCost), encode(p.WriteStoreCost), encode(p.TxnFeeMinRecommended), encode(p.TxnFeeMaxRecommended), encode(p.ContractPrice), encode(p.CollateralCost), encode(p.MaxCollateral), encode(p.MaxDuration), encode(p.WindowSize), encode(p.RegistryEntriesLeft), encode(p.RegistryEntriesTotal), sV4.ProtocolVersion[:], sV4.Release, encode(sV4.WalletAddress), sV4.AcceptingContracts, encode(sV4.MaxCollateral), encode(sV4.MaxContractDuration), encode(sV4.MaxSectorDuration), encode(sV4.MaxSectorBatchSize), encode(sV4.RemainingStorage), encode(sV4.TotalStorage), encode(pV4.ContractPrice), encode(pV4.Collateral), encode(pV4.StoragePrice), encode(pV4.IngressPrice), encode(pV4.EgressPrice), encode(pV4.FreeSectorPrice), encode(pV4.TipHeight), encode(pV4.ValidUntil), encode(pV4.Signature), encode(scan.PublicKey)); err != nil { return err } } diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index 6cc2ea96..ecefa9cb 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -504,7 +504,28 @@ CREATE TABLE host_info ( price_table_max_duration BLOB NOT NULL, price_table_window_size BLOB NOT NULL, price_table_registry_entries_left BLOB NOT NULL, - price_table_registry_entries_total BLOB NOT NULL + price_table_registry_entries_total BLOB NOT NULL, + -- rhp4 settings + rhp4_settings_protocol_version BLOB NOT NULL, + rhp4_settings_release TEXT NOT NULL, + rhp4_settings_wallet_address BLOB NOT NULL, + rhp4_settings_accepting_contracts INTEGER NOT NULL, + rhp4_settings_max_collateral BLOB NOT NULL, + rhp4_settings_max_collateral_duration BLOB NOT NULL, + rhp4_settings_max_sector_duration BLOB NOT NULL, + rhp4_settings_max_sector_batch_size BLOB NOT NULL, + rhp4_settings_remaining_storage BLOB NOT NULL, + rhp4_settings_total_storage BLOB NOT NULL, + -- rhp4 prices + rhp4_prices_contract_price BLOB NOT NULL, + rhp4_prices_collateral_price BLOB NOT NULL, + rhp4_prices_storage_price BLOB NOT NULL, + rhp4_prices_ingress_price BLOB NOT NULL, + rhp4_prices_egress_price BLOB NOT NULL, + rhp4_prices_free_sector_price BLOB NOT NULL, + rhp4_prices_tip_height BLOB NOT NULL, + rhp4_prices_valid_until BLOB NOT NULL, + rhp4_prices_signature BLOB NOT NULL ); CREATE TABLE host_info_v2_netaddresses( @@ -513,7 +534,7 @@ CREATE TABLE host_info_v2_netaddresses( protocol TEXT NOT NULL, address TEXT NOT NULL, - UNIQUE(public_key, netaddress_order) + PRIMARY KEY(public_key, netaddress_order) ); CREATE INDEX host_info_v2_netaddresses_public_key ON host_info_v2_netaddresses(public_key); diff --git a/persist/sqlite/scan_test.go b/persist/sqlite/scan_test.go index 8b0e4642..36e02f7d 100644 --- a/persist/sqlite/scan_test.go +++ b/persist/sqlite/scan_test.go @@ -3,19 +3,27 @@ package sqlite import ( "context" "errors" + "net" "path/filepath" "sort" "testing" "time" + "go.sia.tech/core/consensus" + "go.sia.tech/core/gateway" + proto4 "go.sia.tech/core/rhp/v4" "go.sia.tech/core/types" - "go.sia.tech/coreutils" "go.sia.tech/coreutils/chain" + crhpv4 "go.sia.tech/coreutils/rhp/v4" + "go.sia.tech/coreutils/syncer" ctestutil "go.sia.tech/coreutils/testutil" + "go.sia.tech/coreutils/wallet" "go.sia.tech/explored/config" "go.sia.tech/explored/explorer" "go.sia.tech/explored/internal/testutil" + "lukechampine.com/frand" + "go.uber.org/zap" "go.uber.org/zap/zaptest" ) @@ -43,6 +51,69 @@ func syncDB(t *testing.T, db *Store, cm *chain.Manager) { } } +func startTestNode(tb testing.TB, n *consensus.Network, genesis types.Block) (*chain.Manager, *syncer.Syncer, *wallet.SingleAddressWallet) { + db, tipstate, err := chain.NewDBStore(chain.NewMemDB(), n, genesis) + if err != nil { + tb.Fatal(err) + } + cm := chain.NewManager(db, tipstate) + + syncerListener, err := net.Listen("tcp", ":0") + if err != nil { + tb.Fatal(err) + } + tb.Cleanup(func() { syncerListener.Close() }) + + s := syncer.New(syncerListener, cm, ctestutil.NewMemPeerStore(), gateway.Header{ + GenesisID: genesis.ID(), + UniqueID: gateway.GenerateUniqueID(), + NetAddress: "localhost:1234", + }) + go s.Run(context.Background()) + tb.Cleanup(func() { s.Close() }) + + ws := ctestutil.NewEphemeralWalletStore() + w, err := wallet.NewSingleAddressWallet(types.GeneratePrivateKey(), cm, ws) + if err != nil { + tb.Fatal(err) + } + tb.Cleanup(func() { w.Close() }) + + reorgCh := make(chan struct{}, 1) + tb.Cleanup(func() { close(reorgCh) }) + + go func() { + for range reorgCh { + reverted, applied, err := cm.UpdatesSince(w.Tip(), 1000) + if err != nil { + tb.Error(err) + } + + err = ws.UpdateChainState(func(tx wallet.UpdateTx) error { + return w.UpdateChainState(tx, reverted, applied) + }) + if err != nil { + tb.Error(err) + } + } + }() + + stop := cm.OnReorg(func(index types.ChainIndex) { + select { + case reorgCh <- struct{}{}: + default: + } + }) + tb.Cleanup(stop) + + return cm, s, w +} + +func testRenterHostPair(tb testing.TB, hostKey types.PrivateKey, cm crhpv4.ChainManager, s crhpv4.Syncer, w crhpv4.Wallet, c crhpv4.Contractor, sr crhpv4.Settings, ss crhpv4.Sectors, log *zap.Logger) string { + rs := crhpv4.NewServer(hostKey, cm, s, c, w, sr, ss, crhpv4.WithContractProofWindowBuffer(10), crhpv4.WithPriceTableValidity(2*time.Minute)) + return ctestutil.ServeSiaMux(tb, rs, log.Named("siamux")) +} + func TestScan(t *testing.T) { if testing.Short() { t.Skip() @@ -50,26 +121,44 @@ func TestScan(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() + db, err := OpenDatabase(filepath.Join(dir, "explored.sqlite3"), log.Named("sqlite3")) if err != nil { t.Fatal(err) } defer db.Close() - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - network, genesisBlock := ctestutil.Network() - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) - if err != nil { - t.Fatal(err) - } + cm, s, w := startTestNode(t, network, genesisBlock) + + sr := ctestutil.NewEphemeralSettingsReporter() + sr.Update(proto4.HostSettings{ + ProtocolVersion: [3]uint8{1, 2, 3}, + Release: "test", + AcceptingContracts: true, + WalletAddress: w.Address(), + MaxCollateral: types.Siacoins(10000), + MaxContractDuration: 1000, + MaxSectorDuration: 3 * 144, + MaxSectorBatchSize: 100, + RemainingStorage: 100 * proto4.SectorSize, + TotalStorage: 100 * proto4.SectorSize, + Prices: proto4.HostPrices{ + ContractPrice: types.Siacoins(uint32(frand.Uint64n(10000))), + StoragePrice: types.Siacoins(uint32(frand.Uint64n(10000))), + IngressPrice: types.Siacoins(uint32(frand.Uint64n(10000))), + EgressPrice: types.Siacoins(uint32(frand.Uint64n(10000))), + Collateral: types.Siacoins(uint32(frand.Uint64n(10000))), + }, + }) + ss := ctestutil.NewEphemeralSectorStore() + c := ctestutil.NewEphemeralContractor(cm) + + pk0 := types.GeneratePrivateKey() + pubkey0 := pk0.PublicKey() - cm := chain.NewManager(store, genesisState) + v4Addr := testRenterHostPair(t, pk0, cm, s, w, c, sr, ss, zap.NewNop()) cfg := config.Scanner{ Threads: 10, @@ -94,6 +183,12 @@ func TestScan(t *testing.T) { ts := types.CurrentTimestamp() hosts := []explorer.Host{ + { + PublicKey: pubkey0, + V2NetAddresses: []chain.NetAddress{{Protocol: crhpv4.ProtocolTCPSiaMux, Address: v4Addr}}, + LastAnnouncement: ts, + KnownSince: ts, + }, { PublicKey: pubkey1, NetAddress: "sia1.siahost.ca:9982", @@ -115,18 +210,18 @@ func TestScan(t *testing.T) { } // explorer won't start scanning till a recent block is mined - b := testutil.MineBlock(genesisState, nil, types.VoidAddress) + b := testutil.MineBlock(cm.TipState(), nil, types.VoidAddress) if err := cm.AddBlocks([]types.Block{b}); err != nil { t.Fatal(err) } - time.Sleep(3 * cfg.Timeout) + time.Sleep(4 * cfg.Timeout) - dbHosts, err := e.Hosts([]types.PublicKey{pubkey1, pubkey2}) + dbHosts, err := e.Hosts([]types.PublicKey{pubkey0, pubkey1, pubkey2}) if err != nil { t.Fatal(err) } - testutil.Equal(t, "len(dbHosts)", 2, len(dbHosts)) + testutil.Equal(t, "len(dbHosts)", 3, len(dbHosts)) sort.Slice(hosts, func(i, j int) bool { return hosts[i].NetAddress < hosts[j].NetAddress @@ -134,17 +229,29 @@ func TestScan(t *testing.T) { sort.Slice(dbHosts, func(i, j int) bool { return dbHosts[i].NetAddress < dbHosts[j].NetAddress }) - host1 := dbHosts[0] - testutil.Equal(t, "host1.NetAddress", hosts[0].NetAddress, host1.NetAddress) - testutil.Equal(t, "host1.PublicKey", hosts[0].PublicKey, host1.PublicKey) + + host0 := dbHosts[0] + testutil.Equal(t, "host0.NetAddress", hosts[0].NetAddress, host0.NetAddress) + testutil.Equal(t, "host0.PublicKey", hosts[0].PublicKey, host0.PublicKey) + testutil.Equal(t, "host0.TotalScans", 1, host0.TotalScans) + testutil.Equal(t, "host0.SuccessfulInteractions", 1, host0.SuccessfulInteractions) + testutil.Equal(t, "host0.FailedInteractions", 0, host0.FailedInteractions) + testutil.Equal(t, "host0.LastScanSuccessful", true, host0.LastScanSuccessful) + if !host0.RHPV4Settings.AcceptingContracts { + log.Fatal("AcceptingContracts = false on host that's supposed to be active") + } + + host1 := dbHosts[1] + testutil.Equal(t, "host1.NetAddress", hosts[1].NetAddress, host1.NetAddress) + testutil.Equal(t, "host1.PublicKey", hosts[1].PublicKey, host1.PublicKey) testutil.Equal(t, "host1.TotalScans", 1, host1.TotalScans) testutil.Equal(t, "host1.SuccessfulInteractions", 0, host1.SuccessfulInteractions) testutil.Equal(t, "host1.FailedInteractions", 1, host1.FailedInteractions) testutil.Equal(t, "host1.LastScanSuccessful", false, host1.LastScanSuccessful) - host2 := dbHosts[1] - testutil.Equal(t, "host2.NetAddress", hosts[1].NetAddress, host2.NetAddress) - testutil.Equal(t, "host2.PublicKey", hosts[1].PublicKey, host2.PublicKey) + host2 := dbHosts[2] + testutil.Equal(t, "host2.NetAddress", hosts[2].NetAddress, host2.NetAddress) + testutil.Equal(t, "host2.PublicKey", hosts[2].PublicKey, host2.PublicKey) testutil.Equal(t, "host2.CountryCode", "CA", host2.CountryCode) testutil.Equal(t, "host2.TotalScans", 1, host2.TotalScans) testutil.Equal(t, "host2.SuccessfulInteractions", 1, host2.SuccessfulInteractions)