diff --git a/README.md b/README.md index 3b5c2d3c..6083abd2 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ [![GoDoc](https://godoc.org/go.sia.tech/explored?status.svg)](https://godoc.org/go.sia.tech/explored) `explored` is an explorer for Sia. + +## Required Disclosure + +`explored` uses the IP2Location LITE database for IP geolocation. \ No newline at end of file diff --git a/explorer/scan.go b/explorer/scan.go index d6fdf4a6..5fe4b8f9 100644 --- a/explorer/scan.go +++ b/explorer/scan.go @@ -10,6 +10,7 @@ import ( crhpv2 "go.sia.tech/core/rhp/v2" "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" + "go.sia.tech/explored/internal/geoip" rhpv2 "go.sia.tech/explored/internal/rhp/v2" rhpv3 "go.sia.tech/explored/internal/rhp/v3" "go.uber.org/zap" @@ -51,7 +52,7 @@ func (e *Explorer) waitForSync() error { return nil } -func (e *Explorer) scanHost(host chain.HostAnnouncement) (HostScan, error) { +func (e *Explorer) scanHost(locator geoip.Locator, host chain.HostAnnouncement) (HostScan, error) { ctx, cancel := context.WithTimeout(e.ctx, e.scanCfg.Timeout) defer cancel() @@ -78,6 +79,18 @@ func (e *Explorer) scanHost(host chain.HostAnnouncement) (HostScan, error) { return HostScan{}, fmt.Errorf("scanHost: failed to parse net address: %w", err) } + resolved, err := net.ResolveIPAddr("ip", hostIP) + // if we can resolve the address + if err != nil { + return HostScan{}, fmt.Errorf("scanHost: failed to resolve host address: %w", err) + } + + countryCode, err := locator.CountryCode(resolved) + if err != nil { + e.log.Debug("Failed to resolve IP geolocation, not setting country code", zap.String("addr", host.NetAddress)) + countryCode = "" + } + v3Addr := net.JoinHostPort(hostIP, settings.SiaMuxPort) v3Session, err := rhpv3.NewSession(ctx, host.PublicKey, v3Addr, e.cm, nil) if err != nil { @@ -90,9 +103,10 @@ func (e *Explorer) scanHost(host chain.HostAnnouncement) (HostScan, error) { } return HostScan{ - PublicKey: host.PublicKey, - Success: true, - Timestamp: types.CurrentTimestamp(), + PublicKey: host.PublicKey, + CountryCode: countryCode, + Success: true, + Timestamp: types.CurrentTimestamp(), Settings: settings, PriceTable: table, @@ -100,6 +114,14 @@ func (e *Explorer) scanHost(host chain.HostAnnouncement) (HostScan, error) { } func (e *Explorer) addHostScans(hosts chan chain.HostAnnouncement) { + // use default included ip2location database + locator, err := geoip.NewIP2LocationLocator("") + if err != nil { + e.log.Error("Failed to create geoip database", zap.Error(err)) + return + } + defer locator.Close() + worker := func() { var scans []HostScan for host := range hosts { @@ -107,7 +129,7 @@ func (e *Explorer) addHostScans(hosts chan chain.HostAnnouncement) { break } - scan, err := e.scanHost(host) + scan, err := e.scanHost(locator, host) if err != nil { scans = append(scans, HostScan{ PublicKey: host.PublicKey, diff --git a/explorer/types.go b/explorer/types.go index 57733697..269aa60a 100644 --- a/explorer/types.go +++ b/explorer/types.go @@ -221,9 +221,10 @@ type Metrics struct { // HostScan represents the results of a host scan. type HostScan struct { - PublicKey types.PublicKey `json:"publicKey"` - Success bool `json:"success"` - Timestamp time.Time `json:"timestamp"` + PublicKey types.PublicKey `json:"publicKey"` + CountryCode string `json:"countryCode"` + Success bool `json:"success"` + Timestamp time.Time `json:"timestamp"` Settings rhpv2.HostSettings `json:"settings"` PriceTable rhpv3.HostPriceTable `json:"priceTable"` @@ -231,8 +232,9 @@ type HostScan struct { // Host represents a host and the information gathered from scanning it. type Host struct { - PublicKey types.PublicKey `json:"publicKey"` - NetAddress string `json:"netAddress"` + PublicKey types.PublicKey `json:"publicKey"` + NetAddress string `json:"netAddress"` + CountryCode string `json:"countryCode"` KnownSince time.Time `json:"knownSince"` LastScan time.Time `json:"lastScan"` diff --git a/go.mod b/go.mod index f6aaa199..b62b0836 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( ) require ( + github.com/ip2location/ip2location-go v8.3.0+incompatible // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect diff --git a/go.sum b/go.sum index 6ec573e3..19629693 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/ip2location/ip2location-go v8.3.0+incompatible h1:QwUE+FlSbo6bjOWZpv2Grb57vJhWYFNPyBj2KCvfWaM= +github.com/ip2location/ip2location-go v8.3.0+incompatible/go.mod h1:3JUY1TBjTx1GdA7oRT7Zeqfc0bg3lMMuU5lXmzdpuME= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= diff --git a/internal/geoip/IP2LOCATION-LITE-DB1.BIN b/internal/geoip/IP2LOCATION-LITE-DB1.BIN new file mode 100755 index 00000000..4574d4bf Binary files /dev/null and b/internal/geoip/IP2LOCATION-LITE-DB1.BIN differ diff --git a/internal/geoip/geoip.go b/internal/geoip/geoip.go new file mode 100644 index 00000000..efccd470 --- /dev/null +++ b/internal/geoip/geoip.go @@ -0,0 +1,81 @@ +package geoip + +import ( + _ "embed" // needed for geolocation database + "errors" + "net" + "os" + "sync" + + "github.com/ip2location/ip2location-go" +) + +//go:embed IP2LOCATION-LITE-DB1.BIN +var ip2LocationDB []byte + +// A Locator maps IP addresses to their location. +// It is assumed that it implementations are thread-safe. +type Locator interface { + // Close closes the Locator. + Close() error + // CountryCode maps IP addresses to ISO 3166-1 A-2 country codes. + CountryCode(ip *net.IPAddr) (string, error) +} + +type ip2Location struct { + mu sync.Mutex + + path string + db *ip2location.DB +} + +// Close implements Locator. +func (ip *ip2Location) Close() error { + ip.mu.Lock() + defer ip.mu.Unlock() + + ip.db.Close() + return os.Remove(ip.path) +} + +// CountryCode implements Locator. +func (ip *ip2Location) CountryCode(addr *net.IPAddr) (string, error) { + if ip == nil { + return "", errors.New("nil IP") + } + ip.mu.Lock() + defer ip.mu.Unlock() + + loc, err := ip.db.Get_country_short(addr.String()) + if err != nil { + return "", err + } + return loc.Country_short, nil +} + +// NewIP2LocationLocator returns a Locator that uses an underlying IP2Location +// database. If no path is provided, a default embedded LITE database is used. +func NewIP2LocationLocator(path string) (Locator, error) { + // Unfortunately, ip2location.OpenDB only accepts a file path. So we need + // to write the embedded file to a temporary file on disk, and use that + // instead. + if path == "" { + f, err := os.CreateTemp("", "geoip") + if err != nil { + return nil, err + } else if _, err := f.Write(ip2LocationDB); err != nil { + return nil, err + } else if err := f.Sync(); err != nil { + return nil, err + } else if err := f.Close(); err != nil { + return nil, err + } + path = f.Name() + } + + db, err := ip2location.OpenDB(path) + if err != nil { + return nil, err + } + return &ip2Location{path: path, db: db}, nil +} diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index 73166838..296adb44 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -100,14 +100,14 @@ WHERE ev.event_id = ?`, eventID).Scan(decode(&m.SiacoinOutput.StateElement.ID), } // Hosts returns the hosts with the given public keys. -func (s *Store) Hosts(pks []types.PublicKey) (result []explorer.Host, err error) { - err = s.transaction(func(tx *txn) error { +func (st *Store) Hosts(pks []types.PublicKey) (result []explorer.Host, err error) { + err = st.transaction(func(tx *txn) error { var encoded []any for _, pk := range pks { encoded = append(encoded, encode(pk)) } - rows, err := tx.Query(`SELECT public_key,net_address,known_since,last_scan,last_scan_successful,last_announcement,total_scans,successful_interactions,failed_interactions,settings_accepting_contracts,settings_max_download_batch_size,settings_max_duration,settings_max_revise_batch_size,settings_net_address,settings_remaining_storage,settings_sector_size,settings_total_storage,settings_address,settings_window_size,settings_collateral,settings_max_collateral,settings_base_rpc_price,settings_contract_price,settings_download_bandwidth_price,settings_sector_access_price,settings_storage_price,settings_upload_bandwidth_price,settings_ephemeral_account_expiry,settings_max_ephemeral_account_balance,settings_revision_number,settings_version,settings_release,settings_sia_mux_port,price_table_uid,price_table_validity,price_table_host_block_height,price_table_update_price_table_cost,price_table_account_balance_cost,price_table_fund_account_cost,price_table_latest_revision_cost,price_table_subscription_memory_cost,price_table_subscription_notification_cost,price_table_init_base_cost,price_table_memory_time_cost,price_table_download_bandwidth_cost,price_table_upload_bandwidth_cost,price_table_drop_sectors_base_cost,price_table_drop_sectors_unit_cost,price_table_has_sector_base_cost,price_table_read_base_cost,price_table_read_length_cost,price_table_renew_contract_cost,price_table_revision_base_cost,price_table_swap_sector_base_cost,price_table_write_base_cost,price_table_write_length_cost,price_table_write_store_cost,price_table_txn_fee_min_recommended,price_table_txn_fee_max_recommended,price_table_contract_price,price_table_collateral_cost,price_table_max_collateral,price_table_max_duration,price_table_window_size,price_table_registry_entries_left,price_table_registry_entries_total FROM host_info WHERE public_key IN (`+queryPlaceHolders(len(pks))+`)`, encoded...) + rows, err := tx.Query(`SELECT public_key,net_address,country_code,known_since,last_scan,last_scan_successful,last_announcement,total_scans,successful_interactions,failed_interactions,settings_accepting_contracts,settings_max_download_batch_size,settings_max_duration,settings_max_revise_batch_size,settings_net_address,settings_remaining_storage,settings_sector_size,settings_total_storage,settings_address,settings_window_size,settings_collateral,settings_max_collateral,settings_base_rpc_price,settings_contract_price,settings_download_bandwidth_price,settings_sector_access_price,settings_storage_price,settings_upload_bandwidth_price,settings_ephemeral_account_expiry,settings_max_ephemeral_account_balance,settings_revision_number,settings_version,settings_release,settings_sia_mux_port,price_table_uid,price_table_validity,price_table_host_block_height,price_table_update_price_table_cost,price_table_account_balance_cost,price_table_fund_account_cost,price_table_latest_revision_cost,price_table_subscription_memory_cost,price_table_subscription_notification_cost,price_table_init_base_cost,price_table_memory_time_cost,price_table_download_bandwidth_cost,price_table_upload_bandwidth_cost,price_table_drop_sectors_base_cost,price_table_drop_sectors_unit_cost,price_table_has_sector_base_cost,price_table_read_base_cost,price_table_read_length_cost,price_table_renew_contract_cost,price_table_revision_base_cost,price_table_swap_sector_base_cost,price_table_write_base_cost,price_table_write_length_cost,price_table_write_store_cost,price_table_txn_fee_min_recommended,price_table_txn_fee_max_recommended,price_table_contract_price,price_table_collateral_cost,price_table_max_collateral,price_table_max_duration,price_table_window_size,price_table_registry_entries_left,price_table_registry_entries_total FROM host_info WHERE public_key IN (`+queryPlaceHolders(len(pks))+`)`, encoded...) if err != nil { return err } @@ -116,9 +116,10 @@ func (s *Store) Hosts(pks []types.PublicKey) (result []explorer.Host, err error) for rows.Next() { var host explorer.Host s, p := &host.Settings, &host.PriceTable - if err := rows.Scan(decode(&host.PublicKey), &host.NetAddress, decode(&host.KnownSince), decode(&host.LastScan), &host.LastScanSuccessful, decode(&host.LastAnnouncement), &host.TotalScans, &host.SuccessfulInteractions, &host.FailedInteractions, &s.AcceptingContracts, decode(&s.MaxDownloadBatchSize), decode(&s.MaxDuration), decode(&s.MaxReviseBatchSize), &s.NetAddress, decode(&s.RemainingStorage), decode(&s.SectorSize), decode(&s.TotalStorage), decode(&s.Address), decode(&s.WindowSize), decode(&s.Collateral), decode(&s.MaxCollateral), decode(&s.BaseRPCPrice), decode(&s.ContractPrice), decode(&s.DownloadBandwidthPrice), decode(&s.SectorAccessPrice), decode(&s.StoragePrice), decode(&s.UploadBandwidthPrice), &s.EphemeralAccountExpiry, decode(&s.MaxEphemeralAccountBalance), decode(&s.RevisionNumber), &s.Version, &s.Release, &s.SiaMuxPort, decode(&p.UID), &p.Validity, decode(&p.HostBlockHeight), decode(&p.UpdatePriceTableCost), decode(&p.AccountBalanceCost), decode(&p.FundAccountCost), decode(&p.LatestRevisionCost), decode(&p.SubscriptionMemoryCost), decode(&p.SubscriptionNotificationCost), decode(&p.InitBaseCost), decode(&p.MemoryTimeCost), decode(&p.DownloadBandwidthCost), decode(&p.UploadBandwidthCost), decode(&p.DropSectorsBaseCost), decode(&p.DropSectorsUnitCost), decode(&p.HasSectorBaseCost), decode(&p.ReadBaseCost), decode(&p.ReadLengthCost), decode(&p.RenewContractCost), decode(&p.RevisionBaseCost), decode(&p.SwapSectorBaseCost), decode(&p.WriteBaseCost), decode(&p.WriteLengthCost), decode(&p.WriteStoreCost), decode(&p.TxnFeeMinRecommended), decode(&p.TxnFeeMaxRecommended), decode(&p.ContractPrice), decode(&p.CollateralCost), decode(&p.MaxCollateral), decode(&p.MaxDuration), decode(&p.WindowSize), decode(&p.RegistryEntriesLeft), decode(&p.RegistryEntriesTotal)); err != nil { + if err := rows.Scan(decode(&host.PublicKey), &host.NetAddress, &host.CountryCode, decode(&host.KnownSince), decode(&host.LastScan), &host.LastScanSuccessful, decode(&host.LastAnnouncement), &host.TotalScans, &host.SuccessfulInteractions, &host.FailedInteractions, &s.AcceptingContracts, decode(&s.MaxDownloadBatchSize), decode(&s.MaxDuration), decode(&s.MaxReviseBatchSize), &s.NetAddress, decode(&s.RemainingStorage), decode(&s.SectorSize), decode(&s.TotalStorage), decode(&s.Address), decode(&s.WindowSize), decode(&s.Collateral), decode(&s.MaxCollateral), decode(&s.BaseRPCPrice), decode(&s.ContractPrice), decode(&s.DownloadBandwidthPrice), decode(&s.SectorAccessPrice), decode(&s.StoragePrice), decode(&s.UploadBandwidthPrice), &s.EphemeralAccountExpiry, decode(&s.MaxEphemeralAccountBalance), decode(&s.RevisionNumber), &s.Version, &s.Release, &s.SiaMuxPort, decode(&p.UID), &p.Validity, decode(&p.HostBlockHeight), decode(&p.UpdatePriceTableCost), decode(&p.AccountBalanceCost), decode(&p.FundAccountCost), decode(&p.LatestRevisionCost), decode(&p.SubscriptionMemoryCost), decode(&p.SubscriptionNotificationCost), decode(&p.InitBaseCost), decode(&p.MemoryTimeCost), decode(&p.DownloadBandwidthCost), decode(&p.UploadBandwidthCost), decode(&p.DropSectorsBaseCost), decode(&p.DropSectorsUnitCost), decode(&p.HasSectorBaseCost), decode(&p.ReadBaseCost), decode(&p.ReadLengthCost), decode(&p.RenewContractCost), decode(&p.RevisionBaseCost), decode(&p.SwapSectorBaseCost), decode(&p.WriteBaseCost), decode(&p.WriteLengthCost), decode(&p.WriteStoreCost), decode(&p.TxnFeeMinRecommended), decode(&p.TxnFeeMaxRecommended), decode(&p.ContractPrice), decode(&p.CollateralCost), decode(&p.MaxCollateral), decode(&p.MaxDuration), decode(&p.WindowSize), decode(&p.RegistryEntriesLeft), decode(&p.RegistryEntriesTotal)); err != nil { return err } + result = append(result, host) } return nil diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 65fec026..bd3e0971 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -1129,7 +1129,7 @@ func (ut *updateTx) RevertIndex(state explorer.UpdateState) error { } func addHosts(tx *txn, scans []explorer.Host) error { - stmt, err := tx.Prepare(`INSERT INTO host_info(public_key, net_address, known_since, last_scan, last_scan_successful, last_announcement, total_scans, successful_interactions, failed_interactions, settings_accepting_contracts, settings_max_download_batch_size, settings_max_duration, settings_max_revise_batch_size, settings_net_address, settings_remaining_storage, settings_sector_size, settings_total_storage, settings_address, settings_window_size, settings_collateral, settings_max_collateral, settings_base_rpc_price, settings_contract_price, settings_download_bandwidth_price, settings_sector_access_price, settings_storage_price, settings_upload_bandwidth_price, settings_ephemeral_account_expiry, settings_max_ephemeral_account_balance, settings_revision_number, settings_version, settings_release, settings_sia_mux_port, price_table_uid, price_table_validity, price_table_host_block_height, price_table_update_price_table_cost, price_table_account_balance_cost, price_table_fund_account_cost, price_table_latest_revision_cost, price_table_subscription_memory_cost, price_table_subscription_notification_cost, price_table_init_base_cost, price_table_memory_time_cost, price_table_download_bandwidth_cost, price_table_upload_bandwidth_cost, price_table_drop_sectors_base_cost, price_table_drop_sectors_unit_cost, price_table_has_sector_base_cost, price_table_read_base_cost, price_table_read_length_cost, price_table_renew_contract_cost, price_table_revision_base_cost, price_table_swap_sector_base_cost, price_table_write_base_cost, price_table_write_length_cost, price_table_write_store_cost, price_table_txn_fee_min_recommended, price_table_txn_fee_max_recommended, price_table_contract_price, price_table_collateral_cost, price_table_max_collateral, price_table_max_duration, price_table_window_size, price_table_registry_entries_left, price_table_registry_entries_total) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26,$27,$28,$29,$30,$31,$32,$33,$34,$35,$36,$37,$38,$39,$40,$41,$42,$43,$44,$45,$46,$47,$48,$49,$50,$51,$52,$53,$54,$55,$56,$57,$58,$59,$60,$61,$62,$63,$64,$65,$66) ON CONFLICT (public_key) DO UPDATE SET net_address = $2, last_scan = $4, last_scan_successful = $5, last_announcement = CASE WHEN $6 > 0 THEN last_announcement ELSE $6 END, total_scans = $7, successful_interactions = $8, failed_interactions = failed_interactions + $9, settings_accepting_contracts = $10, settings_max_download_batch_size = $11, settings_max_duration = $12, settings_max_revise_batch_size = $13, settings_net_address = $14, settings_remaining_storage = $15, settings_sector_size = $16, settings_total_storage = $17, settings_address = $18, settings_window_size = $19, settings_collateral = $20, settings_max_collateral = $21, settings_base_rpc_price = $22, settings_contract_price = $23, settings_download_bandwidth_price = $24, settings_sector_access_price = $25, settings_storage_price = $26, settings_upload_bandwidth_price = $27, settings_ephemeral_account_expiry = $28, settings_max_ephemeral_account_balance = $29, settings_revision_number = $30, settings_version = $31, settings_release = $32, settings_sia_mux_port = $33, price_table_uid = $34, price_table_validity = $35, price_table_host_block_height = $36, price_table_update_price_table_cost = $37, price_table_account_balance_cost = $38, price_table_fund_account_cost = $39, price_table_latest_revision_cost = $40, price_table_subscription_memory_cost = $41, price_table_subscription_notification_cost = $42, price_table_init_base_cost = $43, price_table_memory_time_cost = $44, price_table_download_bandwidth_cost = $45, price_table_upload_bandwidth_cost = $46, price_table_drop_sectors_base_cost = $47, price_table_drop_sectors_unit_cost = $48, price_table_has_sector_base_cost = $49, price_table_read_base_cost = $50, price_table_read_length_cost = $51, price_table_renew_contract_cost = $52, price_table_revision_base_cost = $53, price_table_swap_sector_base_cost = $54, price_table_write_base_cost = $55, price_table_write_length_cost = $56, price_table_write_store_cost = $57, price_table_txn_fee_min_recommended = $58, price_table_txn_fee_max_recommended = $59, price_table_contract_price = $60, price_table_collateral_cost = $61, price_table_max_collateral = $62, price_table_max_duration = $63, price_table_window_size = $64, price_table_registry_entries_left = $65, price_table_registry_entries_total = $66`) + stmt, err := tx.Prepare(`INSERT INTO host_info(public_key, net_address, country_code, known_since, last_scan, last_scan_successful, last_announcement, total_scans, successful_interactions, failed_interactions, settings_accepting_contracts, settings_max_download_batch_size, settings_max_duration, settings_max_revise_batch_size, settings_net_address, settings_remaining_storage, settings_sector_size, settings_total_storage, settings_address, settings_window_size, settings_collateral, settings_max_collateral, settings_base_rpc_price, settings_contract_price, settings_download_bandwidth_price, settings_sector_access_price, settings_storage_price, settings_upload_bandwidth_price, settings_ephemeral_account_expiry, settings_max_ephemeral_account_balance, settings_revision_number, settings_version, settings_release, settings_sia_mux_port, price_table_uid, price_table_validity, price_table_host_block_height, price_table_update_price_table_cost, price_table_account_balance_cost, price_table_fund_account_cost, price_table_latest_revision_cost, price_table_subscription_memory_cost, price_table_subscription_notification_cost, price_table_init_base_cost, price_table_memory_time_cost, price_table_download_bandwidth_cost, price_table_upload_bandwidth_cost, price_table_drop_sectors_base_cost, price_table_drop_sectors_unit_cost, price_table_has_sector_base_cost, price_table_read_base_cost, price_table_read_length_cost, price_table_renew_contract_cost, price_table_revision_base_cost, price_table_swap_sector_base_cost, price_table_write_base_cost, price_table_write_length_cost, price_table_write_store_cost, price_table_txn_fee_min_recommended, price_table_txn_fee_max_recommended, price_table_contract_price, price_table_collateral_cost, price_table_max_collateral, price_table_max_duration, price_table_window_size, price_table_registry_entries_left, price_table_registry_entries_total) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26,$27,$28,$29,$30,$31,$32,$33,$34,$35,$36,$37,$38,$39,$40,$41,$42,$43,$44,$45,$46,$47,$48,$49,$50,$51,$52,$53,$54,$55,$56,$57,$58,$59,$60,$61,$62,$63,$64,$65,$66,$67) ON CONFLICT (public_key) DO UPDATE SET net_address = $2, country_code = $3, last_scan = $5, last_scan_successful = $6, last_announcement = CASE WHEN $7 > 0 THEN last_announcement ELSE $7 END, total_scans = $8, successful_interactions = $9, failed_interactions = failed_interactions + $10, settings_accepting_contracts = $11, settings_max_download_batch_size = $12, settings_max_duration = $13, settings_max_revise_batch_size = $14 , settings_net_address = $15, settings_remaining_storage = $16, settings_sector_size = $17, settings_total_storage = $18, settings_address = $19, settings_window_size = $20, settings_collateral = $21, settings_max_collateral = $22, settings_base_rpc_price = $23, settings_contract_price = $24, settings_download_bandwidth_price = $25, settings_sector_access_price = $26, settings_storage_price = $27, settings_upload_bandwidth_price = $28, settings_ephemeral_account_expiry = $29, settings_max_ephemeral_account_balance = $30, settings_revision_number = $31, settings_version = $32, settings_release = $33, settings_sia_mux_port = $34, price_table_uid = $35, price_table_validity = $36, price_table_host_block_height = $37, price_table_update_price_table_cost = $38, price_table_account_balance_cost = $39, price_table_fund_account_cost = $40, price_table_latest_revision_cost = $41, price_table_subscription_memory_cost = $42, price_table_subscription_notification_cost = $43, price_table_init_base_cost = $44, price_table_memory_time_cost = $45, price_table_download_bandwidth_cost = $46, price_table_upload_bandwidth_cost = $47, price_table_drop_sectors_base_cost = $48, price_table_drop_sectors_unit_cost = $49, price_table_has_sector_base_cost = $50, price_table_read_base_cost = $51, price_table_read_length_cost = $52, price_table_renew_contract_cost = $53, price_table_revision_base_cost = $54, price_table_swap_sector_base_cost = $55, price_table_write_base_cost = $56, price_table_write_length_cost = $57, price_table_write_store_cost = $58, price_table_txn_fee_min_recommended = $59, price_table_txn_fee_max_recommended = $60, price_table_contract_price = $61, price_table_collateral_cost = $62, price_table_max_collateral = $63, price_table_max_duration = $64, price_table_window_size = $65, price_table_registry_entries_left = $66, price_table_registry_entries_total = $67`) if err != nil { return err } @@ -1137,7 +1137,7 @@ func addHosts(tx *txn, scans []explorer.Host) error { for _, scan := range scans { s, p := scan.Settings, scan.PriceTable - if _, err := stmt.Exec(encode(scan.PublicKey), scan.NetAddress, encode(scan.KnownSince), encode(scan.LastScan), scan.LastScanSuccessful, encode(scan.LastAnnouncement), scan.TotalScans, scan.SuccessfulInteractions, scan.FailedInteractions, s.AcceptingContracts, encode(s.MaxDownloadBatchSize), encode(s.MaxDuration), encode(s.MaxReviseBatchSize), s.NetAddress, encode(s.RemainingStorage), encode(s.SectorSize), encode(s.TotalStorage), encode(s.Address), encode(s.WindowSize), encode(s.Collateral), encode(s.MaxCollateral), encode(s.BaseRPCPrice), encode(s.ContractPrice), encode(s.DownloadBandwidthPrice), encode(s.SectorAccessPrice), encode(s.StoragePrice), encode(s.UploadBandwidthPrice), s.EphemeralAccountExpiry, encode(s.MaxEphemeralAccountBalance), encode(s.RevisionNumber), s.Version, s.Release, s.SiaMuxPort, encode(p.UID), p.Validity, encode(p.HostBlockHeight), encode(p.UpdatePriceTableCost), encode(p.AccountBalanceCost), encode(p.FundAccountCost), encode(p.LatestRevisionCost), encode(p.SubscriptionMemoryCost), encode(p.SubscriptionNotificationCost), encode(p.InitBaseCost), encode(p.MemoryTimeCost), encode(p.DownloadBandwidthCost), encode(p.UploadBandwidthCost), encode(p.DropSectorsBaseCost), encode(p.DropSectorsUnitCost), encode(p.HasSectorBaseCost), encode(p.ReadBaseCost), encode(p.ReadLengthCost), encode(p.RenewContractCost), encode(p.RevisionBaseCost), encode(p.SwapSectorBaseCost), encode(p.WriteBaseCost), encode(p.WriteLengthCost), encode(p.WriteStoreCost), encode(p.TxnFeeMinRecommended), encode(p.TxnFeeMaxRecommended), encode(p.ContractPrice), encode(p.CollateralCost), encode(p.MaxCollateral), encode(p.MaxDuration), encode(p.WindowSize), encode(p.RegistryEntriesLeft), encode(p.RegistryEntriesTotal)); err != nil { + if _, err := stmt.Exec(encode(scan.PublicKey), scan.NetAddress, scan.CountryCode, encode(scan.KnownSince), encode(scan.LastScan), scan.LastScanSuccessful, encode(scan.LastAnnouncement), scan.TotalScans, scan.SuccessfulInteractions, scan.FailedInteractions, s.AcceptingContracts, encode(s.MaxDownloadBatchSize), encode(s.MaxDuration), encode(s.MaxReviseBatchSize), s.NetAddress, encode(s.RemainingStorage), encode(s.SectorSize), encode(s.TotalStorage), encode(s.Address), encode(s.WindowSize), encode(s.Collateral), encode(s.MaxCollateral), encode(s.BaseRPCPrice), encode(s.ContractPrice), encode(s.DownloadBandwidthPrice), encode(s.SectorAccessPrice), encode(s.StoragePrice), encode(s.UploadBandwidthPrice), s.EphemeralAccountExpiry, encode(s.MaxEphemeralAccountBalance), encode(s.RevisionNumber), s.Version, s.Release, s.SiaMuxPort, encode(p.UID), p.Validity, encode(p.HostBlockHeight), encode(p.UpdatePriceTableCost), encode(p.AccountBalanceCost), encode(p.FundAccountCost), encode(p.LatestRevisionCost), encode(p.SubscriptionMemoryCost), encode(p.SubscriptionNotificationCost), encode(p.InitBaseCost), encode(p.MemoryTimeCost), encode(p.DownloadBandwidthCost), encode(p.UploadBandwidthCost), encode(p.DropSectorsBaseCost), encode(p.DropSectorsUnitCost), encode(p.HasSectorBaseCost), encode(p.ReadBaseCost), encode(p.ReadLengthCost), encode(p.RenewContractCost), encode(p.RevisionBaseCost), encode(p.SwapSectorBaseCost), encode(p.WriteBaseCost), encode(p.WriteLengthCost), encode(p.WriteStoreCost), encode(p.TxnFeeMinRecommended), encode(p.TxnFeeMaxRecommended), encode(p.ContractPrice), encode(p.CollateralCost), encode(p.MaxCollateral), encode(p.MaxDuration), encode(p.WindowSize), encode(p.RegistryEntriesLeft), encode(p.RegistryEntriesTotal)); err != nil { return err } } @@ -1145,7 +1145,7 @@ func addHosts(tx *txn, scans []explorer.Host) error { } func addHostScans(tx *txn, scans []explorer.HostScan) error { - stmt, err := tx.Prepare(`UPDATE host_info SET last_scan = ?, last_scan_successful = ?, total_scans = total_scans + 1, successful_interactions = successful_interactions + ?, failed_interactions = failed_interactions + ?, settings_accepting_contracts = ?, settings_max_download_batch_size = ?, settings_max_duration = ?, settings_max_revise_batch_size = ?, settings_net_address = ?, settings_remaining_storage = ?, settings_sector_size = ?, settings_total_storage = ?, settings_address = ?, settings_window_size = ?, settings_collateral = ?, settings_max_collateral = ?, settings_base_rpc_price = ?, settings_contract_price = ?, settings_download_bandwidth_price = ?, settings_sector_access_price = ?, settings_storage_price = ?, settings_upload_bandwidth_price = ?, settings_ephemeral_account_expiry = ?, settings_max_ephemeral_account_balance = ?, settings_revision_number = ?, settings_version = ?, settings_release = ?, settings_sia_mux_port = ?, price_table_uid = ?, price_table_validity = ?, price_table_host_block_height = ?, price_table_update_price_table_cost = ?, price_table_account_balance_cost = ?, price_table_fund_account_cost = ?, price_table_latest_revision_cost = ?, price_table_subscription_memory_cost = ?, price_table_subscription_notification_cost = ?, price_table_init_base_cost = ?, price_table_memory_time_cost = ?, price_table_download_bandwidth_cost = ?, price_table_upload_bandwidth_cost = ?, price_table_drop_sectors_base_cost = ?, price_table_drop_sectors_unit_cost = ?, price_table_has_sector_base_cost = ?, price_table_read_base_cost = ?, price_table_read_length_cost = ?, price_table_renew_contract_cost = ?, price_table_revision_base_cost = ?, price_table_swap_sector_base_cost = ?, price_table_write_base_cost = ?, price_table_write_length_cost = ?, price_table_write_store_cost = ?, price_table_txn_fee_min_recommended = ?, price_table_txn_fee_max_recommended = ?, price_table_contract_price = ?, price_table_collateral_cost = ?, price_table_max_collateral = ?, price_table_max_duration = ?, price_table_window_size = ?, price_table_registry_entries_left = ?, price_table_registry_entries_total = ? WHERE public_key = ?`) + stmt, err := tx.Prepare(`UPDATE host_info SET country_code = ?, last_scan = ?, last_scan_successful = ?, total_scans = total_scans + 1, successful_interactions = successful_interactions + ?, failed_interactions = failed_interactions + ?, settings_accepting_contracts = ?, settings_max_download_batch_size = ?, settings_max_duration = ?, settings_max_revise_batch_size = ?, settings_net_address = ?, settings_remaining_storage = ?, settings_sector_size = ?, settings_total_storage = ?, settings_address = ?, settings_window_size = ?, settings_collateral = ?, settings_max_collateral = ?, settings_base_rpc_price = ?, settings_contract_price = ?, settings_download_bandwidth_price = ?, settings_sector_access_price = ?, settings_storage_price = ?, settings_upload_bandwidth_price = ?, settings_ephemeral_account_expiry = ?, settings_max_ephemeral_account_balance = ?, settings_revision_number = ?, settings_version = ?, settings_release = ?, settings_sia_mux_port = ?, price_table_uid = ?, price_table_validity = ?, price_table_host_block_height = ?, price_table_update_price_table_cost = ?, price_table_account_balance_cost = ?, price_table_fund_account_cost = ?, price_table_latest_revision_cost = ?, price_table_subscription_memory_cost = ?, price_table_subscription_notification_cost = ?, price_table_init_base_cost = ?, price_table_memory_time_cost = ?, price_table_download_bandwidth_cost = ?, price_table_upload_bandwidth_cost = ?, price_table_drop_sectors_base_cost = ?, price_table_drop_sectors_unit_cost = ?, price_table_has_sector_base_cost = ?, price_table_read_base_cost = ?, price_table_read_length_cost = ?, price_table_renew_contract_cost = ?, price_table_revision_base_cost = ?, price_table_swap_sector_base_cost = ?, price_table_write_base_cost = ?, price_table_write_length_cost = ?, price_table_write_store_cost = ?, price_table_txn_fee_min_recommended = ?, price_table_txn_fee_max_recommended = ?, price_table_contract_price = ?, price_table_collateral_cost = ?, price_table_max_collateral = ?, price_table_max_duration = ?, price_table_window_size = ?, price_table_registry_entries_left = ?, price_table_registry_entries_total = ? WHERE public_key = ?`) if err != nil { return err } @@ -1158,7 +1158,7 @@ func addHostScans(tx *txn, scans []explorer.HostScan) error { } s, p := scan.Settings, scan.PriceTable - if _, err := stmt.Exec(encode(scan.Timestamp), scan.Success, successful, failed, s.AcceptingContracts, encode(s.MaxDownloadBatchSize), encode(s.MaxDuration), encode(s.MaxReviseBatchSize), s.NetAddress, encode(s.RemainingStorage), encode(s.SectorSize), encode(s.TotalStorage), encode(s.Address), encode(s.WindowSize), encode(s.Collateral), encode(s.MaxCollateral), encode(s.BaseRPCPrice), encode(s.ContractPrice), encode(s.DownloadBandwidthPrice), encode(s.SectorAccessPrice), encode(s.StoragePrice), encode(s.UploadBandwidthPrice), s.EphemeralAccountExpiry, encode(s.MaxEphemeralAccountBalance), encode(s.RevisionNumber), s.Version, s.Release, s.SiaMuxPort, encode(p.UID), p.Validity, encode(p.HostBlockHeight), encode(p.UpdatePriceTableCost), encode(p.AccountBalanceCost), encode(p.FundAccountCost), encode(p.LatestRevisionCost), encode(p.SubscriptionMemoryCost), encode(p.SubscriptionNotificationCost), encode(p.InitBaseCost), encode(p.MemoryTimeCost), encode(p.DownloadBandwidthCost), encode(p.UploadBandwidthCost), encode(p.DropSectorsBaseCost), encode(p.DropSectorsUnitCost), encode(p.HasSectorBaseCost), encode(p.ReadBaseCost), encode(p.ReadLengthCost), encode(p.RenewContractCost), encode(p.RevisionBaseCost), encode(p.SwapSectorBaseCost), encode(p.WriteBaseCost), encode(p.WriteLengthCost), encode(p.WriteStoreCost), encode(p.TxnFeeMinRecommended), encode(p.TxnFeeMaxRecommended), encode(p.ContractPrice), encode(p.CollateralCost), encode(p.MaxCollateral), encode(p.MaxDuration), encode(p.WindowSize), encode(p.RegistryEntriesLeft), encode(p.RegistryEntriesTotal), encode(scan.PublicKey)); err != nil { + if _, err := stmt.Exec(scan.CountryCode, encode(scan.Timestamp), scan.Success, successful, failed, s.AcceptingContracts, encode(s.MaxDownloadBatchSize), encode(s.MaxDuration), encode(s.MaxReviseBatchSize), s.NetAddress, encode(s.RemainingStorage), encode(s.SectorSize), encode(s.TotalStorage), encode(s.Address), encode(s.WindowSize), encode(s.Collateral), encode(s.MaxCollateral), encode(s.BaseRPCPrice), encode(s.ContractPrice), encode(s.DownloadBandwidthPrice), encode(s.SectorAccessPrice), encode(s.StoragePrice), encode(s.UploadBandwidthPrice), s.EphemeralAccountExpiry, encode(s.MaxEphemeralAccountBalance), encode(s.RevisionNumber), s.Version, s.Release, s.SiaMuxPort, encode(p.UID), p.Validity, encode(p.HostBlockHeight), encode(p.UpdatePriceTableCost), encode(p.AccountBalanceCost), encode(p.FundAccountCost), encode(p.LatestRevisionCost), encode(p.SubscriptionMemoryCost), encode(p.SubscriptionNotificationCost), encode(p.InitBaseCost), encode(p.MemoryTimeCost), encode(p.DownloadBandwidthCost), encode(p.UploadBandwidthCost), encode(p.DropSectorsBaseCost), encode(p.DropSectorsUnitCost), encode(p.HasSectorBaseCost), encode(p.ReadBaseCost), encode(p.ReadLengthCost), encode(p.RenewContractCost), encode(p.RevisionBaseCost), encode(p.SwapSectorBaseCost), encode(p.WriteBaseCost), encode(p.WriteLengthCost), encode(p.WriteStoreCost), encode(p.TxnFeeMinRecommended), encode(p.TxnFeeMaxRecommended), encode(p.ContractPrice), encode(p.CollateralCost), encode(p.MaxCollateral), encode(p.MaxDuration), encode(p.WindowSize), encode(p.RegistryEntriesLeft), encode(p.RegistryEntriesTotal), encode(scan.PublicKey)); err != nil { return err } } diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index a34074ca..05b56971 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -355,6 +355,7 @@ CREATE TABLE v2_transaction_events ( CREATE TABLE host_info ( public_key BLOB PRIMARY KEY NOT NULL, net_address TEXT NOT NULL, + country_code TEXT NOT NULL, known_since INTEGER NOT NULL, last_scan INTEGER NOT NULL, last_scan_successful INTEGER NOT NULL, diff --git a/persist/sqlite/scan_test.go b/persist/sqlite/scan_test.go index 14ccfeee..8b0e4642 100644 --- a/persist/sqlite/scan_test.go +++ b/persist/sqlite/scan_test.go @@ -145,8 +145,12 @@ func TestScan(t *testing.T) { host2 := dbHosts[1] testutil.Equal(t, "host2.NetAddress", hosts[1].NetAddress, host2.NetAddress) testutil.Equal(t, "host2.PublicKey", hosts[1].PublicKey, host2.PublicKey) + testutil.Equal(t, "host2.CountryCode", "CA", host2.CountryCode) testutil.Equal(t, "host2.TotalScans", 1, host2.TotalScans) testutil.Equal(t, "host2.SuccessfulInteractions", 1, host2.SuccessfulInteractions) testutil.Equal(t, "host2.FailedInteractions", 0, host2.FailedInteractions) testutil.Equal(t, "host2.LastScanSuccessful", true, host2.LastScanSuccessful) + if host2.Settings.SectorSize <= 0 { + log.Fatal("SectorSize = 0 on host that's supposed to be active") + } } diff --git a/persist/sqlite/store.go b/persist/sqlite/store.go index 1f79e421..55a2d8d8 100644 --- a/persist/sqlite/store.go +++ b/persist/sqlite/store.go @@ -103,6 +103,7 @@ func OpenDatabase(fp string, log *zap.Logger) (*Store, error) { if err != nil { return nil, err } + store := &Store{ db: db, log: log,