Merge pull request #131 from SiaFoundation/add-country-code

Add country code field for hosts

README.md

@@ -1,3 +1,7 @@
 [![GoDoc](https://godoc.org/go.sia.tech/explored?status.svg)](https://godoc.org/go.sia.tech/explored)
 
 `explored` is an explorer for Sia.
+
+## Required Disclosure
+
+`explored` uses the IP2Location LITE database for <a href="https://lite.ip2location.com">IP geolocation</a>.

explorer/scan.go

@@ -10,6 +10,7 @@ import (
 	crhpv2 "go.sia.tech/core/rhp/v2"
 	"go.sia.tech/core/types"
 	"go.sia.tech/coreutils/chain"
+	"go.sia.tech/explored/internal/geoip"
 	rhpv2 "go.sia.tech/explored/internal/rhp/v2"
 	rhpv3 "go.sia.tech/explored/internal/rhp/v3"
 	"go.uber.org/zap"
@@ -51,7 +52,7 @@ func (e *Explorer) waitForSync() error {
 	return nil
 }
 
-func (e *Explorer) scanHost(host chain.HostAnnouncement) (HostScan, error) {
+func (e *Explorer) scanHost(locator geoip.Locator, host chain.HostAnnouncement) (HostScan, error) {
 	ctx, cancel := context.WithTimeout(e.ctx, e.scanCfg.Timeout)
 	defer cancel()
 
@@ -78,6 +79,18 @@ func (e *Explorer) scanHost(host chain.HostAnnouncement) (HostScan, error) {
 		return HostScan{}, fmt.Errorf("scanHost: failed to parse net address: %w", err)
 	}
 
+	resolved, err := net.ResolveIPAddr("ip", hostIP)
+	// if we can resolve the address
+	if err != nil {
+		return HostScan{}, fmt.Errorf("scanHost: failed to resolve host address: %w", err)
+	}
+
+	countryCode, err := locator.CountryCode(resolved)
+	if err != nil {
+		e.log.Debug("Failed to resolve IP geolocation, not setting country code", zap.String("addr", host.NetAddress))
+		countryCode = ""
+	}
+
 	v3Addr := net.JoinHostPort(hostIP, settings.SiaMuxPort)
 	v3Session, err := rhpv3.NewSession(ctx, host.PublicKey, v3Addr, e.cm, nil)
 	if err != nil {
@@ -90,24 +103,33 @@ func (e *Explorer) scanHost(host chain.HostAnnouncement) (HostScan, error) {
 	}
 
 	return HostScan{
-		PublicKey: host.PublicKey,
-		Success:   true,
-		Timestamp: types.CurrentTimestamp(),
+		PublicKey:   host.PublicKey,
+		CountryCode: countryCode,
+		Success:     true,
+		Timestamp:   types.CurrentTimestamp(),
 
 		Settings:   settings,
 		PriceTable: table,
 	}, nil
 }
 
 func (e *Explorer) addHostScans(hosts chan chain.HostAnnouncement) {
+	// use default included ip2location database
+	locator, err := geoip.NewIP2LocationLocator("")
+	if err != nil {
+		e.log.Error("Failed to create geoip database", zap.Error(err))
+		return
+	}
+	defer locator.Close()
+
 	worker := func() {
 		var scans []HostScan
 		for host := range hosts {
 			if e.isClosed() {
 				break
 			}
 
-			scan, err := e.scanHost(host)
+			scan, err := e.scanHost(locator, host)
 			if err != nil {
 				scans = append(scans, HostScan{
 					PublicKey: host.PublicKey,

explorer/types.go

@@ -221,18 +221,20 @@ type Metrics struct {
 
 // HostScan represents the results of a host scan.
 type HostScan struct {
-	PublicKey types.PublicKey `json:"publicKey"`
-	Success   bool            `json:"success"`
-	Timestamp time.Time       `json:"timestamp"`
+	PublicKey   types.PublicKey `json:"publicKey"`
+	CountryCode string          `json:"countryCode"`
+	Success     bool            `json:"success"`
+	Timestamp   time.Time       `json:"timestamp"`
 
 	Settings   rhpv2.HostSettings   `json:"settings"`
 	PriceTable rhpv3.HostPriceTable `json:"priceTable"`
 }
 
 // Host represents a host and the information gathered from scanning it.
 type Host struct {
-	PublicKey  types.PublicKey `json:"publicKey"`
-	NetAddress string          `json:"netAddress"`
+	PublicKey   types.PublicKey `json:"publicKey"`
+	NetAddress  string          `json:"netAddress"`
+	CountryCode string          `json:"countryCode"`
 
 	KnownSince             time.Time `json:"knownSince"`
 	LastScan               time.Time `json:"lastScan"`

go.mod

@@ -16,6 +16,7 @@ require (
 )
 
 require (
+	github.com/ip2location/ip2location-go v8.3.0+incompatible // indirect
 	github.com/julienschmidt/httprouter v1.3.0 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/rogpeppe/go-internal v1.10.0 // indirect

go.sum

@@ -1,6 +1,8 @@
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/ip2location/ip2location-go v8.3.0+incompatible h1:QwUE+FlSbo6bjOWZpv2Grb57vJhWYFNPyBj2KCvfWaM=
+github.com/ip2location/ip2location-go v8.3.0+incompatible/go.mod h1:3JUY1TBjTx1GdA7oRT7Zeqfc0bg3lMMuU5lXmzdpuME=
 github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=

internal/geoip/geoip.go

@@ -0,0 +1,81 @@
+package geoip
+
+import (
+	_ "embed" // needed for geolocation database
+	"errors"
+	"net"
+	"os"
+	"sync"
+
+	"github.com/ip2location/ip2location-go"
+)
+
+//go:embed IP2LOCATION-LITE-DB1.BIN
+var ip2LocationDB []byte
+
+// A Locator maps IP addresses to their location.
+// It is assumed that it implementations are thread-safe.
+type Locator interface {
+	// Close closes the Locator.
+	Close() error
+	// CountryCode maps IP addresses to ISO 3166-1 A-2 country codes.
+	CountryCode(ip *net.IPAddr) (string, error)
+}
+
+type ip2Location struct {
+	mu sync.Mutex
+
+	path string
+	db   *ip2location.DB
+}
+
+// Close implements Locator.
+func (ip *ip2Location) Close() error {
+	ip.mu.Lock()
+	defer ip.mu.Unlock()
+
+	ip.db.Close()
+	return os.Remove(ip.path)
+}
+
+// CountryCode implements Locator.
+func (ip *ip2Location) CountryCode(addr *net.IPAddr) (string, error) {
+	if ip == nil {
+		return "", errors.New("nil IP")
+	}
+	ip.mu.Lock()
+	defer ip.mu.Unlock()
+
+	loc, err := ip.db.Get_country_short(addr.String())
+	if err != nil {
+		return "", err
+	}
+	return loc.Country_short, nil
+}
+
+// NewIP2LocationLocator returns a Locator that uses an underlying IP2Location
+// database.  If no path is provided, a default embedded LITE database is used.
+func NewIP2LocationLocator(path string) (Locator, error) {
+	// Unfortunately, ip2location.OpenDB only accepts a file path.  So we need
+	// to write the embedded file to a temporary file on disk, and use that
+	// instead.
+	if path == "" {
+		f, err := os.CreateTemp("", "geoip")
+		if err != nil {
+			return nil, err
+		} else if _, err := f.Write(ip2LocationDB); err != nil {
+			return nil, err
+		} else if err := f.Sync(); err != nil {
+			return nil, err
+		} else if err := f.Close(); err != nil {
+			return nil, err
+		}
+		path = f.Name()
+	}
+
+	db, err := ip2location.OpenDB(path)
+	if err != nil {
+		return nil, err
+	}
+	return &ip2Location{path: path, db: db}, nil
+}

persist/sqlite/addresses.go

@@ -100,14 +100,14 @@ WHERE ev.event_id = ?`, eventID).Scan(decode(&m.SiacoinOutput.StateElement.ID),
 }
 
 // Hosts returns the hosts with the given public keys.
-func (s *Store) Hosts(pks []types.PublicKey) (result []explorer.Host, err error) {
-	err = s.transaction(func(tx *txn) error {
+func (st *Store) Hosts(pks []types.PublicKey) (result []explorer.Host, err error) {
+	err = st.transaction(func(tx *txn) error {
 		var encoded []any
 		for _, pk := range pks {
 			encoded = append(encoded, encode(pk))
 		}
 
-		rows, err := tx.Query(`SELECT public_key,net_address,known_since,last_scan,last_scan_successful,last_announcement,total_scans,successful_interactions,failed_interactions,settings_accepting_contracts,settings_max_download_batch_size,settings_max_duration,settings_max_revise_batch_size,settings_net_address,settings_remaining_storage,settings_sector_size,settings_total_storage,settings_address,settings_window_size,settings_collateral,settings_max_collateral,settings_base_rpc_price,settings_contract_price,settings_download_bandwidth_price,settings_sector_access_price,settings_storage_price,settings_upload_bandwidth_price,settings_ephemeral_account_expiry,settings_max_ephemeral_account_balance,settings_revision_number,settings_version,settings_release,settings_sia_mux_port,price_table_uid,price_table_validity,price_table_host_block_height,price_table_update_price_table_cost,price_table_account_balance_cost,price_table_fund_account_cost,price_table_latest_revision_cost,price_table_subscription_memory_cost,price_table_subscription_notification_cost,price_table_init_base_cost,price_table_memory_time_cost,price_table_download_bandwidth_cost,price_table_upload_bandwidth_cost,price_table_drop_sectors_base_cost,price_table_drop_sectors_unit_cost,price_table_has_sector_base_cost,price_table_read_base_cost,price_table_read_length_cost,price_table_renew_contract_cost,price_table_revision_base_cost,price_table_swap_sector_base_cost,price_table_write_base_cost,price_table_write_length_cost,price_table_write_store_cost,price_table_txn_fee_min_recommended,price_table_txn_fee_max_recommended,price_table_contract_price,price_table_collateral_cost,price_table_max_collateral,price_table_max_duration,price_table_window_size,price_table_registry_entries_left,price_table_registry_entries_total FROM host_info WHERE public_key IN (`+queryPlaceHolders(len(pks))+`)`, encoded...)
+		rows, err := tx.Query(`SELECT public_key,net_address,country_code,known_since,last_scan,last_scan_successful,last_announcement,total_scans,successful_interactions,failed_interactions,settings_accepting_contracts,settings_max_download_batch_size,settings_max_duration,settings_max_revise_batch_size,settings_net_address,settings_remaining_storage,settings_sector_size,settings_total_storage,settings_address,settings_window_size,settings_collateral,settings_max_collateral,settings_base_rpc_price,settings_contract_price,settings_download_bandwidth_price,settings_sector_access_price,settings_storage_price,settings_upload_bandwidth_price,settings_ephemeral_account_expiry,settings_max_ephemeral_account_balance,settings_revision_number,settings_version,settings_release,settings_sia_mux_port,price_table_uid,price_table_validity,price_table_host_block_height,price_table_update_price_table_cost,price_table_account_balance_cost,price_table_fund_account_cost,price_table_latest_revision_cost,price_table_subscription_memory_cost,price_table_subscription_notification_cost,price_table_init_base_cost,price_table_memory_time_cost,price_table_download_bandwidth_cost,price_table_upload_bandwidth_cost,price_table_drop_sectors_base_cost,price_table_drop_sectors_unit_cost,price_table_has_sector_base_cost,price_table_read_base_cost,price_table_read_length_cost,price_table_renew_contract_cost,price_table_revision_base_cost,price_table_swap_sector_base_cost,price_table_write_base_cost,price_table_write_length_cost,price_table_write_store_cost,price_table_txn_fee_min_recommended,price_table_txn_fee_max_recommended,price_table_contract_price,price_table_collateral_cost,price_table_max_collateral,price_table_max_duration,price_table_window_size,price_table_registry_entries_left,price_table_registry_entries_total FROM host_info WHERE public_key IN (`+queryPlaceHolders(len(pks))+`)`, encoded...)
 		if err != nil {
 			return err
 		}
@@ -116,9 +116,10 @@ func (s *Store) Hosts(pks []types.PublicKey) (result []explorer.Host, err error)
 		for rows.Next() {
 			var host explorer.Host
 			s, p := &host.Settings, &host.PriceTable
-			if err := rows.Scan(decode(&host.PublicKey), &host.NetAddress, decode(&host.KnownSince), decode(&host.LastScan), &host.LastScanSuccessful, decode(&host.LastAnnouncement), &host.TotalScans, &host.SuccessfulInteractions, &host.FailedInteractions, &s.AcceptingContracts, decode(&s.MaxDownloadBatchSize), decode(&s.MaxDuration), decode(&s.MaxReviseBatchSize), &s.NetAddress, decode(&s.RemainingStorage), decode(&s.SectorSize), decode(&s.TotalStorage), decode(&s.Address), decode(&s.WindowSize), decode(&s.Collateral), decode(&s.MaxCollateral), decode(&s.BaseRPCPrice), decode(&s.ContractPrice), decode(&s.DownloadBandwidthPrice), decode(&s.SectorAccessPrice), decode(&s.StoragePrice), decode(&s.UploadBandwidthPrice), &s.EphemeralAccountExpiry, decode(&s.MaxEphemeralAccountBalance), decode(&s.RevisionNumber), &s.Version, &s.Release, &s.SiaMuxPort, decode(&p.UID), &p.Validity, decode(&p.HostBlockHeight), decode(&p.UpdatePriceTableCost), decode(&p.AccountBalanceCost), decode(&p.FundAccountCost), decode(&p.LatestRevisionCost), decode(&p.SubscriptionMemoryCost), decode(&p.SubscriptionNotificationCost), decode(&p.InitBaseCost), decode(&p.MemoryTimeCost), decode(&p.DownloadBandwidthCost), decode(&p.UploadBandwidthCost), decode(&p.DropSectorsBaseCost), decode(&p.DropSectorsUnitCost), decode(&p.HasSectorBaseCost), decode(&p.ReadBaseCost), decode(&p.ReadLengthCost), decode(&p.RenewContractCost), decode(&p.RevisionBaseCost), decode(&p.SwapSectorBaseCost), decode(&p.WriteBaseCost), decode(&p.WriteLengthCost), decode(&p.WriteStoreCost), decode(&p.TxnFeeMinRecommended), decode(&p.TxnFeeMaxRecommended), decode(&p.ContractPrice), decode(&p.CollateralCost), decode(&p.MaxCollateral), decode(&p.MaxDuration), decode(&p.WindowSize), decode(&p.RegistryEntriesLeft), decode(&p.RegistryEntriesTotal)); err != nil {
+			if err := rows.Scan(decode(&host.PublicKey), &host.NetAddress, &host.CountryCode, decode(&host.KnownSince), decode(&host.LastScan), &host.LastScanSuccessful, decode(&host.LastAnnouncement), &host.TotalScans, &host.SuccessfulInteractions, &host.FailedInteractions, &s.AcceptingContracts, decode(&s.MaxDownloadBatchSize), decode(&s.MaxDuration), decode(&s.MaxReviseBatchSize), &s.NetAddress, decode(&s.RemainingStorage), decode(&s.SectorSize), decode(&s.TotalStorage), decode(&s.Address), decode(&s.WindowSize), decode(&s.Collateral), decode(&s.MaxCollateral), decode(&s.BaseRPCPrice), decode(&s.ContractPrice), decode(&s.DownloadBandwidthPrice), decode(&s.SectorAccessPrice), decode(&s.StoragePrice), decode(&s.UploadBandwidthPrice), &s.EphemeralAccountExpiry, decode(&s.MaxEphemeralAccountBalance), decode(&s.RevisionNumber), &s.Version, &s.Release, &s.SiaMuxPort, decode(&p.UID), &p.Validity, decode(&p.HostBlockHeight), decode(&p.UpdatePriceTableCost), decode(&p.AccountBalanceCost), decode(&p.FundAccountCost), decode(&p.LatestRevisionCost), decode(&p.SubscriptionMemoryCost), decode(&p.SubscriptionNotificationCost), decode(&p.InitBaseCost), decode(&p.MemoryTimeCost), decode(&p.DownloadBandwidthCost), decode(&p.UploadBandwidthCost), decode(&p.DropSectorsBaseCost), decode(&p.DropSectorsUnitCost), decode(&p.HasSectorBaseCost), decode(&p.ReadBaseCost), decode(&p.ReadLengthCost), decode(&p.RenewContractCost), decode(&p.RevisionBaseCost), decode(&p.SwapSectorBaseCost), decode(&p.WriteBaseCost), decode(&p.WriteLengthCost), decode(&p.WriteStoreCost), decode(&p.TxnFeeMinRecommended), decode(&p.TxnFeeMaxRecommended), decode(&p.ContractPrice), decode(&p.CollateralCost), decode(&p.MaxCollateral), decode(&p.MaxDuration), decode(&p.WindowSize), decode(&p.RegistryEntriesLeft), decode(&p.RegistryEntriesTotal)); err != nil {
 				return err
 			}
+
 			result = append(result, host)
 		}
 		return nil