From 5630e66b48c6d88da971f66fca1b7c0b27f54330 Mon Sep 17 00:00:00 2001 From: Gleb Naumenko Date: Tue, 24 Dec 2019 13:18:44 -0500 Subject: [PATCH] Integrate ASN bucketing in Addrman and add tests Instead of using /16 netgroups to bucket nodes in Addrman for connection diversification, ASN, which better represents an actor in terms of network-layer infrastructure, is used. For testing, asmap.raw is used. It represents a minimal asmap needed for testing purposes. Cherry-picked from: ec45646 --- src/Makefile.test.include | 7 +- src/addrman.cpp | 196 ++++++++++++++++---- src/addrman.h | 275 ++++++++++++++++++---------- src/init.cpp | 26 +++ src/net.cpp | 6 +- src/net.h | 18 ++ src/netaddress.cpp | 44 ++++- src/netaddress.h | 5 +- src/test/addrman_tests.cpp | 356 ++++++++++++++++++++++++++++++++----- src/test/data/asmap.raw | Bin 0 -> 59 bytes src/test/netbase_tests.cpp | 31 ++-- 11 files changed, 756 insertions(+), 208 deletions(-) create mode 100644 src/test/data/asmap.raw diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 39aba1e3b46..981b0a41a7c 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -71,7 +71,8 @@ JSON_TEST_FILES = \ test/data/tx_valid.json \ test/data/sighash.json -RAW_TEST_FILES = +RAW_TEST_FILES = \ + test/data/asmap.raw GENERATED_TEST_FILES = $(JSON_TEST_FILES:.json=.json.h) $(RAW_TEST_FILES:.raw=.raw.h) @@ -225,8 +226,8 @@ endif %.raw.h: %.raw @$(MKDIR_P) $(@D) @{ \ - echo "static unsigned const char $(*F)[] = {" && \ - $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' && \ + echo "static unsigned const char $(*F)_raw[] = {" && \ + $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' && \ echo "};"; \ } > "$@.new" && mv -f "$@.new" "$@" @echo "Generated $@" diff --git a/src/addrman.cpp b/src/addrman.cpp index 33a623c1f2c..40fb3b891e4 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -1,25 +1,24 @@ // Copyright (c) 2012 Pieter Wuille -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2018 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "addrman.h" +#include -#include "hash.h" -#include "serialize.h" -#include "streams.h" +#include +#include -int CAddrInfo::GetTriedBucket(const uint256& nKey) const +int CAddrInfo::GetTriedBucket(const uint256& nKey, const std::vector &asmap) const { uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetKey()).GetHash().GetCheapHash(); - uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup() << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetHash().GetCheapHash(); + uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetHash().GetCheapHash(); return hash2 % ADDRMAN_TRIED_BUCKET_COUNT; } -int CAddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src) const +int CAddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const std::vector &asmap) const { - std::vector vchSourceGroupKey = src.GetGroup(); - uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup() << vchSourceGroupKey).GetHash().GetCheapHash(); + std::vector vchSourceGroupKey = src.GetGroup(asmap); + uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << vchSourceGroupKey).GetHash().GetCheapHash(); uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << vchSourceGroupKey << (hash1 % ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP)).GetHash().GetCheapHash(); return hash2 % ADDRMAN_NEW_BUCKET_COUNT; } @@ -69,13 +68,13 @@ CAddrInfo* CAddrMan::Find(const CNetAddr& addr, int* pnId) { std::map::iterator it = mapAddr.find(addr); if (it == mapAddr.end()) - return NULL; + return nullptr; if (pnId) *pnId = (*it).second; std::map::iterator it2 = mapInfo.find((*it).second); if (it2 != mapInfo.end()) return &(*it2).second; - return NULL; + return nullptr; } CAddrInfo* CAddrMan::Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId) @@ -154,7 +153,7 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) assert(info.nRefCount == 0); // which tried bucket to move the entry to - int nKBucket = info.GetTriedBucket(nKey); + int nKBucket = info.GetTriedBucket(nKey, m_asmap); int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); // first make space to add it (the existing tried entry there is moved to new, deleting whatever is there). @@ -170,7 +169,7 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) nTried--; // find which new bucket it belongs to - int nUBucket = infoOld.GetNewBucket(nKey); + int nUBucket = infoOld.GetNewBucket(nKey, m_asmap); int nUBucketPos = infoOld.GetBucketPosition(nKey, true, nUBucket); ClearNew(nUBucket, nUBucketPos); assert(vvNew[nUBucket][nUBucketPos] == -1); @@ -187,7 +186,7 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) info.fInTried = true; } -void CAddrMan::Good_(const CService& addr, int64_t nTime) +void CAddrMan::Good_(const CService& addr, bool test_before_evict, int64_t nTime) { int nId; @@ -217,7 +216,7 @@ void CAddrMan::Good_(const CService& addr, int64_t nTime) return; // find a bucket it is in now - int nRnd = RandomInt(ADDRMAN_NEW_BUCKET_COUNT); + int nRnd = insecure_rand.randrange(ADDRMAN_NEW_BUCKET_COUNT); int nUBucket = -1; for (unsigned int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; n++) { int nB = (n + nRnd) % ADDRMAN_NEW_BUCKET_COUNT; @@ -233,10 +232,24 @@ void CAddrMan::Good_(const CService& addr, int64_t nTime) if (nUBucket == -1) return; - LogPrint("addrman", "Moving %s to tried\n", addr.ToString()); + // which tried bucket to move the entry to + int tried_bucket = info.GetTriedBucket(nKey, m_asmap); + int tried_bucket_pos = info.GetBucketPosition(nKey, false, tried_bucket); + + // Will moving this address into tried evict another entry? + if (test_before_evict && (vvTried[tried_bucket][tried_bucket_pos] != -1)) { + // Output the entry we'd be colliding with, for debugging purposes + auto colliding_entry = mapInfo.find(vvTried[tried_bucket][tried_bucket_pos]); + LogPrint("addrman", "Collision inserting element into tried table (%s), moving %s to m_tried_collisions=%d\n", colliding_entry != mapInfo.end() ? colliding_entry->second.ToString() : "", addr.ToString(), m_tried_collisions.size()); + if (m_tried_collisions.size() < ADDRMAN_SET_TRIED_COLLISION_SIZE) { + m_tried_collisions.insert(nId); + } + } else { + LogPrint("addrman", "Moving %s to tried\n", addr.ToString()); - // move nId to the tried tables - MakeTried(info, nId); + // move nId to the tried tables + MakeTried(info, nId); + } } bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimePenalty) @@ -279,7 +292,7 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP int nFactor = 1; for (int n = 0; n < pinfo->nRefCount; n++) nFactor *= 2; - if (nFactor > 1 && (RandomInt(nFactor) != 0)) + if (nFactor > 1 && (insecure_rand.randrange(nFactor) != 0)) return false; } else { pinfo = Create(addr, source, &nId); @@ -288,7 +301,7 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP fNew = true; } - int nUBucket = pinfo->GetNewBucket(nKey, source); + int nUBucket = pinfo->GetNewBucket(nKey, source, m_asmap); int nUBucketPos = pinfo->GetBucketPosition(nKey, true, nUBucket); if (vvNew[nUBucket][nUBucketPos] != nId) { bool fInsert = vvNew[nUBucket][nUBucketPos] == -1; @@ -344,12 +357,12 @@ CAddrInfo CAddrMan::Select_(bool newOnly) // Use a 50% chance for choosing between tried and new table entries. if (!newOnly && - (nTried > 0 && (nNew == 0 || RandomInt(2) == 0))) { + (nTried > 0 && (nNew == 0 || insecure_rand.randbool() == 0))) { // use a tried node double fChanceFactor = 1.0; while (1) { - int nKBucket = RandomInt(ADDRMAN_TRIED_BUCKET_COUNT); - int nKBucketPos = RandomInt(ADDRMAN_BUCKET_SIZE); + int nKBucket = insecure_rand.randrange(ADDRMAN_TRIED_BUCKET_COUNT); + int nKBucketPos = insecure_rand.randrange(ADDRMAN_BUCKET_SIZE); while (vvTried[nKBucket][nKBucketPos] == -1) { nKBucket = (nKBucket + insecure_rand.randbits(ADDRMAN_TRIED_BUCKET_COUNT_LOG2)) % ADDRMAN_TRIED_BUCKET_COUNT; nKBucketPos = (nKBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; @@ -357,7 +370,7 @@ CAddrInfo CAddrMan::Select_(bool newOnly) int nId = vvTried[nKBucket][nKBucketPos]; assert(mapInfo.count(nId) == 1); CAddrInfo& info = mapInfo[nId]; - if (RandomInt(1 << 30) < fChanceFactor * info.GetChance() * (1 << 30)) + if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) return info; fChanceFactor *= 1.2; } @@ -365,8 +378,8 @@ CAddrInfo CAddrMan::Select_(bool newOnly) // use a new node double fChanceFactor = 1.0; while (1) { - int nUBucket = RandomInt(ADDRMAN_NEW_BUCKET_COUNT); - int nUBucketPos = RandomInt(ADDRMAN_BUCKET_SIZE); + int nUBucket = insecure_rand.randrange(ADDRMAN_NEW_BUCKET_COUNT); + int nUBucketPos = insecure_rand.randrange(ADDRMAN_BUCKET_SIZE); while (vvNew[nUBucket][nUBucketPos] == -1) { nUBucket = (nUBucket + insecure_rand.randbits(ADDRMAN_NEW_BUCKET_COUNT_LOG2)) % ADDRMAN_NEW_BUCKET_COUNT; nUBucketPos = (nUBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; @@ -374,7 +387,7 @@ CAddrInfo CAddrMan::Select_(bool newOnly) int nId = vvNew[nUBucket][nUBucketPos]; assert(mapInfo.count(nId) == 1); CAddrInfo& info = mapInfo[nId]; - if (RandomInt(1 << 30) < fChanceFactor * info.GetChance() * (1 << 30)) + if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) return info; fChanceFactor *= 1.2; } @@ -387,12 +400,12 @@ int CAddrMan::Check_() std::set setTried; std::map mapNew; - if (vRandom.size() != nTried + nNew) + if (vRandom.size() != (size_t)(nTried + nNew)) return -7; - for (std::map::iterator it = mapInfo.begin(); it != mapInfo.end(); it++) { - int n = (*it).first; - CAddrInfo& info = (*it).second; + for (const auto& entry : mapInfo) { + int n = entry.first; + const CAddrInfo& info = entry.second; if (info.fInTried) { if (!info.nLastSuccess) return -1; @@ -408,7 +421,7 @@ int CAddrMan::Check_() } if (mapAddr[info] != n) return -5; - if (info.nRandomPos < 0 || info.nRandomPos >= vRandom.size() || vRandom[info.nRandomPos] != n) + if (info.nRandomPos < 0 || (size_t)info.nRandomPos >= vRandom.size() || vRandom[info.nRandomPos] != n) return -14; if (info.nLastTry < 0) return -6; @@ -416,9 +429,9 @@ int CAddrMan::Check_() return -8; } - if (setTried.size() != nTried) + if (setTried.size() != (size_t)nTried) return -9; - if (mapNew.size() != nNew) + if (mapNew.size() != (size_t)nNew) return -10; for (int n = 0; n < ADDRMAN_TRIED_BUCKET_COUNT; n++) { @@ -426,7 +439,7 @@ int CAddrMan::Check_() if (vvTried[n][i] != -1) { if (!setTried.count(vvTried[n][i])) return -11; - if (mapInfo[vvTried[n][i]].GetTriedBucket(nKey) != n) + if (mapInfo[vvTried[n][i]].GetTriedBucket(nKey, m_asmap) != n) return -17; if (mapInfo[vvTried[n][i]].GetBucketPosition(nKey, false, n) != i) return -18; @@ -470,7 +483,7 @@ void CAddrMan::GetAddr_(std::vector& vAddr) if (vAddr.size() >= nNodes) break; - int nRndPos = RandomInt(vRandom.size() - n) + n; + int nRndPos = insecure_rand.randrange(vRandom.size() - n) + n; SwapRandom(n, nRndPos); assert(mapInfo.count(vRandom[n]) == 1); @@ -518,6 +531,111 @@ void CAddrMan::SetServices_(const CService& addr, ServiceFlags nServices) info.nServices = nServices; } -int CAddrMan::RandomInt(int nMax){ - return GetRandInt(nMax); +void CAddrMan::ResolveCollisions_() +{ + for (std::set::iterator it = m_tried_collisions.begin(); it != m_tried_collisions.end();) { + int id_new = *it; + + bool erase_collision = false; + + // If id_new not found in mapInfo remove it from m_tried_collisions + if (mapInfo.count(id_new) != 1) { + erase_collision = true; + } else { + CAddrInfo& info_new = mapInfo[id_new]; + + // Which tried bucket to move the entry to. + int tried_bucket = info_new.GetTriedBucket(nKey, m_asmap); + int tried_bucket_pos = info_new.GetBucketPosition(nKey, false, tried_bucket); + if (!info_new.IsValid()) { // id_new may no longer map to a valid address + erase_collision = true; + } else if (vvTried[tried_bucket][tried_bucket_pos] != -1) { // The position in the tried bucket is not empty + + // Get the to-be-evicted address that is being tested + int id_old = vvTried[tried_bucket][tried_bucket_pos]; + CAddrInfo& info_old = mapInfo[id_old]; + + // Has successfully connected in last X hours + if (GetAdjustedTime() - info_old.nLastSuccess < ADDRMAN_REPLACEMENT_HOURS*(60*60)) { + erase_collision = true; + } else if (GetAdjustedTime() - info_old.nLastTry < ADDRMAN_REPLACEMENT_HOURS*(60*60)) { // attempted to connect and failed in last X hours + + // Give address at least 60 seconds to successfully connect + if (GetAdjustedTime() - info_old.nLastTry > 60) { + LogPrint("addrman", "Replacing %s with %s in tried table\n", info_old.ToString(), info_new.ToString()); + + // Replaces an existing address already in the tried table with the new address + Good_(info_new, false, GetAdjustedTime()); + erase_collision = true; + } + } else if (GetAdjustedTime() - info_new.nLastSuccess > ADDRMAN_TEST_WINDOW) { + // If the collision hasn't resolved in some reasonable amount of time, + // just evict the old entry -- we must not be able to + // connect to it for some reason. + LogPrint("addrman", "Unable to test; replacing %s with %s in tried table anyway\n", info_old.ToString(), info_new.ToString()); + Good_(info_new, false, GetAdjustedTime()); + erase_collision = true; + } + } else { // Collision is not actually a collision anymore + Good_(info_new, false, GetAdjustedTime()); + erase_collision = true; + } + } + + if (erase_collision) { + m_tried_collisions.erase(it++); + } else { + it++; + } + } +} + +CAddrInfo CAddrMan::SelectTriedCollision_() +{ + if (m_tried_collisions.size() == 0) return CAddrInfo(); + + std::set::iterator it = m_tried_collisions.begin(); + + // Selects a random element from m_tried_collisions + std::advance(it, insecure_rand.randrange(m_tried_collisions.size())); + int id_new = *it; + + // If id_new not found in mapInfo remove it from m_tried_collisions + if (mapInfo.count(id_new) != 1) { + m_tried_collisions.erase(it); + return CAddrInfo(); + } + + CAddrInfo& newInfo = mapInfo[id_new]; + + // which tried bucket to move the entry to + int tried_bucket = newInfo.GetTriedBucket(nKey, m_asmap); + int tried_bucket_pos = newInfo.GetBucketPosition(nKey, false, tried_bucket); + + int id_old = vvTried[tried_bucket][tried_bucket_pos]; + + return mapInfo[id_old]; +} + +std::vector CAddrMan::DecodeAsmap(fs::path path) +{ + std::vector bits; + FILE *filestr = fsbridge::fopen(path, "rb"); + CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + if (file.IsNull()) { + LogPrintf("Failed to open asmap file from disk.\n"); + return bits; + } + fseek(filestr, 0, SEEK_END); + int length = ftell(filestr); + LogPrintf("Opened asmap file %s (%d bytes) from disk.\n", path, length); + fseek(filestr, 0, SEEK_SET); + char cur_byte; + for (int i = 0; i < length; ++i) { + file >> cur_byte; + for (int bit = 0; bit < 8; ++bit) { + bits.push_back((cur_byte >> bit) & 1); + } + } + return bits; } diff --git a/src/addrman.h b/src/addrman.h index 6e5f946bf28..81e0410f6b9 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -1,55 +1,59 @@ // Copyright (c) 2012 Pieter Wuille -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2018 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_ADDRMAN_H #define BITCOIN_ADDRMAN_H -#include "netaddress.h" -#include "protocol.h" -#include "random.h" -#include "sync.h" -#include "timedata.h" -#include "util.h" +#include +#include +#include +#include +#include +#include +#include #include #include #include #include +#include +#include +#include +#include + /** * Extended statistics about a CAddress */ class CAddrInfo : public CAddress { - - public: //! last try whatsoever by us (memory only) - int64_t nLastTry; + int64_t nLastTry{0}; //! last counted attempt (memory only) - int64_t nLastCountAttempt; + int64_t nLastCountAttempt{0}; private: //! where knowledge about this address first came from CNetAddr source; //! last successful connection by us - int64_t nLastSuccess; + int64_t nLastSuccess{0}; //! connection attempts since last successful attempt - int nAttempts; + int nAttempts{0}; //! reference count in new sets (memory only) - int nRefCount; + int nRefCount{0}; //! in tried set? (memory only) - bool fInTried; + bool fInTried{false}; //! position in vRandom - int nRandomPos; + int nRandomPos{-1}; friend class CAddrMan; @@ -65,37 +69,24 @@ class CAddrInfo : public CAddress READWRITE(nAttempts); } - void Init() - { - nLastSuccess = 0; - nLastTry = 0; - nLastCountAttempt = 0; - nAttempts = 0; - nRefCount = 0; - fInTried = false; - nRandomPos = -1; - } - CAddrInfo(const CAddress &addrIn, const CNetAddr &addrSource) : CAddress(addrIn), source(addrSource) { - Init(); } CAddrInfo() : CAddress(), source() { - Init(); } //! Calculate in which "tried" bucket this entry belongs - int GetTriedBucket(const uint256 &nKey) const; + int GetTriedBucket(const uint256 &nKey, const std::vector &asmap) const; //! Calculate in which "new" bucket this entry belongs, given a certain source - int GetNewBucket(const uint256 &nKey, const CNetAddr& src) const; + int GetNewBucket(const uint256 &nKey, const CNetAddr& src, const std::vector &asmap) const; //! Calculate in which "new" bucket this entry belongs, using its default source - int GetNewBucket(const uint256 &nKey) const + int GetNewBucket(const uint256 &nKey, const std::vector &asmap) const { - return GetNewBucket(nKey, source); + return GetNewBucket(nKey, source, asmap); } //! Calculate in which position of a bucket to store this entry. @@ -106,7 +97,6 @@ class CAddrInfo : public CAddress //! Calculate the relative chance this entry should be given when selecting nodes to connect to double GetChance(int64_t nNow = GetAdjustedTime()) const; - }; /** Stochastic address manager @@ -136,13 +126,13 @@ class CAddrInfo : public CAddress */ //! total number of buckets for tried addresses -#define ADDRMAN_TRIED_BUCKET_COUNT 256 +#define ADDRMAN_TRIED_BUCKET_COUNT_LOG2 8 //! total number of buckets for new addresses -#define ADDRMAN_NEW_BUCKET_COUNT 1024 +#define ADDRMAN_NEW_BUCKET_COUNT_LOG2 10 //! maximum allowed number of entries in buckets for new and tried addresses -#define ADDRMAN_BUCKET_SIZE 64 +#define ADDRMAN_BUCKET_SIZE_LOG2 6 //! over how many buckets entries with tried addresses from a single group (/16 for IPv4) are spread #define ADDRMAN_TRIED_BUCKETS_PER_GROUP 8 @@ -165,47 +155,66 @@ class CAddrInfo : public CAddress //! ... in at least this many days #define ADDRMAN_MIN_FAIL_DAYS 7 +//! how recent a successful connection should be before we allow an address to be evicted from tried +#define ADDRMAN_REPLACEMENT_HOURS 4 + //! the maximum percentage of nodes to return in a getaddr call #define ADDRMAN_GETADDR_MAX_PCT 23 //! the maximum number of nodes to return in a getaddr call #define ADDRMAN_GETADDR_MAX 2500 -/** - * Stochastical (IP) address manager +//! Convenience +#define ADDRMAN_TRIED_BUCKET_COUNT (1 << ADDRMAN_TRIED_BUCKET_COUNT_LOG2) +#define ADDRMAN_NEW_BUCKET_COUNT (1 << ADDRMAN_NEW_BUCKET_COUNT_LOG2) +#define ADDRMAN_BUCKET_SIZE (1 << ADDRMAN_BUCKET_SIZE_LOG2) + +//! the maximum number of tried addr collisions to store +#define ADDRMAN_SET_TRIED_COLLISION_SIZE 10 + +//! the maximum time we'll spend trying to resolve a tried table collision, in seconds +static const int64_t ADDRMAN_TEST_WINDOW = 40*60; // 40 minutes + +/** + * Stochastical (IP) address manager */ class CAddrMan { -private: +friend class CAddrManTest; +protected: //! critical section to protect the inner data structures mutable CCriticalSection cs; +private: //! last used nId - int nIdCount; + int nIdCount GUARDED_BY(cs); //! table with information about all nIds - std::map mapInfo; + std::map mapInfo GUARDED_BY(cs); //! find an nId based on its network address - std::map mapAddr; + std::map mapAddr GUARDED_BY(cs); //! randomly-ordered vector of all nIds - std::vector vRandom; + std::vector vRandom GUARDED_BY(cs); // number of "tried" entries - int nTried; + int nTried GUARDED_BY(cs); //! list of "tried" buckets - int vvTried[ADDRMAN_TRIED_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE]; + int vvTried[ADDRMAN_TRIED_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); //! number of (unique) "new" entries - int nNew; + int nNew GUARDED_BY(cs); //! list of "new" buckets - int vvNew[ADDRMAN_NEW_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE]; + int vvNew[ADDRMAN_NEW_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); //! last time Good was called (memory only) - int64_t nLastGood; + int64_t nLastGood GUARDED_BY(cs); + + //! Holds addrs inserted into tried table that collide with existing entries. Test-before-evict discipline used to resolve these collisions. + std::set m_tried_collisions; protected: //! secret key to randomize bucket select with @@ -215,57 +224,80 @@ class CAddrMan FastRandomContext insecure_rand; //! Find an entry. - CAddrInfo* Find(const CNetAddr& addr, int *pnId = NULL); + CAddrInfo* Find(const CNetAddr& addr, int *pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); //! find an entry, creating it if necessary. //! nTime and nServices of the found node are updated, if necessary. - CAddrInfo* Create(const CAddress &addr, const CNetAddr &addrSource, int *pnId = NULL); + CAddrInfo* Create(const CAddress &addr, const CNetAddr &addrSource, int *pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); //! Swap two elements in vRandom. - void SwapRandom(unsigned int nRandomPos1, unsigned int nRandomPos2); + void SwapRandom(unsigned int nRandomPos1, unsigned int nRandomPos2) EXCLUSIVE_LOCKS_REQUIRED(cs); //! Move an entry from the "new" table(s) to the "tried" table - void MakeTried(CAddrInfo& info, int nId); + void MakeTried(CAddrInfo& info, int nId) EXCLUSIVE_LOCKS_REQUIRED(cs); //! Delete an entry. It must not be in tried, and have refcount 0. - void Delete(int nId); + void Delete(int nId) EXCLUSIVE_LOCKS_REQUIRED(cs); //! Clear a position in a "new" table. This is the only place where entries are actually deleted. - void ClearNew(int nUBucket, int nUBucketPos); + void ClearNew(int nUBucket, int nUBucketPos) EXCLUSIVE_LOCKS_REQUIRED(cs); //! Mark an entry "good", possibly moving it from "new" to "tried". - void Good_(const CService &addr, int64_t nTime); + void Good_(const CService &addr, bool test_before_evict, int64_t time) EXCLUSIVE_LOCKS_REQUIRED(cs); //! Add an entry to the "new" table. - bool Add_(const CAddress &addr, const CNetAddr& source, int64_t nTimePenalty); + bool Add_(const CAddress &addr, const CNetAddr& source, int64_t nTimePenalty) EXCLUSIVE_LOCKS_REQUIRED(cs); //! Mark an entry as attempted to connect. - void Attempt_(const CService &addr, bool fCountFailure, int64_t nTime); + void Attempt_(const CService &addr, bool fCountFailure, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); //! Select an address to connect to, if newOnly is set to true, only the new table is selected from. - CAddrInfo Select_(bool newOnly); + CAddrInfo Select_(bool newOnly) EXCLUSIVE_LOCKS_REQUIRED(cs); - //! Wraps GetRandInt to allow tests to override RandomInt and make it determinismistic. - virtual int RandomInt(int nMax); + //! See if any to-be-evicted tried table entries have been tested and if so resolve the collisions. + void ResolveCollisions_() EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Return a random to-be-evicted tried table address. + CAddrInfo SelectTriedCollision_() EXCLUSIVE_LOCKS_REQUIRED(cs); #ifdef DEBUG_ADDRMAN //! Perform consistency check. Returns an error code or zero. - int Check_(); + int Check_() EXCLUSIVE_LOCKS_REQUIRED(cs); #endif //! Select several addresses at once. - void GetAddr_(std::vector &vAddr); + void GetAddr_(std::vector &vAddr) EXCLUSIVE_LOCKS_REQUIRED(cs); //! Mark an entry as currently-connected-to. - void Connected_(const CService &addr, int64_t nTime); + void Connected_(const CService &addr, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); //! Update an entry's service bits. - void SetServices_(const CService &addr, ServiceFlags nServices); + void SetServices_(const CService &addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs); public: + // Compressed IP->ASN mapping, loaded from a file when a node starts. + // Should be always empty if no file was provided. + // This mapping is then used for bucketing nodes in Addrman. + // + // If asmap is provided, nodes will be bucketed by + // AS they belong to, in order to make impossible for a node + // to connect to several nodes hosted in a single AS. + // This is done in response to Erebus attack, but also to generally + // diversify the connections every node creates, + // especially useful when a large fraction of nodes + // operate under a couple of cloud providers. + // + // If a new asmap was provided, the existing records + // would be re-bucketed accordingly. + std::vector m_asmap; + + // Read asmap from provided binary file + static std::vector DecodeAsmap(fs::path path); + + /** * serialized format: - * * version byte (currently 1) + * * version byte (1 for pre-asmap files, 2 for files including asmap version) * * 0x20 + nKey (serialized as if it were a vector, for backward compatibility) * * nNew * * nTried @@ -297,7 +329,7 @@ class CAddrMan { LOCK(cs); - unsigned char nVersion = 1; + unsigned char nVersion = 2; s << nVersion; s << ((unsigned char)32); s << nKey; @@ -308,9 +340,9 @@ class CAddrMan s << nUBuckets; std::map mapUnkIds; int nIds = 0; - for (std::map::const_iterator it = mapInfo.begin(); it != mapInfo.end(); it++) { - mapUnkIds[(*it).first] = nIds; - const CAddrInfo &info = (*it).second; + for (const auto& entry : mapInfo) { + mapUnkIds[entry.first] = nIds; + const CAddrInfo &info = entry.second; if (info.nRefCount) { assert(nIds != nNew); // this means nNew was wrong, oh ow s << info; @@ -318,8 +350,8 @@ class CAddrMan } } nIds = 0; - for (std::map::const_iterator it = mapInfo.begin(); it != mapInfo.end(); it++) { - const CAddrInfo &info = (*it).second; + for (const auto& entry : mapInfo) { + const CAddrInfo &info = entry.second; if (info.fInTried) { assert(nIds != nTried); // this means nTried was wrong, oh ow s << info; @@ -340,6 +372,13 @@ class CAddrMan } } } + // Store asmap version after bucket entries so that it + // can be ignored by older clients for backward compatibility. + uint256 asmap_version; + if (m_asmap.size() != 0) { + asmap_version = SerializeHash(m_asmap); + } + s << asmap_version; } template @@ -348,7 +387,6 @@ class CAddrMan LOCK(cs); Clear(); - unsigned char nVersion; s >> nVersion; unsigned char nKeySize; @@ -378,16 +416,6 @@ class CAddrMan mapAddr[info] = n; info.nRandomPos = vRandom.size(); vRandom.push_back(n); - if (nVersion != 1 || nUBuckets != ADDRMAN_NEW_BUCKET_COUNT) { - // In case the new table data cannot be used (nVersion unknown, or bucket count wrong), - // immediately try to give them a reference based on their primary source address. - int nUBucket = info.GetNewBucket(nKey); - int nUBucketPos = info.GetBucketPosition(nKey, true, nUBucket); - if (vvNew[nUBucket][nUBucketPos] == -1) { - vvNew[nUBucket][nUBucketPos] = n; - info.nRefCount++; - } - } } nIdCount = nNew; @@ -396,7 +424,7 @@ class CAddrMan for (int n = 0; n < nTried; n++) { CAddrInfo info; s >> info; - int nKBucket = info.GetTriedBucket(nKey); + int nKBucket = info.GetTriedBucket(nKey, m_asmap); int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); if (vvTried[nKBucket][nKBucketPos] == -1) { info.nRandomPos = vRandom.size(); @@ -412,7 +440,9 @@ class CAddrMan } nTried -= nLost; - // Deserialize positions in the new table (if possible). + // Store positions in the new table buckets to apply later (if possible). + std::map entryToBucket; // Represents which entry belonged to which bucket when serializing + for (int bucket = 0; bucket < nUBuckets; bucket++) { int nSize = 0; s >> nSize; @@ -420,12 +450,38 @@ class CAddrMan int nIndex = 0; s >> nIndex; if (nIndex >= 0 && nIndex < nNew) { - CAddrInfo &info = mapInfo[nIndex]; - int nUBucketPos = info.GetBucketPosition(nKey, true, bucket); - if (nVersion == 1 && nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && vvNew[bucket][nUBucketPos] == -1 && info.nRefCount < ADDRMAN_NEW_BUCKETS_PER_ADDRESS) { - info.nRefCount++; - vvNew[bucket][nUBucketPos] = nIndex; - } + entryToBucket[nIndex] = bucket; + } + } + } + + uint256 supplied_asmap_version; + if (m_asmap.size() != 0) { + supplied_asmap_version = SerializeHash(m_asmap); + } + uint256 serialized_asmap_version; + if (nVersion > 1) { + s >> serialized_asmap_version; + } + + for (int n = 0; n < nNew; n++) { + CAddrInfo &info = mapInfo[n]; + int bucket = entryToBucket[n]; + int nUBucketPos = info.GetBucketPosition(nKey, true, bucket); + if (nVersion == 2 && nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && vvNew[bucket][nUBucketPos] == -1 && + info.nRefCount < ADDRMAN_NEW_BUCKETS_PER_ADDRESS && serialized_asmap_version == supplied_asmap_version) { + // Bucketing has not changed, using existing bucket positions for the new table + vvNew[bucket][nUBucketPos] = n; + info.nRefCount++; + } else { + // In case the new table data cannot be used (nVersion unknown, bucket count wrong or new asmap), + // try to give them a reference based on their primary source address. + LogPrint("addrman", "Bucketing method was updated, re-bucketing addrman entries from disk\n"); + bucket = info.GetNewBucket(nKey, m_asmap); + nUBucketPos = info.GetBucketPosition(nKey, true, bucket); + if (vvNew[bucket][nUBucketPos] == -1) { + vvNew[bucket][nUBucketPos] = n; + info.nRefCount++; } } } @@ -450,8 +506,9 @@ class CAddrMan void Clear() { + LOCK(cs); std::vector().swap(vRandom); - nKey = GetRandHash(); + nKey = insecure_rand.rand256(); for (size_t bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; bucket++) { for (size_t entry = 0; entry < ADDRMAN_BUCKET_SIZE; entry++) { vvNew[bucket][entry] = -1; @@ -467,6 +524,8 @@ class CAddrMan nTried = 0; nNew = 0; nLastGood = 1; //Initially at 1 so that "never" is strictly worse. + mapInfo.clear(); + mapAddr.clear(); } CAddrMan() @@ -507,8 +566,9 @@ class CAddrMan Check(); fRet |= Add_(addr, source, nTimePenalty); Check(); - if (fRet) + if (fRet) { LogPrint("addrman", "Added %s from %s: %i tried, %i new\n", addr.ToStringIPPort(), source.ToString(), nTried, nNew); + } return fRet; } @@ -521,17 +581,18 @@ class CAddrMan for (std::vector::const_iterator it = vAddr.begin(); it != vAddr.end(); it++) nAdd += Add_(*it, source, nTimePenalty) ? 1 : 0; Check(); - if (nAdd) + if (nAdd) { LogPrint("addrman", "Added %i addresses from %s: %i tried, %i new\n", nAdd, source.ToString(), nTried, nNew); + } return nAdd > 0; } //! Mark an entry as accessible. - void Good(const CService &addr, int64_t nTime = GetAdjustedTime()) + void Good(const CService &addr, bool test_before_evict = true, int64_t nTime = GetAdjustedTime()) { LOCK(cs); Check(); - Good_(addr, nTime); + Good_(addr, test_before_evict, nTime); Check(); } @@ -544,6 +605,28 @@ class CAddrMan Check(); } + //! See if any to-be-evicted tried table entries have been tested and if so resolve the collisions. + void ResolveCollisions() + { + LOCK(cs); + Check(); + ResolveCollisions_(); + Check(); + } + + //! Randomly select an address in tried that another address is attempting to evict. + CAddrInfo SelectTriedCollision() + { + CAddrInfo ret; + { + LOCK(cs); + Check(); + ret = SelectTriedCollision_(); + Check(); + } + return ret; + } + /** * Choose an address to connect to. */ diff --git a/src/init.cpp b/src/init.cpp index 692270dda92..dc1f271952e 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -99,6 +99,7 @@ enum BindFlags { }; static const char* FEE_ESTIMATES_FILENAME="fee_estimates.dat"; +static const char* DEFAULT_ASMAP_FILENAME="ip_asn.map"; ////////////////////////////////////////////////////////////////////////////// // @@ -1404,6 +1405,31 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) connman.AddOneShot(strDest); } + // Read asmap file if configured + if (IsArgSet("-asmap")) { + fs::path asmap_file = fs::path(GetArg("-asmap", "")); + if (asmap_file.empty()) { + asmap_file = DEFAULT_ASMAP_FILENAME; + } + if (!asmap_file.is_absolute()) { + asmap_file = GetDataDir() / asmap_file; + } + if (!fs::exists(asmap_file)) { + InitError(strprintf(_("Could not find asmap file %s"), asmap_file)); + return false; + } + std::vector asmap = CAddrMan::DecodeAsmap(asmap_file); + if (asmap.size() == 0) { + InitError(strprintf(_("Could not find or parse specified asmap: '%s'"), asmap_file)); + return false; + } + const uint256 asmap_version = SerializeHash(asmap); + connman.SetAsmap(std::move(asmap)); + LogPrintf("Using asmap version %s for IP bucketing.\n", asmap_version.ToString()); + } else { + LogPrintf("Using /16 prefix for IP bucketing.\n"); + } + #if ENABLE_ZMQ pzmqNotificationInterface = CZMQNotificationInterface::Create(); diff --git a/src/net.cpp b/src/net.cpp index b2c1ee9b8c0..8f25074cc62 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1783,7 +1783,7 @@ void CConnman::ThreadOpenConnections() // but inbound and addnode peers do not use our outbound slots. Inbound peers // also have the added issue that they're attacker controlled and could be used // to prevent us from connecting to particular hosts if we used them here. - setConnected.insert(pnode->addr.GetGroup()); + setConnected.insert(pnode->addr.GetGroup(addrman.m_asmap)); nOutbound++; } } @@ -1819,7 +1819,7 @@ void CConnman::ThreadOpenConnections() CAddrInfo addr = addrman.Select(fFeeler); // if we selected an invalid address, restart - if (!addr.IsValid() || setConnected.count(addr.GetGroup()) || IsLocal(addr)) + if (!addr.IsValid() || setConnected.count(addr.GetGroup(addrman.m_asmap)) || IsLocal(addr)) break; // If we didn't find an appropriate destination after trying 100 addresses fetched from addrman, @@ -2844,7 +2844,7 @@ CSipHasher CConnman::GetDeterministicRandomizer(uint64_t id) const uint64_t CConnman::CalculateKeyedNetGroup(const CAddress& ad) const { - std::vector vchNetGroup(ad.GetGroup()); + std::vector vchNetGroup(ad.GetGroup(addrman.m_asmap)); return GetDeterministicRandomizer(RANDOMIZER_ID_NETGROUP).Write(&vchNetGroup[0], vchNetGroup.size()).Finalize(); } diff --git a/src/net.h b/src/net.h index 6cfbd9550c1..87ee05b2cdf 100644 --- a/src/net.h +++ b/src/net.h @@ -147,6 +147,15 @@ class CConnman unsigned int nReceiveFloodSize = 0; uint64_t nMaxOutboundTimeframe = 0; uint64_t nMaxOutboundLimit = 0; + int64_t m_peer_connect_timeout = 60; + std::vector vSeedNodes; + std::vector vWhitelistedRange; + std::vector vWhiteBinds; + std::vector vBinds; + bool m_use_addrman_outgoing = true; + std::vector m_specified_outgoing; + std::vector m_added_nodes; + std::vector m_asmap; }; CConnman(uint64_t seed0, uint64_t seed1); ~CConnman(); @@ -290,6 +299,15 @@ class CConnman void SetMaxConnections(int newMaxConnections); void WakeMessageHandler(); + + /** Attempts to obfuscate tx time through exponentially distributed emitting. + Works assuming that a single interval is used. + Variable intervals will result in privacy decrease. + */ + int64_t PoissonNextSendInbound(int64_t now, int average_interval_seconds); + + void SetAsmap(std::vector asmap) { addrman.m_asmap = asmap; } + private: struct ListenSocket { SOCKET socket; diff --git a/src/netaddress.cpp b/src/netaddress.cpp index d5f1e3df189..a920597e8f8 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -8,10 +8,11 @@ #include "config/bitcoin-config.h" #endif -#include "netaddress.h" -#include "hash.h" -#include "utilstrencodings.h" -#include "tinyformat.h" +#include +#include +#include +#include +#include static const unsigned char pchIPv4[12] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff }; static const unsigned char pchOnionCat[] = {0xFD,0x87,0xD8,0x7E,0xEB,0x43}; @@ -324,9 +325,17 @@ bool CNetAddr::GetIn6Addr(struct in6_addr* pipv6Addr) const return true; } -// get canonical identifier of an address' group -// no two connections will be attempted to addresses with the same group -std::vector CNetAddr::GetGroup() const +/** + * Get the canonical identifier of our network group + * + * The groups are assigned in a way where it should be costly for an attacker to + * obtain addresses with many different group identifiers, even if it is cheap + * to obtain addresses with the same identifier. + * + * @note No two connections will be attempted to addresses with the same network + * group. + */ +std::vector CNetAddr::GetGroup(const std::vector &asmap) const { std::vector vchRet; int nClass = NET_IPV6; @@ -386,6 +395,27 @@ std::vector CNetAddr::GetGroup() const else nBits = 32; + // If asmap is supplied and the address is IPv4/IPv6, + // ignore nBits and use 32/128 bits to obtain ASN from asmap. + // ASN is then returned to be used for bucketing. + if (asmap.size() != 0 && (nClass == NET_IPV4 || nClass == NET_IPV6)) { + nClass = NET_IPV6; + std::vector ip_bits(128); + for (int8_t byte_i = 0; byte_i < 16; ++byte_i) { + uint8_t cur_byte = GetByte(15 - byte_i); + for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) { + ip_bits[byte_i * 8 + bit_i] = (cur_byte >> (7 - bit_i)) & 1; + } + } + + uint32_t asn = Interpret(asmap, ip_bits); + vchRet.push_back(nClass); + for (int i = 0; i < 4; i++) { + vchRet.push_back((asn >> (8 * i)) & 0xFF); + } + return vchRet; + } + vchRet.push_back(nClass); while (nBits >= 8) { diff --git a/src/netaddress.h b/src/netaddress.h index c473482799f..18ad55862dd 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -81,8 +81,9 @@ class CNetAddr unsigned int GetByte(int n) const; uint64_t GetHash() const; bool GetInAddr(struct in_addr* pipv4Addr) const; - std::vector GetGroup() const; - int GetReachabilityFrom(const CNetAddr *paddrPartner = NULL) const; + std::vector GetGroup(const std::vector &asmap) const; + + int GetReachabilityFrom(const CNetAddr *paddrPartner = nullptr) const; CNetAddr(const struct in6_addr& pipv6Addr, const uint32_t scope = 0); bool GetIn6Addr(struct in6_addr* pipv6Addr) const; diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index 3c9d697ae40..5aa130123c2 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -5,6 +5,8 @@ #include "test/test_bitcoin.h" #include #include +#include +#include #include "hash.h" #include "netbase.h" @@ -12,12 +14,18 @@ class CAddrManTest : public CAddrMan { - uint64_t state; - +private: + bool deterministic; public: - CAddrManTest() + explicit CAddrManTest(bool makeDeterministic = true, + std::vector asmap = std::vector()) { - state = 1; + if (makeDeterministic) { + // Set addrman addr placement to be deterministic. + MakeDeterministic(); + } + deterministic = makeDeterministic; + m_asmap = asmap; } //! Ensure that bucket placement is always the same for testing purposes. @@ -27,12 +35,6 @@ class CAddrManTest : public CAddrMan insecure_rand = FastRandomContext(true); } - int RandomInt(int nMax) - { - state = (CHashWriter(SER_GETHASH, 0) << state).GetHash().GetCheapHash(); - return (unsigned int)(state % nMax); - } - CAddrInfo* Find(const CNetAddr& addr, int* pnId = NULL) { return CAddrMan::Find(addr, pnId); @@ -47,6 +49,43 @@ class CAddrManTest : public CAddrMan { CAddrMan::Delete(nId); } + + // Used to test deserialization + std::pair GetBucketAndEntry(const CAddress& addr) + { + LOCK(cs); + int nId = mapAddr[addr]; + for (int bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; ++bucket) { + for (int entry = 0; entry < ADDRMAN_BUCKET_SIZE; ++entry) { + if (nId == vvNew[bucket][entry]) { + return std::pair(bucket, entry); + } + } + } + return std::pair(-1, -1); + } + + // Simulates connection failure so that we can test eviction of offline nodes + void SimConnFail(CService& addr) + { + LOCK(cs); + int64_t nLastSuccess = 1; + Good_(addr, true, nLastSuccess); // Set last good connection in the deep past. + + bool count_failure = false; + int64_t nLastTry = GetAdjustedTime()-61; + Attempt(addr, count_failure, nLastTry); + } + + void Clear() + { + CAddrMan::Clear(); + if (deterministic) { + nKey.SetNull(); + insecure_rand = FastRandomContext(true); + } + } + }; static CNetAddr ResolveIP(const std::string& ip) @@ -64,6 +103,18 @@ static CService ResolveService(const std::string& ip, const int port = 0) } +static std::vector FromBytes(const unsigned char* source, int vector_size) { + std::vector result(vector_size); + for (int byte_i = 0; byte_i < vector_size / 8; ++byte_i) { + unsigned char cur_byte = source[byte_i]; + for (int bit_i = 0; bit_i < 8; ++bit_i) { + result[byte_i * 8 + bit_i] = (cur_byte >> bit_i) & 1; + } + } + return result; +} + + BOOST_FIXTURE_TEST_SUITE(addrman_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(addrman_simple) @@ -410,6 +461,139 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) } +BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) +{ + CAddrManTest addrman; + + CAddress addr1 = CAddress(ResolveService("250.1.1.1", 8333), NODE_NONE); + CAddress addr2 = CAddress(ResolveService("250.1.1.1", 9999), NODE_NONE); + + CNetAddr source1 = ResolveIP("250.1.1.1"); + + + CAddrInfo info1 = CAddrInfo(addr1, source1); + + uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); + uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); + + std::vector asmap; // use /16 + + BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1, asmap), 40); + + // Test: Make sure key actually randomizes bucket placement. A fail on + // this test could be a security issue. + BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info1.GetTriedBucket(nKey2, asmap)); + + // Test: Two addresses with same IP but different ports can map to + // different buckets because they have different keys. + CAddrInfo info2 = CAddrInfo(addr2, source1); + + BOOST_CHECK(info1.GetKey() != info2.GetKey()); + BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info2.GetTriedBucket(nKey1, asmap)); + + std::set buckets; + for (int i = 0; i < 255; i++) { + CAddrInfo infoi = CAddrInfo( + CAddress(ResolveService("250.1.1." + std::to_string(i)), NODE_NONE), + ResolveIP("250.1.1." + std::to_string(i))); + int bucket = infoi.GetTriedBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the same /16 prefix should + // never get more than 8 buckets with legacy grouping + BOOST_CHECK_EQUAL(buckets.size(), 8U); + + buckets.clear(); + for (int j = 0; j < 255; j++) { + CAddrInfo infoj = CAddrInfo( + CAddress(ResolveService("250." + std::to_string(j) + ".1.1"), NODE_NONE), + ResolveIP("250." + std::to_string(j) + ".1.1")); + int bucket = infoj.GetTriedBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the different /16 prefix should map to more than + // 8 buckets with legacy grouping + BOOST_CHECK_EQUAL(buckets.size(), 160U); +} + +BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) +{ + CAddrManTest addrman; + + CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); + CAddress addr2 = CAddress(ResolveService("250.1.2.1", 9999), NODE_NONE); + + CNetAddr source1 = ResolveIP("250.1.2.1"); + + CAddrInfo info1 = CAddrInfo(addr1, source1); + + uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); + uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); + + std::vector asmap; // use /16 + + // Test: Make sure the buckets are what we expect + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), 786); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1, asmap), 786); + + // Test: Make sure key actually randomizes bucket placement. A fail on + // this test could be a security issue. + BOOST_CHECK(info1.GetNewBucket(nKey1, asmap) != info1.GetNewBucket(nKey2, asmap)); + + // Test: Ports should not affect bucket placement in the addr + CAddrInfo info2 = CAddrInfo(addr2, source1); + BOOST_CHECK(info1.GetKey() != info2.GetKey()); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), info2.GetNewBucket(nKey1, asmap)); + + std::set buckets; + for (int i = 0; i < 255; i++) { + CAddrInfo infoi = CAddrInfo( + CAddress(ResolveService("250.1.1." + std::to_string(i)), NODE_NONE), + ResolveIP("250.1.1." + std::to_string(i))); + int bucket = infoi.GetNewBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the same group (\16 prefix for IPv4) should + // always map to the same bucket. + BOOST_CHECK_EQUAL(buckets.size(), 1U); + + buckets.clear(); + for (int j = 0; j < 4 * 255; j++) { + CAddrInfo infoj = CAddrInfo(CAddress( + ResolveService( + std::to_string(250 + (j / 255)) + "." + std::to_string(j % 256) + ".1.1"), NODE_NONE), + ResolveIP("251.4.1.1")); + int bucket = infoj.GetNewBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the same source groups should map to NO MORE + // than 64 buckets. + BOOST_CHECK(buckets.size() <= 64); + + buckets.clear(); + for (int p = 0; p < 255; p++) { + CAddrInfo infoj = CAddrInfo( + CAddress(ResolveService("250.1.1.1"), NODE_NONE), + ResolveIP("250." + std::to_string(p) + ".1.1")); + int bucket = infoj.GetNewBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the different source groups should map to MORE + // than 64 buckets. + BOOST_CHECK(buckets.size() > 64); +} + +// The following three test cases use asmap.raw +// We use an artificial minimal mock mapping +// 250.0.0.0/8 AS1000 +// 101.1.0.0/16 AS1 +// 101.2.0.0/16 AS2 +// 101.3.0.0/16 AS3 +// 101.4.0.0/16 AS4 +// 101.5.0.0/16 AS5 +// 101.6.0.0/16 AS6 +// 101.7.0.0/16 AS7 +// 101.8.0.0/16 AS8 BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) { CAddrManTest addrman; @@ -428,43 +612,44 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); + std::vector asmap1 = FromBytes(asmap, sizeof(asmap) * 8); - BOOST_CHECK(info1.GetTriedBucket(nKey1) == 40); + BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1, asmap1), 236); // Test 26: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. - BOOST_CHECK(info1.GetTriedBucket(nKey1) != info1.GetTriedBucket(nKey2)); + BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap1) != info1.GetTriedBucket(nKey2, asmap1)); // Test 27: Two addresses with same IP but different ports can map to // different buckets because they have different keys. CAddrInfo info2 = CAddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); - BOOST_CHECK(info1.GetTriedBucket(nKey1) != info2.GetTriedBucket(nKey1)); + BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap1) != info2.GetTriedBucket(nKey1, asmap1)); std::set buckets; - for (int i = 0; i < 255; i++) { - CAddrInfo infoi = CAddrInfo( - CAddress(ResolveService("250.1.1." + boost::to_string(i)), NODE_NONE), - ResolveIP("250.1.1." + boost::to_string(i))); - int bucket = infoi.GetTriedBucket(nKey1); + for (int j = 0; j < 255; j++) { + CAddrInfo infoj = CAddrInfo( + CAddress(ResolveService("101." + std::to_string(j) + ".1.1"), NODE_NONE), + ResolveIP("101." + std::to_string(j) + ".1.1")); + int bucket = infoj.GetTriedBucket(nKey1, asmap1); buckets.insert(bucket); } - // Test 28: IP addresses in the same group (\16 prefix for IPv4) should - // never get more than 8 buckets - BOOST_CHECK(buckets.size() == 8); + // Test: IP addresses in the different /16 prefix MAY map to more than + // 8 buckets. + BOOST_CHECK(buckets.size() > 8); buckets.clear(); for (int j = 0; j < 255; j++) { CAddrInfo infoj = CAddrInfo( - CAddress(ResolveService("250." + boost::to_string(j) + ".1.1"), NODE_NONE), - ResolveIP("250." + boost::to_string(j) + ".1.1")); - int bucket = infoj.GetTriedBucket(nKey1); + CAddress(ResolveService("250." + std::to_string(j) + ".1.1"), NODE_NONE), + ResolveIP("250." + std::to_string(j) + ".1.1")); + int bucket = infoj.GetTriedBucket(nKey1, asmap1); buckets.insert(bucket); } - // Test 29: IP addresses in the different groups should map to more than - // 8 buckets. - BOOST_CHECK(buckets.size() == 160); + // Test: IP addresses in the different /16 prefix MAY NOT map to more than + // 8 buckets. + BOOST_CHECK(buckets.size() == 8); } BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) @@ -484,28 +669,32 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); - BOOST_CHECK(info1.GetNewBucket(nKey1) == 786); + std::vector asmap1 = FromBytes(asmap, sizeof(asmap) * 8); + + // Test: Make sure the buckets are what we expect + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap1), 795); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1, asmap1), 795); // Test 30: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. - BOOST_CHECK(info1.GetNewBucket(nKey1) != info1.GetNewBucket(nKey2)); + BOOST_CHECK(info1.GetNewBucket(nKey1, asmap1) != info1.GetNewBucket(nKey2, asmap1)); // Test 31: Ports should not effect bucket placement in the addr CAddrInfo info2 = CAddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); - BOOST_CHECK(info1.GetNewBucket(nKey1) == info2.GetNewBucket(nKey1)); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap1), info2.GetNewBucket(nKey1, asmap1)); std::set buckets; for (int i = 0; i < 255; i++) { CAddrInfo infoi = CAddrInfo( - CAddress(ResolveService("250.1.1." + boost::to_string(i)), NODE_NONE), - ResolveIP("250.1.1." + boost::to_string(i))); - int bucket = infoi.GetNewBucket(nKey1); + CAddress(ResolveService("250.1.1." + std::to_string(i)), NODE_NONE), + ResolveIP("250.1.1." + std::to_string(i))); + int bucket = infoi.GetNewBucket(nKey1, asmap1); buckets.insert(bucket); } - // Test 32: IP addresses in the same group (\16 prefix for IPv4) should - // always map to the same bucket. - BOOST_CHECK(buckets.size() == 1); + // Test: IP addresses in the same /16 prefix + // usually map to the same bucket. + BOOST_CHECK_EQUAL(buckets.size(), 1U); buckets.clear(); for (int j = 0; j < 4 * 255; j++) { @@ -513,23 +702,102 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) ResolveService( boost::to_string(250 + (j / 255)) + "." + boost::to_string(j % 256) + ".1.1"), NODE_NONE), ResolveIP("251.4.1.1")); - int bucket = infoj.GetNewBucket(nKey1); + int bucket = infoj.GetNewBucket(nKey1, asmap1); buckets.insert(bucket); } - // Test 33: IP addresses in the same source groups should map to no more - // than 64 buckets. + // Test: IP addresses in the same source /16 prefix should not map to more + // than 64 buckets. BOOST_CHECK(buckets.size() <= 64); buckets.clear(); for (int p = 0; p < 255; p++) { CAddrInfo infoj = CAddrInfo( CAddress(ResolveService("250.1.1.1"), NODE_NONE), - ResolveIP("250." + boost::to_string(p) + ".1.1")); - int bucket = infoj.GetNewBucket(nKey1); + ResolveIP("101." + std::to_string(p) + ".1.1")); + int bucket = infoj.GetNewBucket(nKey1, asmap1); buckets.insert(bucket); } - // Test 34: IP addresses in the different source groups should map to more - // than 64 buckets. - BOOST_CHECK(buckets.size() > 64); + // Test: IP addresses in the different source /16 prefixes usually map to MORE + // than 1 bucket. + BOOST_CHECK(buckets.size() > 1); + + buckets.clear(); + for (int p = 0; p < 255; p++) { + CAddrInfo infoj = CAddrInfo( + CAddress(ResolveService("250.1.1.1"), NODE_NONE), + ResolveIP("250." + std::to_string(p) + ".1.1")); + int bucket = infoj.GetNewBucket(nKey1, asmap1); + buckets.insert(bucket); + } + // Test: IP addresses in the different source /16 prefixes sometimes map to NO MORE + // than 1 bucket. + BOOST_CHECK(buckets.size() == 1); + +} + +BOOST_AUTO_TEST_CASE(addrman_serialization) +{ + std::vector asmap1 = FromBytes(asmap, sizeof(asmap) * 8); + + CAddrManTest addrman_asmap1(true, asmap1); + CAddrManTest addrman_asmap1_dup(true, asmap1); + CAddrManTest addrman_noasmap; + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + + CAddress addr = CAddress(ResolveService("250.1.1.1"), NODE_NONE); + CNetAddr default_source; + + + addrman_asmap1.Add(addr, default_source); + + stream << addrman_asmap1; + // serizalizing/deserializing addrman with the same asmap + stream >> addrman_asmap1_dup; + + std::pair bucketAndEntry_asmap1 = addrman_asmap1.GetBucketAndEntry(addr); + std::pair bucketAndEntry_asmap1_dup = addrman_asmap1_dup.GetBucketAndEntry(addr); + BOOST_CHECK(bucketAndEntry_asmap1.second != -1); + BOOST_CHECK(bucketAndEntry_asmap1_dup.second != -1); + + BOOST_CHECK(bucketAndEntry_asmap1.first == bucketAndEntry_asmap1_dup.first); + BOOST_CHECK(bucketAndEntry_asmap1.second == bucketAndEntry_asmap1_dup.second); + + // deserializing asmaped peers.dat to non-asmaped addrman + stream << addrman_asmap1; + stream >> addrman_noasmap; + std::pair bucketAndEntry_noasmap = addrman_noasmap.GetBucketAndEntry(addr); + BOOST_CHECK(bucketAndEntry_noasmap.second != -1); + BOOST_CHECK(bucketAndEntry_asmap1.first != bucketAndEntry_noasmap.first); + BOOST_CHECK(bucketAndEntry_asmap1.second != bucketAndEntry_noasmap.second); + + // deserializing non-asmaped peers.dat to asmaped addrman + addrman_asmap1.Clear(); + addrman_noasmap.Clear(); + addrman_noasmap.Add(addr, default_source); + stream << addrman_noasmap; + stream >> addrman_asmap1; + std::pair bucketAndEntry_asmap1_deser = addrman_asmap1.GetBucketAndEntry(addr); + BOOST_CHECK(bucketAndEntry_asmap1_deser.second != -1); + BOOST_CHECK(bucketAndEntry_asmap1_deser.first != bucketAndEntry_noasmap.first); + BOOST_CHECK(bucketAndEntry_asmap1_deser.first == bucketAndEntry_asmap1_dup.first); + BOOST_CHECK(bucketAndEntry_asmap1_deser.second == bucketAndEntry_asmap1_dup.second); + + // used to map to different buckets, now maps to the same bucket. + addrman_asmap1.Clear(); + addrman_noasmap.Clear(); + CAddress addr1 = CAddress(ResolveService("250.1.1.1"), NODE_NONE); + CAddress addr2 = CAddress(ResolveService("250.2.1.1"), NODE_NONE); + addrman_noasmap.Add(addr, default_source); + addrman_noasmap.Add(addr2, default_source); + std::pair bucketAndEntry_noasmap_addr1 = addrman_noasmap.GetBucketAndEntry(addr1); + std::pair bucketAndEntry_noasmap_addr2 = addrman_noasmap.GetBucketAndEntry(addr2); + BOOST_CHECK(bucketAndEntry_noasmap_addr1.first != bucketAndEntry_noasmap_addr2.first); + BOOST_CHECK(bucketAndEntry_noasmap_addr1.second != bucketAndEntry_noasmap_addr2.second); + stream << addrman_noasmap; + stream >> addrman_asmap1; + std::pair bucketAndEntry_asmap1_deser_addr1 = addrman_asmap1.GetBucketAndEntry(addr1); + std::pair bucketAndEntry_asmap1_deser_addr2 = addrman_asmap1.GetBucketAndEntry(addr2); + BOOST_CHECK(bucketAndEntry_asmap1_deser_addr1.first == bucketAndEntry_asmap1_deser_addr2.first); + BOOST_CHECK(bucketAndEntry_asmap1_deser_addr1.second != bucketAndEntry_asmap1_deser_addr2.second); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/data/asmap.raw b/src/test/data/asmap.raw new file mode 100644 index 0000000000000000000000000000000000000000..3dcf1f3940890af14f613b08aa11387fd002c228 GIT binary patch literal 59 zcmey({Dyyn{egcBzfl?P2TK5m Cn;6Xi literal 0 HcmV?d00001 diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index e58f7df0e3f..cbc0680d996 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -278,20 +278,23 @@ BOOST_AUTO_TEST_CASE(subnet_test) BOOST_AUTO_TEST_CASE(netbase_getgroup) { - - BOOST_CHECK(ResolveIP("127.0.0.1").GetGroup() == boost::assign::list_of(0)); // Local -> !Routable() - BOOST_CHECK(ResolveIP("257.0.0.1").GetGroup() == boost::assign::list_of(0)); // !Valid -> !Routable() - BOOST_CHECK(ResolveIP("10.0.0.1").GetGroup() == boost::assign::list_of(0)); // RFC1918 -> !Routable() - BOOST_CHECK(ResolveIP("169.254.1.1").GetGroup() == boost::assign::list_of(0)); // RFC3927 -> !Routable() - BOOST_CHECK(ResolveIP("1.2.3.4").GetGroup() == boost::assign::list_of((unsigned char)NET_IPV4)(1)(2)); // IPv4 - BOOST_CHECK(ResolveIP("::FFFF:0:102:304").GetGroup() == boost::assign::list_of((unsigned char)NET_IPV4)(1)(2)); // RFC6145 - BOOST_CHECK(ResolveIP("64:FF9B::102:304").GetGroup() == boost::assign::list_of((unsigned char)NET_IPV4)(1)(2)); // RFC6052 - BOOST_CHECK(ResolveIP("2002:102:304:9999:9999:9999:9999:9999").GetGroup() == boost::assign::list_of((unsigned char)NET_IPV4)(1)(2)); // RFC3964 - BOOST_CHECK(ResolveIP("2001:0:9999:9999:9999:9999:FEFD:FCFB").GetGroup() == boost::assign::list_of((unsigned char)NET_IPV4)(1)(2)); // RFC4380 - BOOST_CHECK(ResolveIP("FD87:D87E:EB43:edb1:8e4:3588:e546:35ca").GetGroup() == boost::assign::list_of((unsigned char)NET_TOR)(239)); // Tor - BOOST_CHECK(ResolveIP("2001:470:abcd:9999:9999:9999:9999:9999").GetGroup() == boost::assign::list_of((unsigned char)NET_IPV6)(32)(1)(4)(112)(175)); //he.net - BOOST_CHECK(ResolveIP("2001:2001:9999:9999:9999:9999:9999:9999").GetGroup() == boost::assign::list_of((unsigned char)NET_IPV6)(32)(1)(32)(1)); //IPv6 - + std::vector asmap; // use /16 + BOOST_CHECK(ResolveIP("127.0.0.1").GetGroup(asmap) == std::vector({0})); // Local -> !Routable() + BOOST_CHECK(ResolveIP("257.0.0.1").GetGroup(asmap) == std::vector({0})); // !Valid -> !Routable() + BOOST_CHECK(ResolveIP("10.0.0.1").GetGroup(asmap) == std::vector({0})); // RFC1918 -> !Routable() + BOOST_CHECK(ResolveIP("169.254.1.1").GetGroup(asmap) == std::vector({0})); // RFC3927 -> !Routable() + BOOST_CHECK(ResolveIP("1.2.3.4").GetGroup(asmap) == std::vector({(unsigned char)NET_IPV4, 1, 2})); // IPv4 + BOOST_CHECK(ResolveIP("::FFFF:0:102:304").GetGroup(asmap) == std::vector({(unsigned char)NET_IPV4, 1, 2})); // RFC6145 + BOOST_CHECK(ResolveIP("64:FF9B::102:304").GetGroup(asmap) == std::vector({(unsigned char)NET_IPV4, 1, 2})); // RFC6052 + BOOST_CHECK(ResolveIP("2002:102:304:9999:9999:9999:9999:9999").GetGroup(asmap) == std::vector({(unsigned char)NET_IPV4, 1, 2})); // RFC3964 + BOOST_CHECK(ResolveIP("2001:0:9999:9999:9999:9999:FEFD:FCFB").GetGroup(asmap) == std::vector({(unsigned char)NET_IPV4, 1, 2})); // RFC4380 + BOOST_CHECK(ResolveIP("FD87:D87E:EB43:edb1:8e4:3588:e546:35ca").GetGroup(asmap) == std::vector({(unsigned char)NET_TOR, 239})); // Tor + BOOST_CHECK(ResolveIP("2001:470:abcd:9999:9999:9999:9999:9999").GetGroup(asmap) == std::vector({(unsigned char)NET_IPV6, 32, 1, 4, 112, 175})); //he.net + BOOST_CHECK(ResolveIP("2001:2001:9999:9999:9999:9999:9999:9999").GetGroup(asmap) == std::vector({(unsigned char)NET_IPV6, 32, 1, 32, 1})); //IPv6 + + // baz.net sha256 hash: 12929400eb4607c4ac075f087167e75286b179c693eb059a01774b864e8fe505 + std::vector internal_group = {NET_INTERNAL, 0x12, 0x92, 0x94, 0x00, 0xeb, 0x46, 0x07, 0xc4, 0xac, 0x07}; + BOOST_CHECK(CreateInternal("baz.net").GetGroup(asmap) == internal_group); } BOOST_AUTO_TEST_CASE(netbase_dont_resolve_strings_with_embedded_nul_characters)