From ddad4cc6d21437cddb03be9f6dc1d97aeb9b5951 Mon Sep 17 00:00:00 2001 From: MonsieurNicolas Date: Fri, 16 Mar 2018 09:38:13 -0700 Subject: [PATCH 01/11] update VS project (missed updated in previous commit) --- Builds/VisualStudio/stellar-core.vcxproj | 2 ++ Builds/VisualStudio/stellar-core.vcxproj.filters | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/Builds/VisualStudio/stellar-core.vcxproj b/Builds/VisualStudio/stellar-core.vcxproj index 734d201dcb..b8f5e697a3 100644 --- a/Builds/VisualStudio/stellar-core.vcxproj +++ b/Builds/VisualStudio/stellar-core.vcxproj @@ -425,6 +425,7 @@ exit /b 0 + @@ -608,6 +609,7 @@ exit /b 0 + diff --git a/Builds/VisualStudio/stellar-core.vcxproj.filters b/Builds/VisualStudio/stellar-core.vcxproj.filters index ca5ee962ab..58f874eade 100644 --- a/Builds/VisualStudio/stellar-core.vcxproj.filters +++ b/Builds/VisualStudio/stellar-core.vcxproj.filters @@ -852,6 +852,9 @@ main\generated + + overlay + @@ -1466,6 +1469,9 @@ main + + overlay + From 87c84d5b48573e29952fb4acfd69ba9f664a6566 Mon Sep 17 00:00:00 2001 From: MonsieurNicolas Date: Fri, 16 Mar 2018 09:38:20 -0700 Subject: [PATCH 02/11] warning police --- src/overlay/PeerBareAddress.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/overlay/PeerBareAddress.cpp b/src/overlay/PeerBareAddress.cpp index 1f00b264e9..173ccbbb22 100644 --- a/src/overlay/PeerBareAddress.cpp +++ b/src/overlay/PeerBareAddress.cpp @@ -58,7 +58,7 @@ PeerBareAddress::PeerBareAddress(PeerAddress const& pa) : mType{Type::IPv4} ip << (int)pa.ip.ipv4()[0] << "." << (int)pa.ip.ipv4()[1] << "." << (int)pa.ip.ipv4()[2] << "." << (int)pa.ip.ipv4()[3]; mIP = ip.str(); - mPort = pa.port; + mPort = static_cast(pa.port); } PeerBareAddress From 416748f2d50f27e4b9d349dc38967448a3aa2b0a Mon Sep 17 00:00:00 2001 From: MonsieurNicolas Date: Fri, 16 Mar 2018 12:02:29 -0700 Subject: [PATCH 03/11] added fs::getMaxConnections this allows to compute an upper bound for the number of connections possible --- src/util/Fs.cpp | 26 ++++++++++++++++++++++++++ src/util/Fs.h | 3 +++ 2 files changed, 29 insertions(+) diff --git a/src/util/Fs.cpp b/src/util/Fs.cpp index 63f9cab0cc..fa7c7d1538 100644 --- a/src/util/Fs.cpp +++ b/src/util/Fs.cpp @@ -15,6 +15,7 @@ #include #else #include +#include #include #endif @@ -440,5 +441,30 @@ checkNoGzipSuffix(std::string const& filename) throw std::runtime_error("filename ends in .gz"); } } + +#ifdef _WIN32 + +int +getMaxConnections() +{ + // on Windows, there is no limit on handles + // only limits based on ephemeral ports, etc + return 32000; +} + +#else +int +getMaxConnections() +{ + struct rlimit rl; + if (getrlimit(RLIMIT_NOFILE, &rl) == 0) + { + // leave some buffer + return (rl.rlim_cur * 3) / 4; + } + // could not query the limit, default to a value that should work + return 64; +} +#endif } } diff --git a/src/util/Fs.h b/src/util/Fs.h index c11c30c625..644cd7d226 100644 --- a/src/util/Fs.h +++ b/src/util/Fs.h @@ -85,5 +85,8 @@ std::string remoteName(std::string const& type, std::string const& hexStr, void checkGzipSuffix(std::string const& filename); void checkNoGzipSuffix(std::string const& filename); + +// returns the maximum number of connections that can be done at the same time +int getMaxConnections(); } } From a52ead5c8cc358c3b4d52f1b0a943dcf31ea94d3 Mon Sep 17 00:00:00 2001 From: MonsieurNicolas Date: Fri, 16 Mar 2018 12:02:55 -0700 Subject: [PATCH 04/11] reduce default MAX_PENDING_CONNECTIONS to 500 --- src/main/Config.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/Config.cpp b/src/main/Config.cpp index 2f2ebc70e5..a0805608b9 100644 --- a/src/main/Config.cpp +++ b/src/main/Config.cpp @@ -67,7 +67,7 @@ Config::Config() : NODE_SEED(SecretKey::random()) TARGET_PEER_CONNECTIONS = 8; MAX_ADDITIONAL_PEER_CONNECTIONS = -1; MAX_PEER_CONNECTIONS = 12; - MAX_PENDING_CONNECTIONS = 5000; + MAX_PENDING_CONNECTIONS = 500; PEER_AUTHENTICATION_TIMEOUT = 2; PEER_TIMEOUT = 30; PREFERRED_PEERS_ONLY = false; From abd0af9f1e36ee7b570b3cb8504d1eb306d344ad Mon Sep 17 00:00:00 2001 From: MonsieurNicolas Date: Fri, 16 Mar 2018 12:03:57 -0700 Subject: [PATCH 05/11] ensure that number of connections are within bounds MAX_PENDING_CONNECTIONS >= MAX_PEER_CONNECTIONS >= TARGET_PEER_CONNECTIONS --- src/main/Config.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/main/Config.cpp b/src/main/Config.cpp index a0805608b9..88a6d27849 100644 --- a/src/main/Config.cpp +++ b/src/main/Config.cpp @@ -11,6 +11,7 @@ #include "main/ExternalQueue.h" #include "main/StellarCoreVersion.h" #include "scp/LocalNode.h" +#include "util/Fs.h" #include "util/Logging.h" #include "util/types.h" @@ -527,6 +528,17 @@ Config::load(std::string const& filename) static_cast(MAX_ADDITIONAL_PEER_CONNECTIONS + TARGET_PEER_CONNECTIONS)); + // ensure that max pending connections is not above what the system + // supports + MAX_PENDING_CONNECTIONS = static_cast( + std::min(MAX_PENDING_CONNECTIONS, fs::getMaxConnections())); + + // enforce TARGET_PEER_CONNECTIONS <= MAX_PEER_CONNECTIONS <= + // MAX_PENDING_CONNECTIONS + MAX_PEER_CONNECTIONS = + std::min(MAX_PEER_CONNECTIONS, MAX_PENDING_CONNECTIONS); + TARGET_PEER_CONNECTIONS = + std::min(TARGET_PEER_CONNECTIONS, MAX_PEER_CONNECTIONS); validateConfig(); } catch (cpptoml::toml_parse_exception& ex) From f2adb6ab210bbbbbe0cca49cb3a2bf2e17da4bf3 Mon Sep 17 00:00:00 2001 From: MonsieurNicolas Date: Fri, 16 Mar 2018 13:52:42 -0700 Subject: [PATCH 06/11] fixed PeerBareAddress range checks was done after casting down to unsigned short --- src/overlay/LoopbackPeer.cpp | 5 +++-- src/overlay/LoopbackPeer.h | 3 +-- src/overlay/OverlayManagerTests.cpp | 2 +- src/overlay/Peer.h | 3 +-- src/overlay/PeerBareAddress.cpp | 2 +- src/overlay/TCPPeer.cpp | 6 ++++-- src/overlay/TCPPeer.h | 3 +-- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/overlay/LoopbackPeer.cpp b/src/overlay/LoopbackPeer.cpp index 509070edea..bf3e76e345 100644 --- a/src/overlay/LoopbackPeer.cpp +++ b/src/overlay/LoopbackPeer.cpp @@ -28,7 +28,7 @@ LoopbackPeer::LoopbackPeer(Application& app, PeerRole role) : Peer(app, role) } PeerBareAddress -LoopbackPeer::makeAddress(unsigned short remoteListeningPort) const +LoopbackPeer::makeAddress(int remoteListeningPort) const { if (remoteListeningPort <= 0 || remoteListeningPort > UINT16_MAX) { @@ -36,7 +36,8 @@ LoopbackPeer::makeAddress(unsigned short remoteListeningPort) const } else { - return PeerBareAddress{"127.0.0.1", remoteListeningPort}; + return PeerBareAddress{ + "127.0.0.1", static_cast(remoteListeningPort)}; } } diff --git a/src/overlay/LoopbackPeer.h b/src/overlay/LoopbackPeer.h index 4407d3ce47..0ba7a82a63 100644 --- a/src/overlay/LoopbackPeer.h +++ b/src/overlay/LoopbackPeer.h @@ -53,8 +53,7 @@ class LoopbackPeer : public Peer Stats mStats; void sendMessage(xdr::msg_ptr&& xdrBytes) override; - PeerBareAddress - makeAddress(unsigned short remoteListeningPort) const override; + PeerBareAddress makeAddress(int remoteListeningPort) const override; AuthCert getAuthCert() override; void processInQueue(); diff --git a/src/overlay/OverlayManagerTests.cpp b/src/overlay/OverlayManagerTests.cpp index e75a42028d..f640d59982 100644 --- a/src/overlay/OverlayManagerTests.cpp +++ b/src/overlay/OverlayManagerTests.cpp @@ -40,7 +40,7 @@ class PeerStub : public Peer mAddress = addres; } virtual PeerBareAddress - makeAddress(unsigned short) const override + makeAddress(int) const override { REQUIRE(false); // should not be called return {}; diff --git a/src/overlay/Peer.h b/src/overlay/Peer.h index 694f230521..c4df871d64 100644 --- a/src/overlay/Peer.h +++ b/src/overlay/Peer.h @@ -178,8 +178,7 @@ class Peer : public std::enable_shared_from_this, } virtual AuthCert getAuthCert(); - virtual PeerBareAddress - makeAddress(unsigned short remoteListeningPort) const = 0; + virtual PeerBareAddress makeAddress(int remoteListeningPort) const = 0; void startIdleTimer(); void idleTimerExpired(asio::error_code const& error); diff --git a/src/overlay/PeerBareAddress.cpp b/src/overlay/PeerBareAddress.cpp index 173ccbbb22..aac7ffe67a 100644 --- a/src/overlay/PeerBareAddress.cpp +++ b/src/overlay/PeerBareAddress.cpp @@ -150,7 +150,7 @@ PeerBareAddress::toString() const return mIP + ":" + std::to_string(mPort); } default: - assert(false); + abort(); } } diff --git a/src/overlay/TCPPeer.cpp b/src/overlay/TCPPeer.cpp index ca1392d617..ec724d7ace 100644 --- a/src/overlay/TCPPeer.cpp +++ b/src/overlay/TCPPeer.cpp @@ -113,7 +113,7 @@ TCPPeer::~TCPPeer() } PeerBareAddress -TCPPeer::makeAddress(unsigned short remoteListeningPort) const +TCPPeer::makeAddress(int remoteListeningPort) const { asio::error_code ec; auto ep = mSocket->next_layer().remote_endpoint(ec); @@ -123,7 +123,9 @@ TCPPeer::makeAddress(unsigned short remoteListeningPort) const } else { - return PeerBareAddress{ep.address().to_string(), remoteListeningPort}; + return PeerBareAddress{ + ep.address().to_string(), + static_cast(remoteListeningPort)}; } } diff --git a/src/overlay/TCPPeer.h b/src/overlay/TCPPeer.h index 62d9f75720..32e2494199 100644 --- a/src/overlay/TCPPeer.h +++ b/src/overlay/TCPPeer.h @@ -35,8 +35,7 @@ class TCPPeer : public Peer bool mDelayedShutdown{false}; bool mShutdownScheduled{false}; - PeerBareAddress - makeAddress(unsigned short remoteListeningPort) const override; + PeerBareAddress makeAddress(int remoteListeningPort) const override; void recvMessage(); void sendMessage(xdr::msg_ptr&& xdrBytes) override; From bec858b396470765a9dbab386f3579c8bd98323f Mon Sep 17 00:00:00 2001 From: MonsieurNicolas Date: Fri, 16 Mar 2018 13:53:35 -0700 Subject: [PATCH 07/11] fixed: loadPeerRecords could return more items than requested --- src/overlay/PeerRecord.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/overlay/PeerRecord.cpp b/src/overlay/PeerRecord.cpp index 1c62a7814d..0d94e40df4 100644 --- a/src/overlay/PeerRecord.cpp +++ b/src/overlay/PeerRecord.cpp @@ -125,7 +125,7 @@ PeerRecord::loadPeerRecords(Database& db, int batchSize, try { int offset = 0; - bool didSomething; + bool lastRes; do { tm nextAttemptMax = VirtualClock::pointToTm(nextAttemptCutoff); @@ -141,14 +141,14 @@ PeerRecord::loadPeerRecords(Database& db, int batchSize, st.exchange(use(batchSize)); st.exchange(use(offset)); - didSomething = false; + lastRes = false; loadPeerRecords(db, prep, [&](PeerRecord const& pr) { offset++; - didSomething = true; - return pred(pr); + lastRes = pred(pr); + return lastRes; }); - } while (didSomething); + } while (lastRes); } catch (soci_error& err) { From 711cd1eb8f4a73ca061fa13c7d0d15f4a6352cd5 Mon Sep 17 00:00:00 2001 From: MonsieurNicolas Date: Fri, 16 Mar 2018 13:54:46 -0700 Subject: [PATCH 08/11] Overlay: avoid initiating connection to peers when limit is reached --- src/overlay/OverlayManagerImpl.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/overlay/OverlayManagerImpl.cpp b/src/overlay/OverlayManagerImpl.cpp index fe30d8fd26..a5946d85c6 100644 --- a/src/overlay/OverlayManagerImpl.cpp +++ b/src/overlay/OverlayManagerImpl.cpp @@ -135,7 +135,16 @@ OverlayManagerImpl::connectTo(PeerRecord& pr) pr.backOff(mApp.getClock()); pr.storePeerRecord(mApp.getDatabase()); - addPendingPeer(TCPPeer::initiate(mApp, pr.getAddress())); + if (getPendingPeersCount() < mApp.getConfig().MAX_PENDING_CONNECTIONS) + { + addPendingPeer(TCPPeer::initiate(mApp, pr.getAddress())); + } + else + { + CLOG(DEBUG, "Overlay") + << "reached maximum number of pending connections, backing off " + << pr.toString(); + } } else { From 55836f64b5bbe53f8f84e1a6fe6597c47a8d5c11 Mon Sep 17 00:00:00 2001 From: MonsieurNicolas Date: Fri, 16 Mar 2018 13:57:04 -0700 Subject: [PATCH 09/11] guard against too many peers being received --- src/xdr/Stellar-overlay.x | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/xdr/Stellar-overlay.x b/src/xdr/Stellar-overlay.x index 79ae734138..474c0b4856 100644 --- a/src/xdr/Stellar-overlay.x +++ b/src/xdr/Stellar-overlay.x @@ -112,7 +112,7 @@ case DONT_HAVE: case GET_PEERS: void; case PEERS: - PeerAddress peers<>; + PeerAddress peers<100>; case GET_TX_SET: uint256 txSetHash; From 393eb97ea65ac6545b86f07ed9fc79527b115279 Mon Sep 17 00:00:00 2001 From: MonsieurNicolas Date: Fri, 16 Mar 2018 14:06:13 -0700 Subject: [PATCH 10/11] ensure that we don't send more peers than allowed by overlay protocol --- src/overlay/Peer.cpp | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/overlay/Peer.cpp b/src/overlay/Peer.cpp index b0128669c5..46c2b71ca4 100644 --- a/src/overlay/Peer.cpp +++ b/src/overlay/Peer.cpp @@ -378,19 +378,25 @@ Peer::sendGetScpState(uint32 ledgerSeq) void Peer::sendPeers() { - // send top 50 peers we know about + StellarMessage newMsg; + newMsg.type(PEERS); + uint32 maxPeerCount = std::min(50, newMsg.peers().max_size()); + + // send top peers we know about vector peerList; PeerRecord::loadPeerRecords(mApp.getDatabase(), 50, mApp.getClock().now(), [&](PeerRecord const& pr) { - if (!pr.getAddress().isPrivate() && - pr.getAddress() != mAddress) + bool r = peerList.size() < maxPeerCount; + if (r) { - peerList.emplace_back(pr); + if (!pr.getAddress().isPrivate() && + pr.getAddress() != mAddress) + { + peerList.emplace_back(pr); + } } - return peerList.size() < 50; + return r; }); - StellarMessage newMsg; - newMsg.type(PEERS); newMsg.peers().reserve(peerList.size()); for (auto const& pr : peerList) { From 899d7bc3a65da29109b3f52ec600202527a91935 Mon Sep 17 00:00:00 2001 From: MonsieurNicolas Date: Fri, 16 Mar 2018 14:13:26 -0700 Subject: [PATCH 11/11] don't connect to too many peers at once hardcoded for now at 50 peers per calls to `getPeersToConnectTo` --- src/overlay/OverlayManagerImpl.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/overlay/OverlayManagerImpl.cpp b/src/overlay/OverlayManagerImpl.cpp index a5946d85c6..e25f730570 100644 --- a/src/overlay/OverlayManagerImpl.cpp +++ b/src/overlay/OverlayManagerImpl.cpp @@ -231,7 +231,11 @@ OverlayManagerImpl::getPreferredPeersFromConfig() std::vector OverlayManagerImpl::getPeersToConnectTo(int maxNum) { - const int batchSize = std::max(20, maxNum); + // don't connect to too many peers at once + maxNum = std::min(maxNum, 50); + + // batch is how many peers to load from the database every time + const int batchSize = std::max(50, maxNum); std::vector peers;