From a81d60e55d59e4cc4b576bb489ad82060a468dbe Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Fri, 9 Sep 2022 12:26:48 +0200 Subject: [PATCH] update dial timeout, ensure K found nodes (#2083) * update dial timeout, ensure K found nodes * add libp2p config variables to node config * add network module to default test config * bump store protocol version * Updated logs for removing file on path * Updated logs for removing file on path Co-authored-by: Djordje Kovacevic --- config/config.json | 53 +++++++++++++++++++ dependencies.md | 4 ++ package-lock.json | 10 ++-- package.json | 1 + src/constants/constants.js | 2 +- .../network/implementation/libp2p-service.js | 33 +++++++----- src/service/file-service.js | 4 +- .../origintrail-test-bootstrap-config.json | 19 +++++-- .../config/origintrail-test-node-config.json | 21 ++++++-- 9 files changed, 121 insertions(+), 26 deletions(-) diff --git a/config/config.json b/config/config.json index feb6603edf..da0ecacc10 100644 --- a/config/config.json +++ b/config/config.json @@ -36,6 +36,19 @@ "libp2p-service": { "package": "./network/implementation/libp2p-service.js", "config": { + "kBucketSize": 20, + "connectionManager": { + "autoDial": true, + "autoDialInterval": 10e3, + "dialTimeout": 2e3 + }, + "peerRouting": { + "refreshManager": { + "enabled": true, + "interval": 6e5, + "bootDelay": 2e3 + } + }, "port": 9000, "bootstrap": [] } @@ -181,6 +194,33 @@ } } }, + "network": { + "enabled": true, + "implementation": { + "libp2p-service": { + "package": "./network/implementation/libp2p-service.js", + "config": { + "kBucketSize": 20, + "connectionManager": { + "autoDial": true, + "autoDialInterval": 10e3, + "dialTimeout": 2e3 + }, + "peerRouting": { + "refreshManager": { + "enabled": true, + "interval": 6e5, + "bootDelay": 2e3 + } + }, + "port": 9000, + "bootstrap": [ + "/ip4/0.0.0.0/tcp/9000/p2p/QmWyf3dtqJnhuCpzEDTNmNFYc5tjxTrXhGcUUmGHdg2gtj" + ] + } + } + } + }, "repository": { "enabled": true, "implementation": { @@ -286,6 +326,19 @@ "libp2p-service": { "package": "./network/implementation/libp2p-service.js", "config": { + "kBucketSize": 20, + "connectionManager": { + "autoDial": true, + "autoDialInterval": 10e3, + "dialTimeout": 2e3 + }, + "peerRouting": { + "refreshManager": { + "enabled": true, + "interval": 6e5, + "bootDelay": 2e3 + } + }, "port": 9000, "bootstrap": [ "/ip4/139.59.174.24/tcp/9000/p2p/QmU7p61f8qgzCtfTEUynJaqnsLRN4QExjtWfsMRQ7oiMuS", diff --git a/dependencies.md b/dependencies.md index 32d5345146..cabcee5709 100644 --- a/dependencies.md +++ b/dependencies.md @@ -218,6 +218,10 @@ - **version**: ^2.1.3 - **description**: convert expiration time to milliseconds in `token-generation.js` +##### [multiformats](https://www.npmjs.com/package/multiformats) +- **version**: ^9.8.1 +- **description**: calculate sha256 hash of peerIds (libp2p find nodes operation) + ##### [mysql2](https://www.npmjs.com/package/mysql2) - **version**: ^2.3.3 - **description**: diff --git a/package-lock.json b/package-lock.json index d70d8a1f93..0d187fda08 100644 --- a/package-lock.json +++ b/package-lock.json @@ -44,6 +44,7 @@ "libp2p": "^0.38.0", "merkletreejs": "^0.2.32", "ms": "^2.1.3", + "multiformats": "^9.8.1", "mysql2": "^2.3.3", "p-iteration": "^1.1.8", "pino": "^8.4.2", @@ -14522,8 +14523,9 @@ "license": "MIT" }, "node_modules/multiformats": { - "version": "9.7.1", - "license": "(Apache-2.0 AND MIT)" + "version": "9.8.1", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-9.8.1.tgz", + "integrity": "sha512-Cu7NfUYtCV+WN7w59WsRRF138S+um4tTo11ScYsWbNgWyCEGOu8wID1e5eMJs91gFZ0I7afodkkdxCF8NGkqZQ==" }, "node_modules/multihashes": { "version": "0.4.21", @@ -30384,7 +30386,9 @@ } }, "multiformats": { - "version": "9.7.1" + "version": "9.8.1", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-9.8.1.tgz", + "integrity": "sha512-Cu7NfUYtCV+WN7w59WsRRF138S+um4tTo11ScYsWbNgWyCEGOu8wID1e5eMJs91gFZ0I7afodkkdxCF8NGkqZQ==" }, "multihashes": { "version": "0.4.21", diff --git a/package.json b/package.json index 388c5feb00..8f6ed0f379 100644 --- a/package.json +++ b/package.json @@ -94,6 +94,7 @@ "libp2p": "^0.38.0", "merkletreejs": "^0.2.32", "ms": "^2.1.3", + "multiformats": "^9.8.1", "mysql2": "^2.3.3", "p-iteration": "^1.1.8", "pino": "^8.4.2", diff --git a/src/constants/constants.js b/src/constants/constants.js index c7dab1d2b6..ee647be608 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -221,7 +221,7 @@ export const COMMAND_STATUS = { * Network protocols */ export const NETWORK_PROTOCOLS = { - STORE: '/store/1.0.0', + STORE: '/store/1.0.1', GET: '/get/1.0.0', SEARCH: '/search/1.0.0', }; diff --git a/src/modules/network/implementation/libp2p-service.js b/src/modules/network/implementation/libp2p-service.js index 329f56a825..ac3a483428 100644 --- a/src/modules/network/implementation/libp2p-service.js +++ b/src/modules/network/implementation/libp2p-service.js @@ -1,5 +1,6 @@ /* eslint-disable import/no-unresolved */ import { createLibp2p } from 'libp2p'; +import { sha256 } from 'multiformats/hashes/sha2'; import { Bootstrap } from '@libp2p/bootstrap'; import { Mplex } from '@libp2p/mplex'; import { Noise } from '@chainsafe/libp2p-noise'; @@ -10,6 +11,8 @@ import * as lp from 'it-length-prefixed'; import { unmarshalPrivateKey } from '@libp2p/crypto/keys'; import { toString as uint8ArrayToString } from 'uint8arrays/to-string'; import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'; +import { xor as uint8ArrayXor } from 'uint8arrays/xor'; +import { compare as uint8ArrayCompare } from 'uint8arrays/compare'; import map from 'it-map'; import { createFromPrivKey, createRSAPeerId } from '@libp2p/peer-id-factory'; import { InMemoryRateLimiter } from 'rolling-rate-limiter'; @@ -29,7 +32,6 @@ const initializationObject = { transports: [new TCP()], streamMuxers: [new Mplex()], connectionEncryption: [new Noise()], - dht: new KadDHT(), }; class Libp2pService { @@ -37,6 +39,10 @@ class Libp2pService { this.config = config; this.logger = logger; + initializationObject.dht = new KadDHT({ kBucketSize: this.config.kBucketSize }); + initializationObject.peerRouting = this.config.peerRouting; + initializationObject.connectionManager = this.config.connectionManager; + if (this.config.bootstrap.length > 0) { initializationObject.peerDiscovery = [ new Bootstrap({ @@ -87,9 +93,6 @@ class Libp2pService { } _initializeNodeListeners() { - this.node.addEventListener('peer:discovery', (peer) => { - this._onPeerDiscovery(peer); - }); this.node.connectionManager.addEventListener('peer:connect', (connection) => { this._onPeerConnect(connection); }); @@ -114,12 +117,6 @@ class Libp2pService { this.blackList = {}; } - _onPeerDiscovery(peer) { - this.logger.trace( - `Node ${this.node.peerId.toString()} discovered ${peer.detail.id.toString()}`, - ); - } - _onPeerConnect(connection) { this.logger.trace( `Node ${this.node.peerId.toString()} connected to ${connection.detail.remotePeer.toString()}`, @@ -152,12 +149,22 @@ class Libp2pService { }, ); - const nodes = []; + const keyHash = Buffer.from((await sha256.digest(encodedKey)).digest); + const unsortedPeerDistances = []; for await (const finalPeerId of finalPeerIds) { - nodes.push(finalPeerId); + const peerHash = Buffer.from((await sha256.digest(finalPeerId.toBytes())).digest); + + unsortedPeerDistances.push({ + peerId: finalPeerId, + distance: uint8ArrayXor(keyHash, peerHash), + }); } - return nodes; + const sortedPeers = unsortedPeerDistances + .sort((a, b) => uint8ArrayCompare(a.distance, b.distance)) + .map((pd) => pd.peerId); + + return sortedPeers.slice(0, this.config.kBucketSize); } getPeers() { diff --git a/src/service/file-service.js b/src/service/file-service.js index f5c9d3d1fa..4f19c07b88 100644 --- a/src/service/file-service.js +++ b/src/service/file-service.js @@ -64,7 +64,7 @@ class FileService { } async removeFile(filePath) { - this.logger.debug(`Removing file on path: ${filePath}`); + this.logger.trace(`Removing file on path: ${filePath}`); await unlink(filePath); return true; } @@ -107,9 +107,9 @@ class FileService { if (createdDate.getTime() + expiredTimeout < now.getTime()) { // eslint-disable-next-line no-await-in-loop await this.removeFile(filePath); - this.logger.trace(`Successfully removed expired cache file: ${filePath}`); } } + this.logger.trace(`Successfully removed ${fileList.length} expired cache files`); } } diff --git a/test/bdd/steps/config/origintrail-test-bootstrap-config.json b/test/bdd/steps/config/origintrail-test-bootstrap-config.json index eeca35cdd0..2e78cb73a8 100644 --- a/test/bdd/steps/config/origintrail-test-bootstrap-config.json +++ b/test/bdd/steps/config/origintrail-test-bootstrap-config.json @@ -25,9 +25,22 @@ "libp2p-service": { "package": "./network/implementation/libp2p-service.js", "config": { - "port": 9000, - "bootstrap": [], - "privateKey": "CAAS4QQwggJdAgEAAoGBALOYSCZsmINMpFdH8ydA9CL46fB08F3ELfb9qiIq+z4RhsFwi7lByysRnYT/NLm8jZ4RvlsSqOn2ZORJwBywYD5MCvU1TbEWGKxl5LriW85ZGepUwiTZJgZdDmoLIawkpSdmUOc1Fbnflhmj/XzAxlnl30yaa/YvKgnWtZI1/IwfAgMBAAECgYEAiZq2PWqbeI6ypIVmUr87z8f0Rt7yhIWZylMVllRkaGw5WeGHzQwSRQ+cJ5j6pw1HXMOvnEwxzAGT0C6J2fFx60C6R90TPos9W0zSU+XXLHA7AtazjlSnp6vHD+RxcoUhm1RUPeKU6OuUNcQVJu1ZOx6cAcP/I8cqL38JUOOS7XECQQDex9WUKtDnpHEHU/fl7SvCt0y2FbGgGdhq6k8nrWtBladP5SoRUFuQhCY8a20fszyiAIfxQrtpQw1iFPBpzoq1AkEAzl/s3XPGi5vFSNGLsLqbVKbvoW9RUaGN8o4rU9oZmPFL31Jo9FLA744YRer6dYE7jJMel7h9VVWsqa9oLGS8AwJALYwfv45Nbb6yGTRyr4Cg/MtrFKM00K3YEGvdSRhsoFkPfwc0ZZvPTKmoA5xXEC8eC2UeZhYlqOy7lL0BNjCzLQJBAMpvcgtwa8u6SvU5B0ueYIvTDLBQX3YxgOny5zFjeUR7PS+cyPMQ0cyql8jNzEzDLcSg85tkDx1L4wi31Pnm/j0CQFH/6MYn3r9benPm2bYSe9aoJp7y6ht2DmXmoveNbjlEbb8f7jAvYoTklJxmJCcrdbNx/iCj2BuAinPPgEmUzfQ=" + "kBucketSize": 20, + "connectionManager": { + "autoDial": true, + "autoDialInterval": 10e3, + "dialTimeout": 2e3 + }, + "peerRouting": { + "refreshManager": { + "enabled": true, + "interval": 6e5, + "bootDelay": 2e3 + } + }, + "port": 9000, + "bootstrap": [], + "privateKey": "CAAS4QQwggJdAgEAAoGBALOYSCZsmINMpFdH8ydA9CL46fB08F3ELfb9qiIq+z4RhsFwi7lByysRnYT/NLm8jZ4RvlsSqOn2ZORJwBywYD5MCvU1TbEWGKxl5LriW85ZGepUwiTZJgZdDmoLIawkpSdmUOc1Fbnflhmj/XzAxlnl30yaa/YvKgnWtZI1/IwfAgMBAAECgYEAiZq2PWqbeI6ypIVmUr87z8f0Rt7yhIWZylMVllRkaGw5WeGHzQwSRQ+cJ5j6pw1HXMOvnEwxzAGT0C6J2fFx60C6R90TPos9W0zSU+XXLHA7AtazjlSnp6vHD+RxcoUhm1RUPeKU6OuUNcQVJu1ZOx6cAcP/I8cqL38JUOOS7XECQQDex9WUKtDnpHEHU/fl7SvCt0y2FbGgGdhq6k8nrWtBladP5SoRUFuQhCY8a20fszyiAIfxQrtpQw1iFPBpzoq1AkEAzl/s3XPGi5vFSNGLsLqbVKbvoW9RUaGN8o4rU9oZmPFL31Jo9FLA744YRer6dYE7jJMel7h9VVWsqa9oLGS8AwJALYwfv45Nbb6yGTRyr4Cg/MtrFKM00K3YEGvdSRhsoFkPfwc0ZZvPTKmoA5xXEC8eC2UeZhYlqOy7lL0BNjCzLQJBAMpvcgtwa8u6SvU5B0ueYIvTDLBQX3YxgOny5zFjeUR7PS+cyPMQ0cyql8jNzEzDLcSg85tkDx1L4wi31Pnm/j0CQFH/6MYn3r9benPm2bYSe9aoJp7y6ht2DmXmoveNbjlEbb8f7jAvYoTklJxmJCcrdbNx/iCj2BuAinPPgEmUzfQ=" } } } diff --git a/test/bdd/steps/config/origintrail-test-node-config.json b/test/bdd/steps/config/origintrail-test-node-config.json index 74a0bedc08..d8da27fbce 100644 --- a/test/bdd/steps/config/origintrail-test-node-config.json +++ b/test/bdd/steps/config/origintrail-test-node-config.json @@ -25,10 +25,23 @@ "libp2p-service": { "package": "./network/implementation/libp2p-service.js", "config": { - "port": 9001, - "bootstrap": [ - "/ip4/0.0.0.0/tcp/9000/p2p/QmWyf3dtqJnhuCpzEDTNmNFYc5tjxTrXhGcUUmGHdg2gtj" - ] + "kBucketSize": 20, + "connectionManager": { + "autoDial": true, + "autoDialInterval": 10e3, + "dialTimeout": 2e3 + }, + "peerRouting": { + "refreshManager": { + "enabled": true, + "interval": 6e5, + "bootDelay": 2e3 + } + }, + "port": 9001, + "bootstrap": [ + "/ip4/0.0.0.0/tcp/9000/p2p/QmWyf3dtqJnhuCpzEDTNmNFYc5tjxTrXhGcUUmGHdg2gtj" + ] } } }