From cb04ad7b9603b93777273b695065c55c32e745b2 Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Mon, 16 Dec 2019 14:13:01 +0100 Subject: [PATCH 1/3] chore: update libp2p dependencies to refactor versions --- package.json | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index d29278f6de..73e056cb4b 100644 --- a/package.json +++ b/package.json @@ -131,29 +131,29 @@ "kind-of": "^6.0.2", "ky": "^0.15.0", "ky-universal": "~0.3.0", - "libp2p": "^0.26.2", - "libp2p-bootstrap": "~0.9.3", - "libp2p-crypto": "^0.16.2", + "libp2p": "^0.27.0-pre.1", + "libp2p-bootstrap": "^0.10.2", + "libp2p-crypto": "^0.17.1", "libp2p-delegated-content-routing": "^0.4.1", - "libp2p-delegated-peer-routing": "^0.3.1", - "libp2p-floodsub": "^0.18.0", - "libp2p-gossipsub": "~0.0.5", - "libp2p-kad-dht": "~0.16.0", + "libp2p-delegated-peer-routing": "^0.4.0", + "libp2p-floodsub": "^0.20.0", + "libp2p-gossipsub": "^0.2.0", + "libp2p-kad-dht": "^0.18.3", "libp2p-keychain": "^0.5.2", - "libp2p-mdns": "~0.12.0", + "libp2p-mdns": "^0.13.0", "libp2p-record": "~0.7.0", - "libp2p-secio": "~0.11.0", - "libp2p-tcp": "^0.13.0", - "libp2p-webrtc-star": "~0.16.0", + "libp2p-secio": "^0.12.1", + "libp2p-tcp": "^0.14.2", + "libp2p-webrtc-star": "^0.17.0", "libp2p-websocket-star-multi": "~0.4.3", - "libp2p-websockets": "~0.12.3", + "libp2p-websockets": "^0.13.0", "lodash.flatten": "^4.4.0", "mafmt": "^6.0.10", "merge-options": "^2.0.0", "mime-types": "^2.1.21", "mkdirp": "~0.5.1", "mortice": "^2.0.0", - "multiaddr": "^6.1.1", + "multiaddr": "^7.2.1", "multiaddr-to-uri": "^5.0.0", "multibase": "~0.6.0", "multicodec": "~0.5.5", @@ -162,9 +162,8 @@ "node-fetch": "^2.3.0", "p-iteration": "^1.1.8", "p-queue": "^6.1.0", - "peer-book": "^0.9.1", - "peer-id": "~0.12.2", - "peer-info": "~0.15.1", + "peer-id": "^0.13.5", + "peer-info": "^0.17.0", "pretty-bytes": "^5.3.0", "progress": "^2.0.1", "promise-nodeify": "^3.0.1", From a7c14b72cd8ebd86e8e6087116fcd626a2a3d82e Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Tue, 17 Dec 2019 20:46:37 -0600 Subject: [PATCH 2/3] refactor: update libp2p config, component and startup to use async --- src/core/components/index.js | 2 +- src/core/components/libp2p.js | 33 +++++++++++++++--------------- src/core/components/start.js | 16 +++++++-------- src/core/runtime/libp2p-browser.js | 25 ++++++---------------- src/core/runtime/libp2p-nodejs.js | 20 +++++------------- 5 files changed, 35 insertions(+), 61 deletions(-) diff --git a/src/core/components/index.js b/src/core/components/index.js index 4a9bcab492..95d3ae7e77 100644 --- a/src/core/components/index.js +++ b/src/core/components/index.js @@ -23,6 +23,7 @@ exports.object = { put: require('./object/put'), stat: require('./object/stat') } +exports.libp2p = require('./libp2p') exports.ping = require('./ping') exports.start = require('./start') exports.stop = require('./stop') @@ -37,6 +38,5 @@ exports.version = require('./version') exports.legacy = { // TODO: these will be removed as the new API is completed dag: require('./dag'), - libp2p: require('./libp2p'), pin: require('./pin') } diff --git a/src/core/components/libp2p.js b/src/core/components/libp2p.js index e0b670b7d9..348c2509f7 100644 --- a/src/core/components/libp2p.js +++ b/src/core/components/libp2p.js @@ -9,25 +9,27 @@ const DelegatedPeerRouter = require('libp2p-delegated-peer-routing') const DelegatedContentRouter = require('libp2p-delegated-content-routing') const PubsubRouters = require('../runtime/libp2p-pubsub-routers-nodejs') -module.exports = function libp2p (self, config) { - const options = self._options || {} - config = config || {} +module.exports = ({ + constructorOptions, + peerInfo, + repo, + print, + config +}) => { + const { datastore } = repo + const libp2pOptions = getLibp2pOptions({ options: constructorOptions, config, datastore, peerInfo }) - const { datastore } = self._repo - const peerInfo = self._peerInfo - const peerBook = self._peerInfoBook - - const libp2pOptions = getLibp2pOptions({ options, config, datastore, peerInfo, peerBook }) let libp2p if (typeof options.libp2p === 'function') { - libp2p = options.libp2p({ libp2pOptions, options, config, datastore, peerInfo, peerBook }) + libp2p = options.libp2p({ libp2pOptions, options, config, datastore, peerInfo }) } else { // Required inline to reduce startup time const Libp2p = require('libp2p') libp2p = new Libp2p(mergeOptions(libp2pOptions, get(options, 'libp2p', {}))) } + libp2p.on('stop', () => { // Clear our addresses so we can start clean peerInfo.multiaddrs.clear() @@ -35,16 +37,14 @@ module.exports = function libp2p (self, config) { libp2p.on('start', () => { peerInfo.multiaddrs.forEach((ma) => { - self._print('Swarm listening on', ma.toString()) + print('Swarm listening on', ma.toString()) }) }) - libp2p.on('peer:connect', peerInfo => peerBook.put(peerInfo)) - return libp2p } -function getLibp2pOptions ({ options, config, datastore, peerInfo, peerBook }) { +function getLibp2pOptions ({ options, config, datastore, peerInfo }) { // Set up Delegate Routing based on the presence of Delegates in the config let contentRouting let peerRouting @@ -82,7 +82,6 @@ function getLibp2pOptions ({ options, config, datastore, peerInfo, peerBook }) { const libp2pDefaults = { datastore, peerInfo, - peerBook, modules: { contentRouting, peerRouting @@ -138,8 +137,8 @@ function getLibp2pOptions ({ options, config, datastore, peerInfo, peerBook }) { }, connectionManager: get(options, 'connectionManager', { - maxPeers: get(config, 'Swarm.ConnMgr.HighWater'), - minPeers: get(config, 'Swarm.ConnMgr.LowWater') + maxConnections: get(config, 'Swarm.ConnMgr.HighWater'), + minConnections: get(config, 'Swarm.ConnMgr.LowWater') }) } @@ -150,7 +149,7 @@ function getLibp2pOptions ({ options, config, datastore, peerInfo, peerBook }) { // Merge defaults with Node.js/browser/other environments options and configuration return mergeOptions( libp2pDefaults, - getEnvLibp2pOptions({ options, config, datastore, peerInfo, peerBook }), + getEnvLibp2pOptions(), libp2pOptions ) } diff --git a/src/core/components/start.js b/src/core/components/start.js index 2465be61f7..0c31349636 100644 --- a/src/core/components/start.js +++ b/src/core/components/start.js @@ -1,7 +1,6 @@ 'use strict' const Bitswap = require('ipfs-bitswap') -const PeerBook = require('peer-book') const IPNS = require('../ipns') const routingConfig = require('../ipns/routing/config') const defer = require('p-defer') @@ -33,14 +32,13 @@ module.exports = ({ const config = await repo.config.get() - const peerBook = new PeerBook() - const libp2p = Commands.legacy.libp2p({ - _options: constructorOptions, - _repo: repo, - _peerInfo: peerInfo, - _peerInfoBook: peerBook, - _print: print - }, config) + const libp2p = Commands.libp2p({ + constructorOptions, + repo, + peerInfo, + print, + config + }) await libp2p.start() diff --git a/src/core/runtime/libp2p-browser.js b/src/core/runtime/libp2p-browser.js index 7c8a921886..9b18bec8dd 100644 --- a/src/core/runtime/libp2p-browser.js +++ b/src/core/runtime/libp2p-browser.js @@ -2,36 +2,23 @@ const WS = require('libp2p-websockets') const WebRTCStar = require('libp2p-webrtc-star') -const WebSocketStarMulti = require('libp2p-websocket-star-multi') const Multiplex = require('pull-mplex') const SECIO = require('libp2p-secio') const Bootstrap = require('libp2p-bootstrap') const KadDHT = require('libp2p-kad-dht') const GossipSub = require('libp2p-gossipsub') -const multiaddr = require('multiaddr') - -module.exports = ({ peerInfo, options }) => { - const wrtcstar = new WebRTCStar({ id: peerInfo.id }) - - // this can be replaced once optional listening is supported with the below code. ref: https://github.com/libp2p/interface-transport/issues/41 - // const wsstar = new WebSocketStar({ id: _options.peerInfo.id }) - const wsstarServers = peerInfo.multiaddrs.toArray().map(String).filter(addr => addr.includes('p2p-websocket-star')) - peerInfo.multiaddrs.replace(wsstarServers.map(multiaddr), '/p2p-websocket-star') // the ws-star-multi module will replace this with the chosen ws-star servers - const wsstar = new WebSocketStarMulti({ servers: wsstarServers, id: peerInfo.id, ignore_no_online: !wsstarServers.length || options.wsStarIgnoreErrors }) +module.exports = () => { return { - switch: { - denyTTL: 2 * 60 * 1e3, // 2 minute base - denyAttempts: 5, // back off 5 times - maxParallelDials: 100, - maxColdCalls: 25, - dialTimeout: 20e3 + dialer: { + maxParallelDials: 150, // 150 total parallel multiaddr dials + maxDialsPerPeer: 4, // Allow 4 multiaddrs to be dialed per peer in parallel + dialTimeout: 10e3 // 10 second dial timeout per peer dial }, modules: { transport: [ WS, - wrtcstar, - wsstar + WebRTCStar ], streamMuxer: [ Multiplex diff --git a/src/core/runtime/libp2p-nodejs.js b/src/core/runtime/libp2p-nodejs.js index fb8d488752..9ac35ccca5 100644 --- a/src/core/runtime/libp2p-nodejs.js +++ b/src/core/runtime/libp2p-nodejs.js @@ -3,28 +3,18 @@ const TCP = require('libp2p-tcp') const MulticastDNS = require('libp2p-mdns') const WS = require('libp2p-websockets') -const WebSocketStarMulti = require('libp2p-websocket-star-multi') const Bootstrap = require('libp2p-bootstrap') const KadDHT = require('libp2p-kad-dht') const GossipSub = require('libp2p-gossipsub') const Multiplex = require('pull-mplex') const SECIO = require('libp2p-secio') -const multiaddr = require('multiaddr') - -module.exports = ({ peerInfo, options }) => { - // this can be replaced once optional listening is supported with the below code. ref: https://github.com/libp2p/interface-transport/issues/41 - // const wsstar = new WebSocketStar({ id: _options.peerInfo.id }) - const wsstarServers = peerInfo.multiaddrs.toArray().map(String).filter(addr => addr.includes('p2p-websocket-star')) - peerInfo.multiaddrs.replace(wsstarServers.map(multiaddr), '/p2p-websocket-star') // the ws-star-multi module will replace this with the chosen ws-star servers - const wsstar = new WebSocketStarMulti({ servers: wsstarServers, id: peerInfo.id, ignore_no_online: !wsstarServers.length || options.wsStarIgnoreErrors }) +module.exports = () => { return { - switch: { - denyTTL: 2 * 60 * 1e3, // 2 minute base - denyAttempts: 5, // back off 5 times - maxParallelDials: 150, - maxColdCalls: 50, - dialTimeout: 10e3 // Be strict with dial time + dialer: { + maxParallelDials: 150, // 150 total parallel multiaddr dials + maxDialsPerPeer: 4, // Allow 4 multiaddrs to be dialed per peer in parallel + dialTimeout: 10e3 // 10 second dial timeout per peer dial }, modules: { transport: [ From f1da753d1e6952f36666ff42a428bf45cce75e0a Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Wed, 18 Dec 2019 09:36:39 -0800 Subject: [PATCH 3/3] fix: misc fixes --- src/core/components/libp2p.js | 43 +++++++++++------------------- src/core/components/start.js | 2 +- src/core/runtime/libp2p-browser.js | 17 +++++++++--- src/core/runtime/libp2p-nodejs.js | 15 ++++++----- 4 files changed, 39 insertions(+), 38 deletions(-) diff --git a/src/core/components/libp2p.js b/src/core/components/libp2p.js index 348c2509f7..6d18081d63 100644 --- a/src/core/components/libp2p.js +++ b/src/core/components/libp2p.js @@ -3,21 +3,20 @@ const get = require('dlv') const mergeOptions = require('merge-options') const errCode = require('err-code') -const ipnsUtils = require('../ipns/routing/utils') const multiaddr = require('multiaddr') const DelegatedPeerRouter = require('libp2p-delegated-peer-routing') const DelegatedContentRouter = require('libp2p-delegated-content-routing') const PubsubRouters = require('../runtime/libp2p-pubsub-routers-nodejs') module.exports = ({ - constructorOptions, + constructorOptions: options, peerInfo, repo, print, config }) => { const { datastore } = repo - const libp2pOptions = getLibp2pOptions({ options: constructorOptions, config, datastore, peerInfo }) + const libp2pOptions = getLibp2pOptions({ options, config, datastore, peerInfo }) let libp2p @@ -68,10 +67,6 @@ function getLibp2pOptions ({ options, config, datastore, peerInfo }) { const getPubsubRouter = () => { let router = get(config, 'Pubsub.Router', 'gossipsub') - if (!router) { - router = 'gossipsub' - } - if (!PubsubRouters[router]) { throw errCode(new Error(`Router unavailable. Configure libp2p.modules.pubsub to use the ${router} router.`), 'ERR_NOT_SUPPORTED') } @@ -88,6 +83,7 @@ function getLibp2pOptions ({ options, config, datastore, peerInfo }) { } } + const bootstrapList = get(options, 'config.Bootstrap', get(config, 'Bootstrap', [])) const libp2pOptions = { modules: { pubsub: getPubsubRouter() @@ -103,8 +99,7 @@ function getLibp2pOptions ({ options, config, datastore, peerInfo }) { get(config, 'Discovery.webRTCStar.Enabled', true)) }, bootstrap: { - list: get(options, 'config.Bootstrap', - get(config, 'Bootstrap', [])) + list: bootstrapList } }, relay: { @@ -118,28 +113,16 @@ function getLibp2pOptions ({ options, config, datastore, peerInfo }) { } }, dht: { - kBucketSize: get(options, 'dht.kBucketSize', 20), - // enabled: !get(options, 'offline', false), // disable if offline, on by default - enabled: false, - randomWalk: { - enabled: false // disabled waiting for https://github.com/libp2p/js-libp2p-kad-dht/issues/86 - }, - validators: { - ipns: ipnsUtils.validator - }, - selectors: { - ipns: ipnsUtils.selector - } + kBucketSize: get(options, 'dht.kBucketSize', 20) }, pubsub: { enabled: get(config, 'Pubsub.Enabled', true) } }, - connectionManager: get(options, 'connectionManager', - { - maxConnections: get(config, 'Swarm.ConnMgr.HighWater'), - minConnections: get(config, 'Swarm.ConnMgr.LowWater') - }) + connectionManager: get(options, 'connectionManager', { + maxConnections: get(config, 'Swarm.ConnMgr.HighWater'), + minConnections: get(config, 'Swarm.ConnMgr.LowWater') + }) } // Required inline to reduce startup time @@ -147,9 +130,15 @@ function getLibp2pOptions ({ options, config, datastore, peerInfo }) { const getEnvLibp2pOptions = require('../runtime/libp2p-nodejs') // Merge defaults with Node.js/browser/other environments options and configuration - return mergeOptions( + const libp2pConfig = mergeOptions( libp2pDefaults, getEnvLibp2pOptions(), libp2pOptions ) + + if (bootstrapList.length > 0) { + libp2pConfig.modules.peerDiscovery.push(require('libp2p-bootstrap')) + } + + return libp2pConfig } diff --git a/src/core/components/start.js b/src/core/components/start.js index 0c31349636..8d7a4a3073 100644 --- a/src/core/components/start.js +++ b/src/core/components/start.js @@ -32,7 +32,7 @@ module.exports = ({ const config = await repo.config.get() - const libp2p = Commands.libp2p({ + const libp2p = Components.libp2p({ constructorOptions, repo, peerInfo, diff --git a/src/core/runtime/libp2p-browser.js b/src/core/runtime/libp2p-browser.js index 9b18bec8dd..e69ce57d4e 100644 --- a/src/core/runtime/libp2p-browser.js +++ b/src/core/runtime/libp2p-browser.js @@ -4,9 +4,9 @@ const WS = require('libp2p-websockets') const WebRTCStar = require('libp2p-webrtc-star') const Multiplex = require('pull-mplex') const SECIO = require('libp2p-secio') -const Bootstrap = require('libp2p-bootstrap') const KadDHT = require('libp2p-kad-dht') const GossipSub = require('libp2p-gossipsub') +const ipnsUtils = require('../ipns/routing/utils') module.exports = () => { return { @@ -28,8 +28,7 @@ module.exports = () => { ], peerDiscovery: [ wrtcstar.discovery, - wsstar.discovery, - Bootstrap + wsstar.discovery ], dht: KadDHT, pubsub: GossipSub @@ -48,7 +47,17 @@ module.exports = () => { } }, dht: { - enabled: false + kBucketSize: 20, + enabled: false, + randomWalk: { + enabled: false + }, + validators: { + ipns: ipnsUtils.validator + }, + selectors: { + ipns: ipnsUtils.selector + } }, pubsub: { enabled: true, diff --git a/src/core/runtime/libp2p-nodejs.js b/src/core/runtime/libp2p-nodejs.js index 9ac35ccca5..7f22d9d216 100644 --- a/src/core/runtime/libp2p-nodejs.js +++ b/src/core/runtime/libp2p-nodejs.js @@ -3,11 +3,11 @@ const TCP = require('libp2p-tcp') const MulticastDNS = require('libp2p-mdns') const WS = require('libp2p-websockets') -const Bootstrap = require('libp2p-bootstrap') const KadDHT = require('libp2p-kad-dht') const GossipSub = require('libp2p-gossipsub') const Multiplex = require('pull-mplex') const SECIO = require('libp2p-secio') +const ipnsUtils = require('../ipns/routing/utils') module.exports = () => { return { @@ -19,8 +19,7 @@ module.exports = () => { modules: { transport: [ TCP, - WS, - wsstar + WS ], streamMuxer: [ Multiplex @@ -29,9 +28,7 @@ module.exports = () => { SECIO ], peerDiscovery: [ - MulticastDNS, - Bootstrap, - wsstar.discovery + MulticastDNS ], dht: KadDHT, pubsub: GossipSub @@ -54,6 +51,12 @@ module.exports = () => { enabled: false, randomWalk: { enabled: false + }, + validators: { + ipns: ipnsUtils.validator + }, + selectors: { + ipns: ipnsUtils.selector } }, pubsub: {