diff --git a/package.json b/package.json index be5c0cfef3..4048f7e3b7 100644 --- a/package.json +++ b/package.json @@ -92,9 +92,12 @@ "async": "^2.5.0", "bl": "^1.2.1", "boom": "^5.2.0", + "byteman": "^1.3.5", "cids": "^0.5.1", "debug": "^3.0.0", + "duplexify": "^3.5.1", "fsm-event": "^2.1.0", + "get-folder-size": "^1.0.0", "glob": "^7.1.2", "hapi": "^16.5.2", "hapi-set-header": "^1.0.2", @@ -130,11 +133,14 @@ "mkdirp": "^0.5.1", "multiaddr": "^2.3.0", "multihashes": "~0.4.5", + "multiplex": "^6.7.0", + "nan": "^2.7.0", "once": "^1.4.0", "path-exists": "^3.0.0", "peer-book": "^0.5.0", "peer-id": "^0.9.0", "peer-info": "^0.10.0", + "progress": "^2.0.0", "promisify-es6": "^1.0.3", "pull-file": "^1.0.0", "pull-paramap": "^1.2.2", @@ -151,6 +157,7 @@ "temp": "^0.8.3", "through2": "^2.0.3", "update-notifier": "^2.2.0", + "webrtcsupport": "^2.2.0", "yargs": "8.0.2" }, "contributors": [ @@ -204,4 +211,4 @@ "tcme ", "ᴠɪᴄᴛᴏʀ ʙᴊᴇʟᴋʜᴏʟᴍ " ] -} \ No newline at end of file +} diff --git a/src/cli/bin.js b/src/cli/bin.js index 8da1473cdd..fc3858ccf6 100755 --- a/src/cli/bin.js +++ b/src/cli/bin.js @@ -15,12 +15,6 @@ updateNotifier({ }).notify() const cli = yargs - .option('q', { - alias: 'quiet', - desc: 'suppress output', - type: 'boolean', - coerce: (quiet) => { if (quiet) { utils.disablePrinting() } } - }) .commandDir('commands') .demandCommand(1) .fail((msg, err, yargs) => { diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index 9e7d192106..0f130cba60 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -2,12 +2,16 @@ const fs = require('fs') const path = require('path') +const async = require('async') const glob = require('glob') const sortBy = require('lodash.sortby') const pull = require('pull-stream') const paramap = require('pull-paramap') const zip = require('pull-zip') const toPull = require('stream-to-pull-stream') +const Progress = require('progress') +const getFolderSize = require('get-folder-size') +const byteman = require('byteman') const utils = require('../../utils') const print = require('../../utils').print @@ -37,6 +41,27 @@ function checkPath (inPath, recursive) { return inPath } +function getTotalBytes (path, recursive, cb) { + if (recursive) { + getFolderSize(path, cb) + } else { + fs.stat(path, (err, stat) => cb(err, stat.size)) + } +} + +function createProgressBar (totalBytes) { + const total = byteman(totalBytes, 2, 'MB') + const barFormat = `:progress / ${total} [:bar] :percent :etas` + + // 16 MB / 34 MB [=========== ] 48% 5.8s // + return new Progress(barFormat, { + incomplete: ' ', + clear: true, + stream: process.stdout, + total: totalBytes + }) +} + function addPipeline (index, addStream, list, wrapWithDirectory) { pull( zip( @@ -89,6 +114,12 @@ module.exports = { describe: 'Add a file to IPFS using the UnixFS data format', builder: { + progress: { + alias: 'p', + type: 'boolean', + default: true, + describe: 'Stream progress data' + }, recursive: { alias: 'r', type: 'boolean', @@ -116,46 +147,69 @@ module.exports = { }, handler (argv) { - const inPath = checkPath(argv.file, argv.recursive) + const { + file, + recursive, + progress, + wrapWithDirectory, + trickle, + enableShardingExperiment, + shardSplitThreshold + } = argv + + const inPath = checkPath(file, recursive) const index = inPath.lastIndexOf('/') + 1 + const options = { - strategy: argv.trickle ? 'trickle' : 'balanced', - shardSplitThreshold: argv.enableShardingExperiment ? argv.shardSplitThreshold : Infinity + strategy: trickle ? 'trickle' : 'balanced', + shardSplitThreshold: enableShardingExperiment ? shardSplitThreshold : Infinity } - if (argv.enableShardingExperiment && utils.isDaemonOn()) { + if (enableShardingExperiment && utils.isDaemonOn()) { throw new Error('Error: Enabling the sharding experiment should be done on the daemon') } - const ipfs = argv.ipfs - // TODO: revist when interface-ipfs-core exposes pull-streams - let createAddStream = (cb) => { - ipfs.files.createAddStream(options, (err, stream) => { - cb(err, err ? null : toPull.transform(stream)) - }) - } + const ipfs = argv.ipfs - if (typeof ipfs.files.createAddPullStream === 'function') { - createAddStream = (cb) => { - cb(null, ipfs.files.createAddPullStream(options)) - } - } + let list = [] + let currentBytes = 0 + + async.waterfall([ + (next) => glob(path.join(inPath, '/**/*'), next), + (globResult, next) => { + list = globResult.length === 0 ? [inPath] : globResult + + getTotalBytes(inPath, recursive, next) + }, + (totalBytes, next) => { + if (progress) { + const bar = createProgressBar(totalBytes) + options.progress = function (byteLength) { + currentBytes += byteLength + bar.tick(byteLength, {progress: byteman(currentBytes, 2, 'MB')}) + } + } - createAddStream((err, addStream) => { - if (err) { - throw err - } + // TODO: revist when interface-ipfs-core exposes pull-streams - glob(path.join(inPath, '/**/*'), (err, list) => { - if (err) { - throw err + let createAddStream = (cb) => { + ipfs.files.createAddStream(options, (err, stream) => { + cb(err, err ? null : toPull.transform(stream)) + }) } - if (list.length === 0) { - list = [inPath] + + if (typeof ipfs.files.createAddPullStream === 'function') { + createAddStream = (cb) => { + cb(null, ipfs.files.createAddPullStream(options)) + } } - addPipeline(index, addStream, list, argv.wrapWithDirectory) - }) + createAddStream(next) + } + ], (err, addStream) => { + if (err) throw err + + addPipeline(index, addStream, list, wrapWithDirectory) }) } } diff --git a/src/core/components/files.js b/src/core/components/files.js index 3d650e74bc..df64c6c47f 100644 --- a/src/core/components/files.js +++ b/src/core/components/files.js @@ -19,7 +19,6 @@ module.exports = function files (self) { const opts = Object.assign({}, { shardSplitThreshold: self._options.EXPERIMENTAL.sharding ? 1000 : Infinity }, options) - return pull( pull.map(normalizeContent), pull.flatten(), diff --git a/test/cli/files.js b/test/cli/files.js index 7dc01599c6..70792cec50 100644 --- a/test/cli/files.js +++ b/test/cli/files.js @@ -8,7 +8,7 @@ const compareDir = require('dir-compare').compareSync const rimraf = require('rimraf').sync const runOnAndOff = require('../utils/on-and-off') -describe('files', () => runOnAndOff((thing) => { +describe.only('files', () => runOnAndOff((thing) => { let ipfs const readme = fs.readFileSync(path.join(process.cwd(), '/src/init-files/init-docs/readme')) .toString('utf-8') @@ -128,12 +128,13 @@ describe('files', () => runOnAndOff((thing) => { }) it('add and wrap with a directory', () => { - return ipfs('add -w src/init-files/init-docs/readme').then((out) => { - expect(out).to.be.eql([ - 'added QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB readme', - 'added QmapdaVPjHXdcswef82FnGQUauMNpk9xYFkLDZKgAxhMwq' - ].join('\n') + '\n') - }) + return ipfs('add -w src/init-files/init-docs/readme') + .then((out) => { + expect(out).to.be.eql([ + 'added QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB readme', + 'added QmapdaVPjHXdcswef82FnGQUauMNpk9xYFkLDZKgAxhMwq' + ].join('\n') + '\n') + }) }) it('cat', () => {