diff --git a/README.md b/README.md index 76011e8a..b3a8032d 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,7 @@ The input's file paths and directory structure will be preserved in the [`dag-pb - `hamt` (object): the options for the HAMT sharded directory builder - bits (positive integer, defaults to `8`): the number of bits at each bucket of the HAMT - `progress` (function): a function that will be called with the byte length of chunks as a file is added to ipfs. +- `onlyHash` (boolean, defaults to false): Only chunk and hash - do not write to disk ### Exporter diff --git a/package.json b/package.json index 433303dd..fc1acb2d 100644 --- a/package.json +++ b/package.json @@ -81,6 +81,7 @@ }, "contributors": [ "Alan Shaw ", + "Bernard Mordan ", "David Dias ", "Francisco Baio Dias ", "Friedel Ziegelmayer ", diff --git a/src/builder/builder.js b/src/builder/builder.js index 1a5852f8..126e89e0 100644 --- a/src/builder/builder.js +++ b/src/builder/builder.js @@ -57,9 +57,12 @@ module.exports = function (createChunker, ipldResolver, createReducer, _options) // 2. write it to the dag store const d = new UnixFS('directory') + waterfall([ (cb) => DAGNode.create(d.marshal(), [], options.hashAlg, cb), (node, cb) => { + if (options.onlyHash) return cb(null, node) + ipldResolver.put(node, { cid: new CID(node.multihash) }, (err) => cb(err, node)) @@ -106,6 +109,8 @@ module.exports = function (createChunker, ipldResolver, createReducer, _options) }) }), pull.asyncMap((leaf, callback) => { + if (options.onlyHash) return callback(null, leaf) + ipldResolver.put(leaf.DAGNode, { cid: new CID(leaf.DAGNode.multihash) }, (err) => callback(err, leaf) diff --git a/src/builder/reduce.js b/src/builder/reduce.js index 83d43ac7..3b7ea1c9 100644 --- a/src/builder/reduce.js +++ b/src/builder/reduce.js @@ -34,6 +34,8 @@ module.exports = function (file, ipldResolver, options) { waterfall([ (cb) => DAGNode.create(f.marshal(), links, cb), (node, cb) => { + if (options.onlyHash) return cb(null, node) + ipldResolver.put(node, { cid: new CID(node.multihash) }, (err) => cb(err, node)) diff --git a/src/importer/dir-flat.js b/src/importer/dir-flat.js index ead6f5fe..59751c12 100644 --- a/src/importer/dir-flat.js +++ b/src/importer/dir-flat.js @@ -10,10 +10,9 @@ const DAGNode = dagPB.DAGNode const Dir = require('./dir') class DirFlat extends Dir { - constructor (props) { - super() + constructor (props, _options) { + super(props, _options) this._children = {} - Object.assign(this, props) } put (name, value, callback) { @@ -57,10 +56,13 @@ class DirFlat extends Dir { }) const dir = new UnixFS('directory') + waterfall( [ (callback) => DAGNode.create(dir.marshal(), links, callback), (node, callback) => { + if (this._options.onlyHash) return callback(null, node) + ipldResolver.put( node, { @@ -86,6 +88,6 @@ class DirFlat extends Dir { module.exports = createDirFlat -function createDirFlat (props) { - return new DirFlat(props) +function createDirFlat (props, _options) { + return new DirFlat(props, _options) } diff --git a/src/importer/dir-sharded.js b/src/importer/dir-sharded.js index 676361c6..f120614b 100644 --- a/src/importer/dir-sharded.js +++ b/src/importer/dir-sharded.js @@ -41,11 +41,9 @@ const defaultOptions = { class DirSharded extends Dir { constructor (props, _options) { - super() const options = Object.assign({}, defaultOptions, _options) - this._options = options + super(props, options) this._bucket = Bucket(options) - Object.assign(this, props) } put (name, value, callback) { @@ -87,8 +85,8 @@ class DirSharded extends Dir { module.exports = createDirSharded -function createDirSharded (props) { - return new DirSharded(props) +function createDirSharded (props, _options) { + return new DirSharded(props, _options) } function flush (options, bucket, path, ipldResolver, source, callback) { @@ -148,6 +146,8 @@ function flush (options, bucket, path, ipldResolver, source, callback) { [ (callback) => DAGNode.create(dir.marshal(), links, callback), (node, callback) => { + if (options.onlyHash) return callback(null, node) + ipldResolver.put( node, { diff --git a/src/importer/dir.js b/src/importer/dir.js index cd1a0e51..fda1f7bf 100644 --- a/src/importer/dir.js +++ b/src/importer/dir.js @@ -1,4 +1,8 @@ 'use strict' module.exports = class Dir { + constructor (props, _options) { + this._options = _options || {} + Object.assign(this, props) + } } diff --git a/src/importer/flat-to-shard.js b/src/importer/flat-to-shard.js index 1b525e98..3bae8b4f 100644 --- a/src/importer/flat-to-shard.js +++ b/src/importer/flat-to-shard.js @@ -5,8 +5,8 @@ const DirSharded = require('./dir-sharded') module.exports = flatToShard -function flatToShard (child, dir, threshold, callback) { - maybeFlatToShardOne(dir, threshold, (err, newDir) => { +function flatToShard (child, dir, threshold, options, callback) { + maybeFlatToShardOne(dir, threshold, options, (err, newDir) => { if (err) { callback(err) return // early @@ -27,7 +27,7 @@ function flatToShard (child, dir, threshold, callback) { }, (callback) => { if (parent) { - flatToShard(newDir, parent, threshold, callback) + flatToShard(newDir, parent, threshold, options, callback) } else { callback(null, newDir) } @@ -40,15 +40,15 @@ function flatToShard (child, dir, threshold, callback) { }) } -function maybeFlatToShardOne (dir, threshold, callback) { +function maybeFlatToShardOne (dir, threshold, options, callback) { if (dir.flat && dir.directChildrenCount() >= threshold) { - definitelyShardOne(dir, callback) + definitelyShardOne(dir, options, callback) } else { callback(null, dir) } } -function definitelyShardOne (oldDir, callback) { +function definitelyShardOne (oldDir, options, callback) { const newDir = DirSharded({ root: oldDir.root, dir: true, @@ -57,7 +57,7 @@ function definitelyShardOne (oldDir, callback) { path: oldDir.path, dirty: oldDir.dirty, flat: false - }) + }, options) oldDir.eachChildSeries( (key, value, callback) => { diff --git a/src/importer/index.js b/src/importer/index.js index 8ae157a7..f1dbdd8b 100644 --- a/src/importer/index.js +++ b/src/importer/index.js @@ -64,7 +64,7 @@ module.exports = function (ipldResolver, _options) { return node }), treeBuilderStream - ) + ) return { sink: entry.sink, diff --git a/src/importer/tree-builder.js b/src/importer/tree-builder.js index 22c17906..c04cb0ac 100644 --- a/src/importer/tree-builder.js +++ b/src/importer/tree-builder.js @@ -14,14 +14,14 @@ module.exports = createTreeBuilder const defaultOptions = { wrap: false, - shardSplitThreshold: 1000 + shardSplitThreshold: 1000, + onlyHash: false } function createTreeBuilder (ipldResolver, _options) { const options = Object.assign({}, defaultOptions, _options) const queue = createQueue(consumeQueue, 1) - // returned stream let stream = createStream() @@ -32,7 +32,7 @@ function createTreeBuilder (ipldResolver, _options) { dir: true, dirty: false, flat: true - }) + }, options) return { flush: flushRoot, @@ -101,7 +101,6 @@ function createTreeBuilder (ipldResolver, _options) { currentPath += '/' } currentPath += pathElem - const last = (index === lastIndex) parent.dirty = true parent.multihash = null @@ -110,7 +109,7 @@ function createTreeBuilder (ipldResolver, _options) { if (last) { waterfall([ (callback) => parent.put(pathElem, elem, callback), - (callback) => flatToShard(null, parent, options.shardSplitThreshold, callback), + (callback) => flatToShard(null, parent, options.shardSplitThreshold, options, callback), (newRoot, callback) => { tree = newRoot callback() @@ -131,7 +130,7 @@ function createTreeBuilder (ipldResolver, _options) { path: currentPath, dirty: true, flat: true - }) + }, options) } const parentDir = parent parent = dir diff --git a/test/browser.js b/test/browser.js index 4d3e10a0..c2bc0c2d 100644 --- a/test/browser.js +++ b/test/browser.js @@ -52,4 +52,5 @@ describe('IPFS data importing tests on the Browser', function () { require('./test-hash-parity-with-go-ipfs')(repo) require('./test-nested-dir-import-export')(repo) require('./test-dirbuilder-sharding')(repo) + require('./test-builder-only-hash')(repo) }) diff --git a/test/node.js b/test/node.js index 01c7c0c5..4790b05b 100644 --- a/test/node.js +++ b/test/node.js @@ -53,4 +53,5 @@ describe('IPFS UnixFS Engine', () => { require('./test-nested-dir-import-export')(repo) require('./test-dirbuilder-sharding')(repo) require('./test-dag-api') + require('./test-builder-only-hash')(repo) }) diff --git a/test/test-builder-only-hash.js b/test/test-builder-only-hash.js new file mode 100644 index 00000000..eb48de8e --- /dev/null +++ b/test/test-builder-only-hash.js @@ -0,0 +1,53 @@ +/* eslint-env mocha */ +'use strict' + +const chai = require('chai') +chai.use(require('dirty-chai')) +const expect = chai.expect +const BlockService = require('ipfs-block-service') +const pull = require('pull-stream') +const IPLDResolver = require('ipld-resolver') +const CID = require('cids') +const createBuilder = require('../src/builder') +const FixedSizeChunker = require('../src/chunker/fixed-size') + +module.exports = (repo) => { + describe('builder', () => { + let ipldResolver + + before(() => { + const bs = new BlockService(repo) + ipldResolver = new IPLDResolver(bs) + }) + + it('will only chunk and hash if passed an "onlyHash" option', (done) => { + const onCollected = (err, nodes) => { + if (err) return done(err) + + const node = nodes[0] + expect(node).to.exist() + + ipldResolver.get(new CID(node.multihash), (err, res) => { + expect(err).to.exist() + done() + }) + } + + const content = String(Math.random() + Date.now()) + const inputFile = { + path: content + '.txt', + content: Buffer.from(content) + } + + const options = { + onlyHash: true + } + + pull( + pull.values([inputFile]), + createBuilder(FixedSizeChunker, ipldResolver, options), + pull.collect(onCollected) + ) + }) + }) +} diff --git a/test/test-importer.js b/test/test-importer.js index 3e40cc64..40afab90 100644 --- a/test/test-importer.js +++ b/test/test-importer.js @@ -11,6 +11,7 @@ const sinon = require('sinon') const BlockService = require('ipfs-block-service') const pull = require('pull-stream') const mh = require('multihashes') +const CID = require('cids') const IPLDResolver = require('ipld-resolver') const loadFixture = require('aegir/fixtures') @@ -419,6 +420,36 @@ module.exports = (repo) => { } }) + it('will not write to disk if passed "onlyHash" option', (done) => { + const content = String(Math.random() + Date.now()) + const inputFile = { + path: content + '.txt', + content: Buffer.from(content) + } + + const options = { + onlyHash: true + } + + const onCollected = (err, files) => { + if (err) return done(err) + + const file = files[0] + expect(file).to.exist() + + ipldResolver.get(new CID(file.multihash), (err, res) => { + expect(err).to.exist() + done() + }) + } + + pull( + pull.values([inputFile]), + importer(ipldResolver, options), + pull.collect(onCollected) + ) + }) + it('will call an optional progress function', (done) => { options.progress = sinon.spy()