feat: support --raw-leaves

Goes some way towards fixing ipfs/js-ipfs#1432 - will need follow up PRs for js-ipfs-mfs and js-ipfs itself (🔜).

There are three ways of importing a file we need to support and each will end up with slightly different DAG structure.

ipfs add will result in a balanced DAG with leaf nodes that are unixfs nodes of type file
ipfs files write results in a trickle DAG with leaf nodes that are unixfs nodes of type raw
ipfs add --raw-leaves and ipfs files write --raw-leaves have the balanced/trickle DAG of above, but the leaf nodes are chunks of file data not wrapped in protobufs.
In all cases above the root node is a unixfs file node with a v0 CID, unless you specify --cid-version=1.

This PR:

Changes meaning of existing rawLeaves argument. Now means the leaf node is just data - a chunk of the file, previously it was meant a unixfs node with type raw. So far the only code using this is js-ipfs-mfs so changing it shouldn't be too disruptive.
Adds a leafType option which can be file or raw - when --raw-leaves is false, this is what the unixfs leaf type will be.
Uses CIDv1 for raw leaves with the codec raw

Author: achingbrain

README.md

@@ -149,7 +149,7 @@ The input's file paths and directory structure will be preserved in the [`dag-pb
 - `onlyHash` (boolean, defaults to false): Only chunk and hash - do not write to disk
 - `hashAlg` (string): multihash hashing algorithm to use
 - `cidVersion` (integer, default 0): the CID version to use when storing the data (storage keys are based on the CID, _including_ it's version)
-- `rawLeafNodes` (boolean, defaults to false): When a file would span multiple DAGNodes, if this is true the leaf nodes will be marked as `raw` `unixfs` nodes
+- `rawLeaves` (boolean, defaults to false): When a file would span multiple DAGNodes, if this is true the leaf nodes will not be wrapped in `UnixFS` protobufs and will instead contain the raw file bytes
 
 ### Exporter
 

src/builder/builder.js

@@ -8,6 +8,7 @@ const parallel = require('async/parallel')
 const waterfall = require('async/waterfall')
 const dagPB = require('ipld-dag-pb')
 const CID = require('cids')
+const multihash = require('multihashing-async')
 
 const reduce = require('./reduce')
 
@@ -17,7 +18,9 @@ const defaultOptions = {
   chunkerOptions: {
     maxChunkSize: 262144
   },
-  rawLeafNodes: false
+  rawLeaves: false,
+  hashAlg: 'sha2-256',
+  leafType: 'file'
 }
 
 module.exports = function (createChunker, ipld, createReducer, _options) {
@@ -97,7 +100,6 @@ module.exports = function (createChunker, ipld, createReducer, _options) {
 
     let previous
     let count = 0
-    const leafType = options.rawLeafNodes ? 'raw' : 'file'
 
     pull(
       file.content,
@@ -108,30 +110,58 @@ module.exports = function (createChunker, ipld, createReducer, _options) {
         }
         return Buffer.from(chunk)
       }),
-      pull.map(buffer => new UnixFS(leafType, buffer)),
-      pull.asyncMap((fileNode, callback) => {
-        DAGNode.create(fileNode.marshal(), [], options.hashAlg, (err, node) => {
-          callback(err, { DAGNode: node, fileNode: fileNode })
+      pull.asyncMap((buffer, callback) => {
+        if (options.rawLeaves) {
+          return multihash(buffer, options.hashAlg, (error, hash) => {
+            if (error) {
+              return callback(error)
+            }
+
+            return callback(null, {
+              multihash: hash,
+              size: buffer.length,
+              leafSize: buffer.length,
+              cid: new CID(1, 'raw', hash),
+              data: buffer
+            })
+          })
+        }
+
+        const file = new UnixFS(options.leafType, buffer)
+
+        DAGNode.create(file.marshal(), [], options.hashAlg, (err, node) => {
+          let cid = new CID(0, 'dag-pb', node.multihash)
+
+          if (options.cidVersion === 1) {
+            cid = cid.toV1()
+          }
+
+          callback(err, {
+            multihash: node.multihash,
+            size: node.size,
+            leafSize: file.fileSize(),
+            cid,
+            data: node
+          })
         })
       }),
       pull.asyncMap((leaf, callback) => {
-        if (options.onlyHash) return callback(null, leaf)
-
-        let cid = new CID(leaf.DAGNode.multihash)
-
-        if (options.cidVersion === 1) {
-          cid = cid.toV1()
+        if (options.onlyHash) {
+          return callback(null, leaf)
         }
 
-        ipld.put(leaf.DAGNode, { cid }, (err) => callback(err, leaf))
+        ipld.put(leaf.data, {
+          cid: leaf.cid
+        }, (error) => callback(error, leaf))
       }),
       pull.map((leaf) => {
         return {
           path: file.path,
-          multihash: leaf.DAGNode.multihash,
-          size: leaf.DAGNode.size,
-          leafSize: leaf.fileNode.fileSize(),
-          name: ''
+          multihash: leaf.multihash,
+          size: leaf.size,
+          leafSize: leaf.leafSize,
+          name: '',
+          cid: leaf.cid
         }
       }),
       through( // mark as single node if only one single node

src/builder/reduce.js

@@ -13,7 +13,7 @@ module.exports = function (file, ipld, options) {
     if (leaves.length === 1 && leaves[0].single && options.reduceSingleLeafToSelf) {
       const leaf = leaves[0]
 
-      if (!options.rawLeafNodes) {
+      if (options.leafType === 'file' && !options.rawLeaves) {
         return callback(null, {
           path: file.path,
           multihash: leaf.multihash,
@@ -23,19 +23,22 @@ module.exports = function (file, ipld, options) {
         })
       }
 
-      // we are using raw leaf nodes, this file only has one node but it'll be marked raw
-      // so convert it back to a file node
+      // we're using raw leaf nodes so we convert the node into a UnixFS `file` node.
       return waterfall([
-        (cb) => ipld.get(new CID(leaf.multihash), cb),
+        (cb) => ipld.get(leaf.cid, cb),
         (result, cb) => {
-          const meta = UnixFS.unmarshal(result.value.data)
-          const fileNode = new UnixFS('file', meta.data)
+          const data = result.value.data
+          const fileNode = new UnixFS('file', data)
 
           DAGNode.create(fileNode.marshal(), [], options.hashAlg, (err, node) => {
             cb(err, { DAGNode: node, fileNode: fileNode })
           })
         },
         (result, cb) => {
+          if (options.onlyHash) {
+            return cb(null, result)
+          }
+
           let cid = new CID(result.DAGNode.multihash)
 
           if (options.cidVersion === 1) {
@@ -46,10 +49,11 @@ module.exports = function (file, ipld, options) {
         },
         (result, cb) => {
           cb(null, {
+            path: file.path,
             multihash: result.DAGNode.multihash,
             size: result.DAGNode.size,
             leafSize: result.fileNode.fileSize(),
-            name: ''
+            name: leaf.name
           })
         }
       ], callback)
@@ -61,13 +65,26 @@ module.exports = function (file, ipld, options) {
     const links = leaves.map((leaf) => {
       f.addBlockSize(leaf.leafSize)
 
-      return new DAGLink(leaf.name, leaf.size, leaf.multihash)
+      let cid = leaf.cid
+
+      if (!cid) {
+        // we are an intermediate node
+        cid = new CID(0, 'dag-pb', leaf.multihash)
+
+        if (options.cidVersion === 1) {
+          cid = cid.toV1()
+        }
+      }
+
+      return new DAGLink(leaf.name, leaf.size, cid.buffer)
     })
 
     waterfall([
       (cb) => DAGNode.create(f.marshal(), links, options.hashAlg, cb),
       (node, cb) => {
-        if (options.onlyHash) return cb(null, node)
+        if (options.onlyHash) {
+          return cb(null, node)
+        }
 
         let cid = new CID(node.multihash)
 
@@ -83,10 +100,16 @@ module.exports = function (file, ipld, options) {
         return // early
       }
 
+      let cid = new CID(0, 'dag-pb', node.multihash)
+
+      if (options.cidVersion === 1) {
+        cid = cid.toV1()
+      }
+
       const root = {
         name: '',
         path: file.path,
-        multihash: node.multihash,
+        multihash: cid.buffer,
         size: node.size,
         leafSize: f.fileSize()
       }

src/exporter/file.js

@@ -63,6 +63,11 @@ function streamBytes (dag, node, fileSize, offset, length) {
 
   function getData ({ node, start }) {
     try {
+      if (Buffer.isBuffer(node)) {
+        // this is a raw node
+        return extractDataFromBlock(node, start, offset, end)
+      }
+
       const file = UnixFS.unmarshal(node.data)
 
       if (!file.data) {
@@ -80,6 +85,11 @@ function streamBytes (dag, node, fileSize, offset, length) {
   let streamPosition = 0
 
   function visitor ({ node }) {
+    if (Buffer.isBuffer(node)) {
+      // this is a raw node
+      return pull.empty()
+    }
+
     const file = UnixFS.unmarshal(node.data)
     const nodeHasData = Boolean(file.data && file.data.length)
 

src/importer/index.js

@@ -15,11 +15,29 @@ const chunkers = {
 
 const defaultOptions = {
   chunker: 'fixed',
-  rawLeafNodes: false
+  rawLeaves: false,
+  hashOnly: false,
+  cidVersion: 0,
+  hash: null,
+  leafType: 'file',
+  hashAlg: 'sha2-256'
 }
 
 module.exports = function (ipld, _options) {
   const options = Object.assign({}, defaultOptions, _options)
+
+  if (options.cidVersion > 0 && _options.rawLeaves === undefined) {
+    // if the cid version is 1 or above, use raw leaves as this is
+    // what go does.
+    options.rawLeaves = true
+  }
+
+  if (_options && _options.hash !== undefined && _options.rawLeaves === undefined) {
+    // if a non-default hash alg has been specified, use raw leaves as this is
+    // what go does.
+    options.rawLeaves = true
+  }
+
   const Chunker = chunkers[options.chunker]
   assert(Chunker, 'Unknkown chunker named ' + options.chunker)
 

test/exporter.js

@@ -422,6 +422,37 @@ module.exports = (repo) => {
       )
     })
 
+    it('exports a large file > 5mb imported with raw leaves', function (done) {
+      this.timeout(30 * 1000)
+
+      pull(
+        pull.values([{
+          path: '200Bytes.txt',
+          content: pull.values([bigFile])
+        }]),
+        importer(ipld, {
+          rawLeaves: true
+        }),
+        pull.collect(collected)
+      )
+
+      function collected (err, files) {
+        expect(err).to.not.exist()
+        expect(files.length).to.equal(1)
+
+        pull(
+          exporter(files[0].multihash, ipld),
+          pull.collect((err, files) => {
+            expect(err).to.not.exist()
+
+            expect(bs58.encode(files[0].hash)).to.equal('QmQLTvhjmSa7657mKdSfTjxFBdwxmK8n9tZC9Xdp9DtxWY')
+
+            fileEql(files[0], bigFile, done)
+          })
+        )
+      }
+    })
+
     it('returns an empty stream for dir', (done) => {
       const hash = 'QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn'
 

test/helpers/collect-leaf-cids.js

@@ -0,0 +1,32 @@
+'use strict'
+
+const pull = require('pull-stream')
+const traverse = require('pull-traverse')
+const CID = require('cids')
+
+module.exports = (ipld, multihash, callback) => {
+  pull(
+    traverse.depthFirst(new CID(multihash), (cid) => {
+      return pull(
+        pull.values([cid]),
+        pull.asyncMap((cid, callback) => {
+          ipld.get(cid, (error, result) => {
+            callback(error, !error && result.value)
+          })
+        }),
+        pull.asyncMap((node, callback) => {
+          if (!node.links) {
+            return callback()
+          }
+
+          return callback(
+            null, node.links.map(link => new CID(link.multihash))
+          )
+        }),
+        pull.filter(Boolean),
+        pull.flatten()
+      )
+    }),
+    pull.collect(callback)
+  )
+}