feat: add mfs implementation

Author: achingbrain

README.md

@@ -233,7 +233,7 @@ Creates and returns an instance of an IPFS node. Use the `options` argument to s
         - `hop` (object)
             - `enabled` (boolean): Make this node a relay (other nodes can connect *through* it). (Default: `false`)
             - `active` (boolean): Make this an *active* relay node. Active relay nodes will attempt to dial a destination peer even if that peer is not yet connected to the relay. (Default: `false`)
-            
+
 - `config` (object) Modify the default IPFS node config. Find the Node.js defaults at [`src/core/runtime/config-nodejs.js`](https://github.com/ipfs/js-ipfs/tree/master/src/core/runtime/config-nodejs.js) and the browser defaults at [`src/core/runtime/config-browser.js`](https://github.com/ipfs/js-ipfs/tree/master/src/core/runtime/config-browser.js). This object will be *merged* with the default config; it will not replace it.
 
 - `libp2p` (object) add custom modules to the libp2p stack of your node

examples/circuit-relaying/package.json

@@ -18,7 +18,7 @@
     "ipfs-pubsub-room": "~0.3.0"
   },
   "devDependencies": {
-    "aegir": "^13.0.5",
+    "aegir": "^14.0.0",
     "http-server": "~0.10.0",
     "ipfs-css": "~0.2.0",
     "parcel-bundler": "^1.6.2",

package.json

@@ -61,11 +61,12 @@
   },
   "homepage": "https://github.com/ipfs/js-ipfs#readme",
   "devDependencies": {
-    "aegir": "^13.1.0",
+    "aegir": "^14.0.0",
     "buffer-loader": "~0.0.1",
     "chai": "^4.1.2",
     "delay": "^2.0.0",
     "detect-node": "^2.0.3",
+    "detect-webworker": "^1.0.0",
     "dir-compare": "^1.4.0",
     "dirty-chai": "^2.0.1",
     "eslint-plugin-react": "^7.7.0",
@@ -111,7 +112,8 @@
     "ipfs-bitswap": "~0.20.0",
     "ipfs-block": "~0.7.1",
     "ipfs-block-service": "~0.14.0",
-    "ipfs-http-response": "^0.1.2",
+    "ipfs-http-response": "~0.1.2",
+    "ipfs-mfs": "~0.0.4",
     "ipfs-multipart": "~0.1.0",
     "ipfs-repo": "~0.22.1",
     "ipfs-unixfs": "~0.1.15",
@@ -175,7 +177,8 @@
     "through2": "^2.0.3",
     "update-notifier": "^2.5.0",
     "yargs": "^11.0.0",
-    "yargs-parser": "^10.0.0"
+    "yargs-parser": "^10.0.0",
+    "yargs-promise": "^1.1.0"
   },
   "optionalDependencies": {
     "prom-client": "^11.0.0",

src/cli/bin.js

@@ -2,13 +2,14 @@
 
 'use strict'
 
+const YargsPromise = require('yargs-promise')
 const yargs = require('yargs')
 const updateNotifier = require('update-notifier')
 const readPkgUp = require('read-pkg-up')
-const fs = require('fs')
-const path = require('path')
 const utils = require('./utils')
 const print = utils.print
+const mfs = require('ipfs-mfs/cli')
+const debug = require('debug')('ipfs:cli')
 
 const pkg = readPkgUp.sync({cwd: __dirname}).pkg
 updateNotifier({
@@ -18,10 +19,6 @@ updateNotifier({
 
 const args = process.argv.slice(2)
 
-// Determine if the first argument is a sub-system command
-const commandNames = fs.readdirSync(path.join(__dirname, 'commands'))
-const isCommand = commandNames.includes(`${args[0]}.js`)
-
 const cli = yargs
   .option('silent', {
     desc: 'Write no output',
@@ -34,14 +31,6 @@ const cli = yargs
     type: 'string',
     default: ''
   })
-  .commandDir('commands', {
-    // Only include the commands for the sub-system we're using, or include all
-    // if no sub-system command has been passed.
-    include (path, filename) {
-      if (!isCommand) return true
-      return `${args[0]}.js` === filename
-    }
-  })
   .epilog(utils.ipfsPathHelp)
   .demandCommand(1)
   .fail((msg, err, yargs) => {
@@ -56,27 +45,15 @@ const cli = yargs
     yargs.showHelp()
   })
 
-// If not a sub-system command then load the top level aliases
-if (!isCommand) {
-  // NOTE: This creates an alias of
-  // `jsipfs files {add, get, cat}` to `jsipfs {add, get, cat}`.
-  // This will stay until https://github.com/ipfs/specs/issues/98 is resolved.
-  const addCmd = require('./commands/files/add')
-  const catCmd = require('./commands/files/cat')
-  const getCmd = require('./commands/files/get')
-  const aliases = [addCmd, catCmd, getCmd]
-  aliases.forEach((alias) => {
-    cli.command(alias.command, alias.describe, alias.builder, alias.handler)
-  })
-}
-
 // Need to skip to avoid locking as these commands
 // don't require a daemon
 if (args[0] === 'daemon' || args[0] === 'init') {
   cli
     .help()
     .strict()
     .completion()
+    .command(require('./commands/daemon'))
+    .command(require('./commands/init'))
     .parse(args)
 } else {
   // here we have to make a separate yargs instance with
@@ -86,19 +63,54 @@ if (args[0] === 'daemon' || args[0] === 'init') {
     if (err) {
       throw err
     }
+
     utils.getIPFS(argv, (err, ipfs, cleanup) => {
-      if (err) { throw err }
+      if (err) {
+        throw err
+      }
+
+      // add mfs commands
+      mfs(cli)
+
+      // NOTE: This creates an alias of
+      // `jsipfs files {add, get, cat}` to `jsipfs {add, get, cat}`.
+      // This will stay until https://github.com/ipfs/specs/issues/98 is resolved.
+      const addCmd = require('./commands/files/add')
+      const catCmd = require('./commands/files/cat')
+      const getCmd = require('./commands/files/get')
+      const aliases = [addCmd, catCmd, getCmd]
+      aliases.forEach((alias) => {
+        cli.command(alias)
+      })
 
       cli
+        .commandDir('commands')
         .help()
         .strict()
         .completion()
-        .parse(args, { ipfs: ipfs }, (err, argv, output) => {
-          if (output) { print(output) }
 
-          cleanup(() => {
-            if (err) { throw err }
-          })
+      const parser = new YargsPromise(cli, { ipfs })
+      parser.parse(args)
+        .then(({ data, argv }) => {
+          if (data) {
+            print(data)
+          }
+
+          return cleanup()
+        })
+        .catch((arg) => {
+          debug(arg)
+
+          // the argument can have a different shape depending on where the error came from
+          if (arg.message) {
+            print(arg.message)
+          } else if (arg.error && arg.error.message) {
+            print(arg.error.message)
+          } else {
+            print('Unknown error, please re-run the command with DEBUG=ipfs:cli to see debug output')
+          }
+
+          process.exit(1)
         })
     })
   })

src/cli/commands/dag/get.js

@@ -41,7 +41,7 @@ module.exports = {
       // * reads as 'agree in'
       if (node._json) {
         delete node._json.multihash
-        node._json.data = '0x' + node._json.data.toString('hex')
+        node._json.data = node._json.data.toString('base64')
         print(JSON.stringify(node._json))
         return
       }

src/cli/commands/files.js

@@ -30,6 +30,7 @@ function fileHandler (dir) {
         callback(err)
       } else {
         const fullFilePath = path.join(dir, file.path)
+
         if (file.content) {
           file.content
             .pipe(fs.createWriteStream(fullFilePath))

src/cli/commands/files/get.js

@@ -9,6 +9,7 @@ const log = debug('cli')
 log.error = debug('cli:error')
 const Progress = require('progress')
 const byteman = require('byteman')
+const promisify = require('promisify-es6')
 
 exports = module.exports
 
@@ -40,7 +41,7 @@ function getAPICtl (apiAddr) {
 
 exports.getIPFS = (argv, callback) => {
   if (argv.api || isDaemonOn()) {
-    return callback(null, getAPICtl(argv.api), (cb) => cb())
+    return callback(null, getAPICtl(argv.api), promisify((cb) => cb()))
   }
 
   // Required inline to reduce startup time
@@ -55,13 +56,13 @@ exports.getIPFS = (argv, callback) => {
     }
   })
 
-  const cleanup = (cb) => {
+  const cleanup = promisify((cb) => {
     if (node && node._repo && !node._repo.closed) {
-      node._repo.close(() => cb())
+      node._repo.close((err) => cb(err))
     } else {
       cb()
     }
-  }
+  })
 
   node.on('error', (err) => {
     throw err

src/cli/utils.js

@@ -26,3 +26,4 @@ exports.dht = require('./dht')
 exports.dns = require('./dns')
 exports.key = require('./key')
 exports.stats = require('./stats')
+exports.mfs = require('ipfs-mfs/core')

src/core/components/index.js

@@ -16,7 +16,8 @@ const schema = Joi.object().keys({
   EXPERIMENTAL: Joi.object().keys({
     pubsub: Joi.boolean(),
     sharding: Joi.boolean(),
-    dht: Joi.boolean()
+    dht: Joi.boolean(),
+    mfs: Joi.boolean()
   }).allow(null),
   config: Joi.object().keys({
     Addresses: Joi.object().keys({

src/core/config.js

@@ -137,6 +137,15 @@ class IPFS extends EventEmitter {
       isIPFS: isIPFS
     }
 
+    // ipfs.files
+    const mfs = components.mfs(this)
+
+    Object.keys(mfs).forEach(key => {
+      if (mfs.hasOwnProperty(key)) {
+        this.files[key] = mfs[key]
+      }
+    })
+
     boot(this)
   }
 }

src/core/index.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const resources = require('./../resources')
+const mfs = require('ipfs-mfs/http')
 
 module.exports = (server) => {
   const api = server.select('API')
@@ -54,4 +55,6 @@ module.exports = (server) => {
       handler: resources.files.immutableLs.handler
     }
   })
+
+  mfs(api)
 }

src/http/api/routes/files.js

@@ -0,0 +1,8 @@
+'use strict'
+
+const isWebWorker = require('detect-webworker')
+
+if (isWebWorker) {
+  // https://github.com/Joris-van-der-Wel/karma-mocha-webworker/issues/4
+  global.MFS_DISABLE_CONCURRENCY = true
+}

test/browser.js

@@ -4,7 +4,8 @@
 const expect = require('chai').expect
 const runOnAndOff = require('../utils/on-and-off')
 
-const commandCount = 78
+const commandCount = 77
+
 describe('commands', () => runOnAndOff((thing) => {
   let ipfs
 

test/cli/commands.js

@@ -12,7 +12,7 @@ describe('file ls', () => runOnAndOff((thing) => {
   before(function () {
     this.timeout(50 * 1000)
     ipfs = thing.ipfs
-    return ipfs('files add -r test/fixtures/test-data/recursive-get-dir')
+    return ipfs('add -r test/fixtures/test-data/recursive-get-dir')
   })
 
   it('prints a filename', () => {