feat: migration 10 to allow upgrading level in the browser

We use the [level](https://www.npmjs.com/package/level) module to supply
either [leveldown](http://npmjs.com/package/leveldown) or [level-js](https://www.npmjs.com/package/level-js)
to [datastore-level](https://www.npmjs.com/package/datastore-level) depending
on if we're running under node or in the browser.

`level@6.x.x` upgrades the `level-js` dependency from `4.x.x` to `5.x.x`
which includes the changes from [Level/level-js#179](Level/level-js#179)
so `>5.x.x` requires all database keys/values to be Uint8Arrays and they
can no longer be strings.

We already store values as Uint8Arrays but our keys are strings, so here
we add a migration to converts all datastore keys to Uint8Arrays.

N.b. `leveldown` already does this conversion for us so this migration
only needs to run in the browser.

Author: achingbrain

README.md

@@ -42,6 +42,11 @@ This package is inspired by the [go-ipfs repo migration tool](https://github.com
   - [Tests](#tests)
   - [Empty migrations](#empty-migrations)
   - [Migrations matrix](#migrations-matrix)
+  - [Migrations](#migrations)
+    - [7](#7)
+    - [8](#8)
+    - [9](#9)
+    - [10](#10)
 - [Developer](#developer)
   - [Module versioning notes](#module-versioning-notes)
 - [Contribute](#contribute)
@@ -268,6 +273,24 @@ This will create an empty migration with the next version.
 |                  8 | v0.48.0          |
 |                  9 | v0.49.0          |
 
+### Migrations
+
+#### 7
+
+This is the initial version of the datastore, inherited from go-IPFS in an attempt to maintain cross-compatibility between the two implementations.
+
+#### 8
+
+Blockstore keys are transformed into base32 representations of the multihash from the CID of the block.
+
+#### 9
+
+Pins were migrated from a DAG to a Datastore - see [ipfs/js-ipfs#2771](https://github.com/ipfs/js-ipfs/pull/2771)
+
+#### 10
+
+`level@6.x.x` upgrades the `level-js` dependency from `4.x.x` to `5.x.x`.  This update requires a database migration to convert all string keys/values into buffers. Only runs in the browser, node is unaffected. See [Level/level-js#179](https://github.com/Level/level-js/pull/179)
+
 ## Developer
 
 ### Module versioning notes

migrations/index.js

@@ -16,5 +16,6 @@ module.exports = [
   Object.assign({version: 6}, emptyMigration),
   Object.assign({version: 7}, emptyMigration),
   require('./migration-8'),
-  require('./migration-9')
+  require('./migration-9'),
+  require('./migration-10')
 ]

migrations/migration-10/index.js

@@ -0,0 +1,168 @@
+'use strict'
+
+const { createStore } = require('../../src/utils')
+const { Key } = require('interface-datastore')
+const fromString = require('uint8arrays/from-string')
+const toString = require('uint8arrays/to-string')
+
+const findUpgradableDb = (store) => {
+  let db = store
+
+  while (db.db || db.child) {
+    db = db.db || db.child
+
+    // Will stop at Level in the browser, LevelDOWN in node
+    if (db.constructor.name === 'Level') {
+      return db
+    }
+  }
+}
+
+async function keysToBinary (name, store, onProgress = () => {}) {
+  let db = findUpgradableDb(store)
+
+  // only interested in Level
+  if (!db) {
+    onProgress(`${name} did not need an upgrade`)
+
+    return
+  }
+
+  onProgress(`Upgrading ${name}`)
+
+  await withEach(db, (key, value) => {
+    return [
+      { type: 'del', key: key },
+      { type: 'put', key: fromString(key), value: value }
+    ]
+  })
+}
+
+async function keysToStrings (name, store, onProgress = () => {}) {
+  let db = findUpgradableDb(store)
+
+  // only interested in Level
+  if (!db) {
+    onProgress(`${name} did not need a downgrade`)
+
+    return
+  }
+
+  onProgress(`Downgrading ${name}`)
+
+  await withEach(db, (key, value) => {
+    return [
+      { type: 'del', key: key },
+      { type: 'put', key: toString(key), value: value }
+    ]
+  })
+}
+
+async function process (repoPath, repoOptions, onProgress, fn) {
+  const datastores = Object.keys(repoOptions.storageBackends)
+    .filter(key => repoOptions.storageBackends[key].name === 'LevelDatastore')
+    .map(name => ({
+      name,
+      store: createStore(repoPath, name, repoOptions)
+    }))
+
+  onProgress(0, `Migrating ${datastores.length} dbs`)
+  let migrated = 0
+
+  for (const { name, store } of datastores) {
+    await store.open()
+
+    try {
+      await fn(name, store, (message) => {
+        onProgress(parseInt((migrated / datastores.length) * 100), message)
+      })
+    } finally {
+      migrated++
+      store.close()
+    }
+  }
+
+  onProgress(100, `Migrated ${datastores.length} dbs`)
+}
+
+module.exports = {
+  version: 10,
+  description: 'Migrates datastore-level keys to binary',
+  migrate: (repoPath, repoOptions, onProgress) => {
+    return process(repoPath, repoOptions, onProgress, keysToBinary)
+  },
+  revert: (repoPath, repoOptions, onProgress) => {
+    return process(repoPath, repoOptions, onProgress, keysToStrings)
+  }
+}
+
+/**
+ * @typedef {Error | undefined} Err
+ * @typedef {Uint8Array|string} Key
+ * @typedef {Uint8Array} Value
+ * @typedef {{ type: 'del', key: Key } | { type: 'put', key: Key, value: Value }} Operation
+ *
+ * Uses the upgrade strategy from level-js@5.x.x - note we can't call the `.upgrade` command
+ * directly because it will be removed in level-js@6.x.x and we can't guarantee users will
+ * have migrated by then - e.g. they may jump from level-js@4.x.x straight to level-js@6.x.x
+ * so we have to duplicate the code here.
+ *
+ * @param {import('interface-datastore').Datastore} db
+ * @param {function (Err, Key, Value): Operation[]} fn
+ */
+function withEach (db, fn) {
+  function batch (operations, next) {
+    const store = db.store('readwrite')
+    const transaction = store.transaction
+    let index = 0
+    let error
+
+    transaction.onabort = () => next(error || transaction.error || new Error('aborted by user'))
+    transaction.oncomplete = () => next()
+
+    function loop () {
+      var op = operations[index++]
+      var key = op.key
+
+      try {
+        var req = op.type === 'del' ? store.delete(key) : store.put(op.value, key)
+      } catch (err) {
+        error = err
+        transaction.abort()
+        return
+      }
+
+      if (index < operations.length) {
+        req.onsuccess = loop
+      }
+    }
+
+    loop()
+  }
+
+  return new Promise((resolve, reject) => {
+    const it = db.iterator()
+    // raw keys and values only
+    it._deserializeKey = it._deserializeValue = (data) => data
+    next()
+
+    function next () {
+      it.next((err, key, value) => {
+        if (err || key === undefined) {
+          it.end((err2) => {
+            if (err2) {
+              reject(err2)
+              return
+            }
+
+            resolve()
+          })
+
+          return
+        }
+
+        batch(fn(key, value), next)
+      })
+    }
+  })
+}

package.json

@@ -64,6 +64,8 @@
     "datastore-level": "^3.0.0",
     "it-all": "^1.0.2",
     "just-safe-set": "^2.1.0",
+    "level-5": "npm:level@^5.0.0",
+    "level-6": "npm:level@^6.0.0",
     "ncp": "^2.0.0",
     "rimraf": "^3.0.0",
     "sinon": "^9.0.2"

test/browser.js

@@ -1,16 +1,17 @@
 /* eslint-env mocha */
 'use strict'
 
+const DatastoreLevel = require('datastore-level')
 const { createRepo, createAndLoadRepo } = require('./fixtures/repo')
 
 const repoOptions = {
   lock: 'memory',
   storageBackends: {
-    root: require('datastore-level'),
-    blocks: require('datastore-level'),
-    keys: require('datastore-level'),
-    datastore: require('datastore-level'),
-    pins: require('datastore-level')
+    root: DatastoreLevel,
+    blocks: DatastoreLevel,
+    keys: DatastoreLevel,
+    datastore: DatastoreLevel,
+    pins: DatastoreLevel
   },
   storageBackendOptions: {
     root: {
@@ -51,7 +52,7 @@ describe('Browser specific tests', () => {
   })
 
   describe('migrations tests', () => {
-    require('./migrations')(() => createRepo(repoOptions), repoCleanup, repoOptions)
+    require('./migrations')(() => createRepo(repoOptions), repoCleanup)
   })
 
   describe('init tests', () => {

test/migrations/index.js

@@ -77,6 +77,7 @@ module.exports = (createRepo, repoCleanup) => {
     describe(name, () => {
       require('./migration-8-test')(createRepo, repoCleanup, options)
       require('./migration-9-test')(createRepo, repoCleanup, options)
+      require('./migration-10-test')(createRepo, repoCleanup, options)
     })
   })
 }

test/migrations/migration-10-test.js

@@ -0,0 +1,125 @@
+/* eslint-env mocha */
+/* eslint-disable max-nested-callbacks */
+'use strict'
+
+const { expect } = require('aegir/utils/chai')
+
+const { createStore } = require('../../src/utils')
+const migration = require('../../migrations/migration-10')
+const Key = require('interface-datastore').Key
+const fromString = require('uint8arrays/from-string')
+const Level5 = require('level-5')
+const Level6 = require('level-6')
+
+const keys = {
+  CIQCKN76QUQUGYCHIKGFE6V6P3GJ2W26YFFPQW6YXV7NFHH3QB2RI3I: 'hello',
+  CIQKKLBWAIBQZOIS5X7E32LQAL6236OUKZTMHPQSFIXPWXNZHQOV7JQ: fromString('derp')
+}
+
+async function bootstrap (dir, backend, repoOptions) {
+  const store = createStore(dir, backend, repoOptions)
+  await store.open()
+
+  for (const name of Object.keys(keys)) {
+    await store.put(new Key(name), keys[name])
+  }
+
+  await store.close()
+}
+
+async function validate (dir, backend, repoOptions) {
+  const store = createStore(dir, backend, repoOptions)
+
+  await store.open()
+
+  for (const name of Object.keys(keys)) {
+    const key = new Key(`/${name}`)
+
+    expect(await store.has(key)).to.be.true(`Could not read key ${name}`)
+    expect(store.get(key)).to.eventually.equal(keys[name], `Could not read value for key ${keys[name]}`)
+  }
+
+  await store.close()
+}
+
+function withLevel (repoOptions, levelImpl) {
+  const stores = Object.keys(repoOptions.storageBackends)
+    .filter(key => repoOptions.storageBackends[key].name === 'LevelDatastore')
+
+  const output = {
+    ...repoOptions
+  }
+
+  stores.forEach(store => {
+    // override version of level passed to datastore options
+    output.storageBackendOptions[store] = {
+      ...output.storageBackendOptions[store],
+      db: levelImpl
+    }
+  })
+
+  return output
+}
+
+module.exports = (setup, cleanup, repoOptions) => {
+  describe('migration 10', function () {
+    this.timeout(240 * 1000)
+    let dir
+
+    beforeEach(async () => {
+      dir = await setup()
+    })
+
+    afterEach(async () => {
+      await cleanup(dir)
+    })
+
+    describe('forwards', () => {
+      beforeEach(async () => {
+        for (const backend of Object.keys(repoOptions.storageBackends)) {
+          await bootstrap(dir, backend, withLevel(repoOptions, Level5))
+        }
+      })
+
+      it('should migrate keys and values forward', async () => {
+        await migration.migrate(dir, withLevel(repoOptions, Level6), () => {})
+
+        for (const backend of Object.keys(repoOptions.storageBackends)) {
+          await validate(dir, backend, withLevel(repoOptions, Level6))
+        }
+      })
+    })
+
+    describe('backwards using level@6.x.x', () => {
+      beforeEach(async () => {
+        for (const backend of Object.keys(repoOptions.storageBackends)) {
+          await bootstrap(dir, backend, withLevel(repoOptions, Level6))
+        }
+      })
+
+      it('should migrate keys and values backward', async () => {
+        await migration.revert(dir, withLevel(repoOptions, Level6), () => {})
+
+        for (const backend of Object.keys(repoOptions.storageBackends)) {
+          await validate(dir, backend, withLevel(repoOptions, Level5))
+        }
+      })
+    })
+
+    describe('backwards using level@5.x.x', () => {
+      beforeEach(async () => {
+        for (const backend of Object.keys(repoOptions.storageBackends)) {
+          await bootstrap(dir, backend, withLevel(repoOptions, Level6))
+        }
+      })
+
+      it('should migrate keys and values backward', async () => {
+        await migration.revert(dir, withLevel(repoOptions, Level5), () => {})
+
+        for (const backend of Object.keys(repoOptions.storageBackends)) {
+          await validate(dir, backend, withLevel(repoOptions, Level5))
+        }
+      })
+    })
+  })
+}