feat: integration of js-ipfs-repo-migrations

Integration of js-ipfs-repo-migrations brings automatic repo migrations
to ipfs-repo (both in-browser and fs). It is possible to control the
automatic migration using either config's setting
'repoDisableAutoMigration' or IPFSRepo's option 'disableAutoMigration'.

BREAKING CHANGE: repo.blocks.query() now returns multihashes as a key
instead of CID. If you want to have CID returned call it as query({},
true), which will constructs CIDv1 using IPLD's RAW codec. This means
that this constructed CID might not equal to the one that the block was originally
saved. Related to ipfs/js-ipfs#2415

Author: AuHau

README.md

@@ -41,8 +41,11 @@ This is the implementation of the [IPFS repo spec](https://github.com/ipfs/specs
   - [Blocks](#blocks)
     - [`Promise<Block> repo.blocks.put(block:Block)`](#promiseblock-repoblocksputblockblock)
     - [`AsyncIterator<Block> repo.blocks.putMany(source)`](#asynciteratorblock-repoblocksputmanysource)
-    - [`Promise<Buffer> repo.blocks.get(cid)`](#promisebuffer-repoblocksgetcid)
-    - [`AsyncIterable<Buffer> repo.blocks.getMany(source)`](#asynciterablebuffer-repoblocksgetmanysource)
+    - [`Promise<Block> repo.blocks.get(cid)`](#promiseblock-repoblocksgetcid)
+    - [`AsyncIterable<Block> repo.blocks.getMany(source)`](#asynciterableblock-repoblocksgetmanysource)
+    - [`Promise<boolean> repo.blocks.has (obj)`](#promiseboolean-repoblockshas-obj)
+    - [`Promise<boolean> repo.blocks.delete (obj)`](#promiseboolean-repoblocksdelete-obj)
+    - [`Promise<Array<Object>> repo.blocks.query (query, reconstructsCids)`](#promisearrayobject-repoblocksquery-query-reconstructscids)
     - [`Promise<CID> repo.blocks.delete(cid:CID)`](#promisecid-repoblocksdeletecidcid)
     - [`AsyncIterator<CID> repo.blocks.deleteMany(source)`](#asynciteratorcid-repoblocksdeletemanysource)
   - [Datastore](#datastore)
@@ -235,18 +238,39 @@ Put many blocks.
 
 * `source` should be an AsyncIterable that yields entries of type [Block][]
 
-#### `Promise<Buffer> repo.blocks.get(cid)`
+#### `Promise<Block> repo.blocks.get(cid)`
 
 Get block.
 
 * `cid` is the content id of type [CID][]
 
-#### `AsyncIterable<Buffer> repo.blocks.getMany(source)`
+#### `AsyncIterable<Block> repo.blocks.getMany(source)`
 
-Get block.
+Get many blocks
 
 * `source` should be an AsyncIterable that yields entries of type [CID][]
 
+#### `Promise<boolean> repo.blocks.has (obj)`
+
+Indicate if block is present
+
+* `obj` is either the content id of [type CID](https://github.com/ipld/js-cid#readme) or [multihash](https://github.com/multiformats/js-multihashing).
+
+#### `Promise<boolean> repo.blocks.delete (obj)`
+
+Deletes
+
+* `obj` is either the content id of [type CID](https://github.com/ipld/js-cid#readme) or [multihash](https://github.com/multiformats/js-multihashing).
+
+#### `Promise<Array<Object>> repo.blocks.query (query, reconstructsCids)`
+
+Query what blocks are available in blockstore.
+
+* `query` is a object as specified in [interface-datastore](https://github.com/ipfs/interface-datastore#query).
+* `reconstructsCids` a flag defining if the block's key is a reconstructed CID (eq. CIDv1 with RAW IPLD codec) or multihash
+
+Datastore:
+
 #### `Promise<CID> repo.blocks.delete(cid:CID)`
 
 * `cid` should be of the type [CID][]
@@ -379,7 +403,7 @@ Returned promise resolves to a `boolean` indicating the existence of the lock.
 
 ### Migrations
 
-When there is a new repo migration and the version of repo is increased, don't
+When there is a new repo migration and the version of the repo is increased, don't
 forget to propagate the changes into the test repo (`test/test-repo`).
 
 **For tools that run mainly in the browser environment, be aware that disabling automatic

package.json

@@ -59,6 +59,7 @@
     "sinon": "^9.0.2"
   },
   "dependencies": {
+    "base32.js": "^0.1.0",
     "bignumber.js": "^9.0.0",
     "buffer": "^5.6.0",
     "bytes": "^3.1.0",
@@ -69,7 +70,7 @@
     "debug": "^4.1.0",
     "err-code": "^2.0.0",
     "interface-datastore": "^1.0.2",
-    "ipfs-repo-migrations": "^0.2.1",
+    "ipfs-repo-migrations": "github:ipfs/js-ipfs-repo-migrations#migration/8-multihash_and_keys",
     "ipfs-utils": "^2.2.0",
     "ipld-block": "^0.9.1",
     "it-map": "^1.0.2",

src/blockstore-utils.js

@@ -2,8 +2,8 @@
 
 const { Key } = require('interface-datastore')
 const CID = require('cids')
-const multibase = require('multibase')
-const errcode = require('err-code')
+const errCode = require('err-code')
+const base32 = require('base32.js')
 
 /**
  * Transform a cid to the appropriate datastore key.
@@ -12,19 +12,25 @@ const errcode = require('err-code')
  * @returns {Key}
  */
 exports.cidToKey = cid => {
-  if (!CID.isCID(cid)) {
-    throw errcode(new Error('Not a valid cid'), 'ERR_INVALID_CID')
+  if (!cid || !cid.multihash) {
+    throw errCode(new Error('Invalid Multihash'), 'ERR_INVALID_CID')
   }
 
-  return new Key('/' + multibase.encode('base32', cid.buffer).toString().slice(1).toUpperCase(), false)
+  const enc = new base32.Encoder()
+  return new Key('/' + enc.write(cid.multihash).finalize(), false)
 }
 
 /**
  * Transform a datastore Key instance to a CID
+ * As Key is a multihash of the CID, it is reconstructed using IPLD's RAW codec.
+ * Hence it is highly probable that stored CID will differ from a CID retrieved from blockstore.
  *
  * @param {Key} key
  * @returns {CID}
  */
 exports.keyToCid = key => {
-  return new CID(multibase.decode('b' + key.toString().slice(1).toLowerCase()))
+  // Block key is of the form /<base32 encoded string>
+  const decoder = new base32.Decoder()
+  const buff = decoder.write(key.toString().slice(1)).finalize()
+  return new CID(1, 'raw', Buffer.from(buff))
 }

src/blockstore.js

@@ -3,9 +3,7 @@
 const core = require('datastore-core')
 const ShardingStore = core.ShardingDatastore
 const Block = require('ipld-block')
-const { cidToKey, keyToCid } = require('./blockstore-utils')
-const map = require('it-map')
-const pipe = require('it-pipe')
+const { cidToKey } = require('./blockstore-utils')
 
 module.exports = async (filestore, options) => {
   const store = await maybeWithSharding(filestore, options)
@@ -25,7 +23,7 @@ function createBaseStore (store) {
     /**
      * Query the store.
      *
-     * @param {Object} query
+     * @param {object} query
      * @param {Object} options
      * @returns {AsyncIterator<Block>}
      */
@@ -41,26 +39,9 @@ function createBaseStore (store) {
      */
     async get (cid, options) {
       const key = cidToKey(cid)
-      let blockData
-      try {
-        blockData = await store.get(key, options)
-        return new Block(blockData, cid)
-      } catch (err) {
-        if (err.code === 'ERR_NOT_FOUND') {
-          const otherCid = cidToOtherVersion(cid)
+      const blockData = await store.get(key, options)
 
-          if (!otherCid) {
-            throw err
-          }
-
-          const otherKey = cidToKey(otherCid)
-          const blockData = await store.get(otherKey, options)
-          await store.put(key, blockData)
-          return new Block(blockData, cid)
-        }
-
-        throw err
-      }
+      return new Block(blockData, cid)
     },
     /**
      * Like get, but for more.
@@ -86,15 +67,14 @@ function createBaseStore (store) {
         throw new Error('invalid block')
       }
 
-      const exists = await this.has(block.cid)
+      const key = cidToKey(block.cid)
+      const exists = await store.has(key, options)
 
       if (exists) {
-        return this.get(block.cid, options)
+        return
       }
 
-      await store.put(cidToKey(block.cid), block.data, options)
-
-      return block
+      return store.put(key, block.data, options)
     },
 
     /**
@@ -104,43 +84,32 @@ function createBaseStore (store) {
      * @param {Object} options
      * @returns {AsyncIterable<Block>}
      */
-    async * putMany (blocks, options) { // eslint-disable-line require-await
-      yield * pipe(
-        blocks,
-        (source) => {
-          // turn them into a key/value pair
-          return map(source, (block) => {
-            return { key: cidToKey(block.cid), value: block.data }
-          })
-        },
-        (source) => {
-          // put them into the datastore
-          return store.putMany(source, options)
-        },
-        (source) => {
-          // map the returned key/value back into a block
-          return map(source, ({ key, value }) => {
-            return new Block(value, keyToCid(key))
-          })
+    async * putMany (blocks, options) {
+      for await (const block of blocks) {
+        const key = cidToKey(block.cid)
+
+        if (await store.has(key)) {
+          continue
         }
-      )
+
+        await store.put(key, block.data, options)
+
+        yield block
+      }
     },
+
     /**
-     * Does the store contain block with this cid?
+     * Does the store contain block with this CID?
      *
      * @param {CID} cid
      * @param {Object} options
      * @returns {Promise<bool>}
      */
-    async has (cid, options) {
-      const exists = await store.has(cidToKey(cid), options)
-      if (exists) return exists
-      const otherCid = cidToOtherVersion(cid)
-      if (!otherCid) return false
-      return store.has(cidToKey(otherCid), options)
+    async has (cid, options) { // eslint-disable-line require-await
+      return store.has(cidToKey(cid), options)
     },
     /**
-     * Delete a block from the store
+     * Delete a CID or multihash from the store
      *
      * @param {CID} cid
      * @param {Object} options
@@ -173,11 +142,3 @@ function createBaseStore (store) {
     }
   }
 }
-
-function cidToOtherVersion (cid) {
-  try {
-    return cid.version === 0 ? cid.toV1() : cid.toV0()
-  } catch (err) {
-    return null
-  }
-}

src/constants.js

@@ -1,5 +1,5 @@
 'use strict'
 
 module.exports = {
-  repoVersion: 7
+  repoVersion: 8
 }

src/index.js

@@ -350,11 +350,11 @@ class IpfsRepo {
     let count = new Big(0)
     let size = new Big(0)
 
-    for await (const block of this.blocks.query({})) {
+    for await (const block of this.blocks.query({}, false)) {
       count = count.plus(1)
       size = size
         .plus(block.value.byteLength)
-        .plus(block.key._buf.byteLength)
+        .plus(block.key.toBuffer().byteLength)
     }
 
     return { count, size }
@@ -365,7 +365,7 @@ async function getSize (queryFn) {
   const sum = new Big(0)
   for await (const block of queryFn.query({})) {
     sum.plus(block.value.byteLength)
-      .plus(block.key._buf.byteLength)
+      .plus(block.key.toBuffer().byteLength)
   }
   return sum
 }

test/blockstore-test.js

@@ -157,6 +157,10 @@ module.exports = (repo) => {
                   throw err
                 }
               }
+
+              has () {
+                return true
+              }
             }
           },
           storageBackendOptions: {

test/blockstore-utils-test.js

@@ -9,7 +9,8 @@ const Repo = require('../src')
 module.exports = () => {
   describe('blockstore utils', () => {
     it('converts a CID to a datastore Key and back', () => {
-      const originalCid = new CID('Qme6KJdKcp85TYbLxuLV7oQzMiLremD7HMoXLZEmgo6Rnh')
+      // CIDv1 in base32 with IPLD raw codec
+      const originalCid = new CID('bafkreihkb3vrxxex5zvzkr3s3a6noe223r7jka4ofjy2nkzu27kueg76ii')
       const key = Repo.utils.blockstore.cidToKey(originalCid)
       expect(key instanceof Key).to.be.true()
       const cid = Repo.utils.blockstore.keyToCid(key)

test/repo-test.js

@@ -56,12 +56,12 @@ module.exports = (repo) => {
 
     describe('version', () => {
       afterEach(async () => {
-        await repo.version.set(7)
+        await repo.version.set(8)
       })
 
       it('get version', async () => {
         const version = await repo.version.get()
-        expect(version).to.equal(7)
+        expect(version).to.equal(8)
       })
 
       it('set version', async () => {