feat: tests for GC

Author: dirkmc

src/repo/gc.js

@@ -1,7 +1,21 @@
 /* eslint-env mocha */
 'use strict'
 
+const { promisify } = require('es6-promisify')
 const { getDescribe, getIt, expect } = require('../utils/mocha')
+const { DAGNode, DAGLink } = require('ipld-dag-pb')
+
+const createDAGNode = promisify((data, links, cb) => {
+  DAGNode.create(data, links, cb)
+})
+
+const createDAGLink = promisify((name, size, cid, cb) => {
+  DAGLink.create(name, size, cid, cb)
+})
+
+const addDAGLink = promisify((node, link, cb) => {
+  DAGNode.addLink(node, link, cb)
+})
 
 module.exports = (createCommon, options) => {
   const describe = getDescribe(options)
@@ -41,5 +55,188 @@ module.exports = (createCommon, options) => {
         expect(res).to.exist()
       })
     })
+
+    it('should clean up unpinned data', async () => {
+      // Get initial list of local blocks
+      const refsBeforeAdd = await ipfs.refs.local()
+
+      // Add some data. Note: this will implicitly pin the data, which causes
+      // some blocks to be added for the data itself and for the pinning
+      // information that refers to the blocks
+      const addRes = await ipfs.add(Buffer.from('apples'))
+      const hash = addRes[0].hash
+
+      // Get the list of local blocks after the add, should be bigger than
+      // the initial list and contain hash
+      const refsAfterAdd = await ipfs.refs.local()
+      expect(refsAfterAdd.length).to.be.gt(refsBeforeAdd.length)
+      expect(refsAfterAdd.map(r => r.ref)).includes(hash)
+
+      // Run garbage collection
+      await ipfs.repo.gc()
+
+      // Get the list of local blocks after GC, should still contain the hash,
+      // because the file is still pinned
+      const refsAfterGc = await ipfs.refs.local()
+      expect(refsAfterGc.map(r => r.ref)).includes(hash)
+
+      // Unpin the data
+      await ipfs.pin.rm(hash)
+
+      // Run garbage collection
+      await ipfs.repo.gc()
+
+      // The list of local blocks should now be the same as at the start
+      const refsAfterUnpinAndGc = await ipfs.refs.local()
+      expect(refsAfterUnpinAndGc).to.eql(refsBeforeAdd)
+    })
+
+    it('should clean up removed MFS files', async () => {
+      // Get initial list of local blocks
+      const refsBeforeAdd = await ipfs.refs.local()
+
+      // Add a file to MFS
+      await ipfs.files.write('/test', Buffer.from('oranges'), { create: true })
+      const stats = await ipfs.files.stat('/test')
+      expect(stats.type).to.equal('file')
+      const hash = stats.hash
+
+      // Get the list of local blocks after the add, should be bigger than
+      // the initial list and contain hash
+      const refsAfterAdd = await ipfs.refs.local()
+      expect(refsAfterAdd.length).to.be.gt(refsBeforeAdd.length)
+      expect(refsAfterAdd.map(r => r.ref)).includes(hash)
+
+      // Run garbage collection
+      await ipfs.repo.gc()
+
+      // Get the list of local blocks after GC, should still contain the hash,
+      // because the file is in MFS
+      const refsAfterGc = await ipfs.refs.local()
+      expect(refsAfterGc.map(r => r.ref)).includes(hash)
+
+      // Remove the file
+      await ipfs.files.rm('/test')
+
+      // Run garbage collection
+      await ipfs.repo.gc()
+
+      // The list of local blocks should now be the same as at the start
+      const refsAfterUnpinAndGc = await ipfs.refs.local()
+      expect(refsAfterUnpinAndGc).to.eql(refsBeforeAdd)
+    })
+
+    it('should clean up block only after unpinned and removed from MFS', async () => {
+      // Get initial list of local blocks
+      const refsBeforeAdd = await ipfs.refs.local()
+
+      // Add a file to MFS
+      await ipfs.files.write('/test', Buffer.from('peaches'), { create: true })
+      const stats = await ipfs.files.stat('/test')
+      expect(stats.type).to.equal('file')
+      const mfsFileHash = stats.hash
+
+      // Get the CID of the data in the file
+      const block = await ipfs.block.get(mfsFileHash)
+
+      // Add the data to IPFS (which implicitly pins the data)
+      const addRes = await ipfs.add(block.data)
+      const dataHash = addRes[0].hash
+
+      // Get the list of local blocks after the add, should be bigger than
+      // the initial list and contain the data hash
+      const refsAfterAdd = await ipfs.refs.local()
+      expect(refsAfterAdd.length).to.be.gt(refsBeforeAdd.length)
+      const hashesAfterAdd = refsAfterAdd.map(r => r.ref)
+      expect(hashesAfterAdd).includes(dataHash)
+
+      // Run garbage collection
+      await ipfs.repo.gc()
+
+      // Get the list of local blocks after GC, should still contain the hash,
+      // because the file is pinned and in MFS
+      const refsAfterGc = await ipfs.refs.local()
+      const hashesAfterGc = refsAfterGc.map(r => r.ref)
+      expect(hashesAfterGc).includes(dataHash)
+
+      // Remove the file
+      await ipfs.files.rm('/test')
+
+      // Run garbage collection
+      await ipfs.repo.gc()
+
+      // Get the list of local blocks after GC, should still contain the hash,
+      // because the file is still pinned
+      const refsAfterRmAndGc = await ipfs.refs.local()
+      const hashesAfterRmAndGc = refsAfterRmAndGc.map(r => r.ref)
+      expect(hashesAfterRmAndGc).not.includes(mfsFileHash)
+      expect(hashesAfterRmAndGc).includes(dataHash)
+
+      // Unpin the data
+      await ipfs.pin.rm(dataHash)
+
+      // Run garbage collection
+      await ipfs.repo.gc()
+
+      // The list of local blocks should now be the same as at the start
+      const refsAfterUnpinAndGc = await ipfs.refs.local()
+      expect(refsAfterUnpinAndGc).to.eql(refsBeforeAdd)
+    })
+
+    it('should clean up indirectly pinned data after recursive pin removal', async () => {
+      // Get initial list of local blocks
+      const refsBeforeAdd = await ipfs.refs.local()
+
+      // Add some data
+      const addRes = await ipfs.add(Buffer.from('pears'))
+      const dataHash = addRes[0].hash
+
+      // Unpin the data
+      await ipfs.pin.rm(dataHash)
+
+      // Create a link to the data from an object
+      const link = await createDAGLink('p', addRes[0].size, dataHash)
+      const node = await createDAGNode(Buffer.from('fruit'))
+      const obj = await addDAGLink(node, link)
+
+      // Put the object into IPFS
+      const objHash = (await ipfs.object.put(obj)).toString()
+
+      // Putting an object doesn't pin it
+      expect((await ipfs.pin.ls()).map(p => p.hash)).not.includes(objHash)
+
+      // Get the list of local blocks after the add, should be bigger than
+      // the initial list and contain data and object hash
+      const refsAfterAdd = await ipfs.refs.local()
+      expect(refsAfterAdd.length).to.be.gt(refsBeforeAdd.length)
+      const hashesAfterAdd = refsAfterAdd.map(r => r.ref)
+      expect(hashesAfterAdd).includes(objHash)
+      expect(hashesAfterAdd).includes(dataHash)
+
+      // Recursively pin the object
+      await ipfs.pin.add(objHash, { recursive: true })
+
+      // The data should now be indirectly pinned
+      const pins = await ipfs.pin.ls()
+      expect(pins.find(p => p.hash === dataHash).type).to.eql('indirect')
+
+      // Run garbage collection
+      await ipfs.repo.gc()
+
+      // Get the list of local blocks after GC, should still contain the data
+      // hash, because the data is still (indirectly) pinned
+      const refsAfterGc = await ipfs.refs.local()
+      expect(refsAfterGc.map(r => r.ref)).includes(dataHash)
+
+      // Recursively unpin the object
+      await ipfs.pin.rm(objHash)
+
+      // Run garbage collection
+      await ipfs.repo.gc()
+
+      // The list of local blocks should now be the same as at the start
+      const refsAfterUnpinAndGc = await ipfs.refs.local()
+      expect(refsAfterUnpinAndGc).to.eql(refsBeforeAdd)
+    })
   })
 }