feat: dynamic format loading

This backwards compatible PR allows missing IPLD formats to be loaded dynamically.

This follows on from the discussion here #164 (comment)

A new constructor option `loadFormat` allows users to asynchronously load an IPLD format. The IPLD instance will call this function automatically when it is requested to resolve a format that it doesn't currently understand.

This can re-enable lazy loading for IPLD modules in IPFS and also means that people who just want to add _more_ resolvers can do so without having to depend directly on the defaults and pass them in as well.

License: MIT
Signed-off-by: Alan Shaw <alan.shaw@protocol.ai>

README.md

@@ -130,6 +130,26 @@ const ipld = new Ipld({
 })
 ```
 
+##### `options.loadFormat(codec, callback)`
+
+| Type | Default |
+|------|---------|
+| `Function` | `null` |
+
+Function to dynamically load an [IPLD Format](https://github.com/ipld/interface-ipld-format). It is passed a string `codec`, the multicodec of the IPLD format to load and a callback function to call when the format has been loaded. e.g.
+
+```js
+const ipld = new Ipld({
+  loadFormat (codec, callback) {
+    if (codec === 'git-raw') {
+      callback(null, require('ipld-git'))
+    } else {
+      callback(new Error('unable to load format ' + codec))
+    }
+  }
+})
+```
+
 ### `.put(node, options, callback)`
 
 > Store the given node of a recognized IPLD Format.

src/index.js

@@ -48,6 +48,10 @@ class IPLDResolver {
       }
     }
 
+    this.support.load = options.loadFormat || ((codec, callback) => {
+      callback(new Error(`No resolver found for codec "${codec}"`))
+    })
+
     this.support.rm = (multicodec) => {
       if (this.resolvers[multicodec]) {
         delete this.resolvers[multicodec]
@@ -99,26 +103,24 @@ class IPLDResolver {
 
     doUntil(
       (cb) => {
-        const r = this.resolvers[cid.codec]
-
-        if (!r) {
-          return cb(new Error('No resolver found for codec "' + cid.codec + '"'))
-        }
+        this._getFormat(cid.codec, (err, format) => {
+          if (err) return cb(err)
 
-        // get block
-        // use local resolver
-        // update path value
-        this.bs.get(cid, (err, block) => {
-          if (err) {
-            return cb(err)
-          }
-          r.resolver.resolve(block.data, path, (err, result) => {
+          // get block
+          // use local resolver
+          // update path value
+          this.bs.get(cid, (err, block) => {
             if (err) {
               return cb(err)
             }
-            value = result.value
-            path = result.remainderPath
-            cb()
+            format.resolver.resolve(block.data, path, (err, result) => {
+              if (err) {
+                return cb(err)
+              }
+              value = result.value
+              path = result.remainderPath
+              cb()
+            })
           })
         })
       },
@@ -182,17 +184,9 @@ class IPLDResolver {
         return callback(err)
       }
       map(blocks, (block, mapCallback) => {
-        const resolver = this.resolvers[block.cid.codec]
-        if (!resolver) {
-          return mapCallback(
-            new Error('No resolver found for codec "' + block.cid.codec + '"'))
-        }
-
-        resolver.util.deserialize(block.data, (err, deserialized) => {
-          if (err) {
-            return mapCallback(err)
-          }
-          return mapCallback(null, deserialized)
+        this._getFormat(block.cid.codec, (err, format) => {
+          if (err) return mapCallback(err)
+          format.util.deserialize(block.data, mapCallback)
         })
       },
       callback)
@@ -216,20 +210,20 @@ class IPLDResolver {
       return this._put(options.cid, node, callback)
     }
 
-    const r = this.resolvers[options.format]
-    if (!r) {
-      return callback(new Error('No resolver found for codec "' + options.format + '"'))
-    }
-    r.util.cid(node, options, (err, cid) => {
-      if (err) {
-        return callback(err)
-      }
+    this._getFormat(options.format, (err, format) => {
+      if (err) return callback(err)
 
-      if (options.onlyHash) {
-        return callback(null, cid)
-      }
+      format.util.cid(node, options, (err, cid) => {
+        if (err) {
+          return callback(err)
+        }
 
-      this._put(cid, node, callback)
+        if (options.onlyHash) {
+          return callback(null, cid)
+        }
+
+        this._put(cid, node, callback)
+      })
     })
   }
 
@@ -245,15 +239,14 @@ class IPLDResolver {
 
     if (!options.recursive) {
       p = pullDeferSource()
-      const r = this.resolvers[cid.codec]
-      if (!r) {
-        p.abort(new Error('No resolver found for codec "' + cid.codec + '"'))
-        return p
-      }
 
       waterfall([
-        (cb) => this.bs.get(cid, cb),
-        (block, cb) => r.resolver.tree(block.data, cb)
+        (cb) => this._getFormat(cid.codec, cb),
+        (format, cb) => this.bs.get(cid, (err, block) => {
+          if (err) return cb(err)
+          cb(null, format, block)
+        }),
+        (format, block, cb) => format.resolver.tree(block.data, cb)
       ], (err, paths) => {
         if (err) {
           p.abort(err)
@@ -280,20 +273,19 @@ class IPLDResolver {
 
           const deferred = pullDeferSource()
           const cid = el.cid
-          const r = this.resolvers[cid.codec]
-          if (!r) {
-            deferred.abort(new Error('No resolver found for codec "' + cid.codec + '"'))
-            return deferred
-          }
 
           waterfall([
-            (cb) => this.bs.get(el.cid, cb),
-            (block, cb) => r.resolver.tree(block.data, (err, paths) => {
+            (cb) => this._getFormat(cid.codec, cb),
+            (format, cb) => this.bs.get(cid, (err, block) => {
+              if (err) return cb(err)
+              cb(null, format, block)
+            }),
+            (format, block, cb) => format.resolver.tree(block.data, (err, paths) => {
               if (err) {
                 return cb(err)
               }
               map(paths, (p, cb) => {
-                r.resolver.isLink(block.data, p, (err, link) => {
+                format.resolver.isLink(block.data, p, (err, link) => {
                   if (err) {
                     return cb(err)
                   }
@@ -356,38 +348,42 @@ class IPLDResolver {
   /*           */
 
   _get (cid, callback) {
-    const r = this.resolvers[cid.codec]
-    if (!r) {
-      return callback(new Error('No resolver found for codec "' + cid.codec + '"'))
-    }
-
     waterfall([
-      (cb) => this.bs.get(cid, cb),
-      (block, cb) => {
-        if (r) {
-          r.util.deserialize(block.data, (err, deserialized) => {
-            if (err) {
-              return cb(err)
-            }
-            cb(null, deserialized)
-          })
-        } else { // multicodec unknown, send back raw data
-          cb(null, block.data)
-        }
+      (cb) => this._getFormat(cid.codec, cb),
+      (format, cb) => this.bs.get(cid, (err, block) => {
+        if (err) return cb(err)
+        cb(null, format, block)
+      }),
+      (format, block, cb) => {
+        format.util.deserialize(block.data, (err, deserialized) => {
+          if (err) {
+            return cb(err)
+          }
+          cb(null, deserialized)
+        })
       }
     ], callback)
   }
 
+  _getFormat (codec, callback) {
+    if (this.resolvers[codec]) {
+      return callback(null, this.resolvers[codec])
+    }
+
+    // If not supported, attempt to dynamically load this format
+    this.support.load(codec, (err, format) => {
+      if (err) return callback(err)
+      this.resolvers[codec] = format
+      callback(null, format)
+    })
+  }
+
   _put (cid, node, callback) {
     callback = callback || noop
 
-    const r = this.resolvers[cid.codec]
-    if (!r) {
-      return callback(new Error('No resolver found for codec "' + cid.codec + '"'))
-    }
-
     waterfall([
-      (cb) => r.util.serialize(node, cb),
+      (cb) => this._getFormat(cid.codec, cb),
+      (format, cb) => format.util.serialize(node, cb),
       (buf, cb) => this.bs.put(new Block(buf, cid), cb)
     ], (err) => {
       if (err) {
@@ -425,7 +421,7 @@ IPLDResolver.defaultOptions = {
 }
 
 /**
- * Create an IPLD resolver with an inmemory blockservice and
+ * Create an IPLD resolver with an in memory blockservice and
  * repo.
  *
  * @param {function(Error, IPLDResolver)} callback

test/browser.js

@@ -38,6 +38,7 @@ describe('Browser', () => {
   })
 
   require('./basics')(repo)
+  require('./format-support')(repo)
   require('./ipld-dag-pb')(repo)
   require('./ipld-dag-cbor')(repo)
   require('./ipld-git')(repo)

test/format-support.js

@@ -0,0 +1,83 @@
+/* eslint-env mocha */
+'use strict'
+
+const chai = require('chai')
+const dirtyChai = require('dirty-chai')
+const expect = chai.expect
+chai.use(dirtyChai)
+const BlockService = require('ipfs-block-service')
+const dagCBOR = require('ipld-dag-cbor')
+
+const IPLDResolver = require('../src')
+
+module.exports = (repo) => {
+  describe('IPLD format support', () => {
+    let data, cid
+
+    before((done) => {
+      const bs = new BlockService(repo)
+      const resolver = new IPLDResolver({ blockService: bs })
+
+      data = { now: Date.now() }
+
+      dagCBOR.util.cid(data, (err, c) => {
+        expect(err).to.not.exist()
+        cid = c
+        resolver.put(data, { cid }, done)
+      })
+    })
+
+    describe('Dynamic format loading', () => {
+      it('should fail to dynamically load format', (done) => {
+        const bs = new BlockService(repo)
+        const resolver = new IPLDResolver({
+          blockService: bs,
+          formats: []
+        })
+
+        resolver.get(cid, '/', (err) => {
+          expect(err).to.exist()
+          expect(err.message).to.equal('No resolver found for codec "dag-cbor"')
+          done()
+        })
+      })
+
+      it('should fail to dynamically load format via loadFormat option', (done) => {
+        const errMsg = 'BOOM' + Date.now()
+        const bs = new BlockService(repo)
+        const resolver = new IPLDResolver({
+          blockService: bs,
+          formats: [],
+          loadFormat (codec, callback) {
+            if (codec !== 'dag-cbor') return callback(new Error('unexpected codec'))
+            setTimeout(() => callback(new Error(errMsg)))
+          }
+        })
+
+        resolver.get(cid, '/', (err) => {
+          expect(err).to.exist()
+          expect(err.message).to.equal(errMsg)
+          done()
+        })
+      })
+
+      it('should dynamically load missing format', (done) => {
+        const bs = new BlockService(repo)
+        const resolver = new IPLDResolver({
+          blockService: bs,
+          formats: [],
+          loadFormat (codec, callback) {
+            if (codec !== 'dag-cbor') return callback(new Error('unexpected codec'))
+            setTimeout(() => callback(null, dagCBOR))
+          }
+        })
+
+        resolver.get(cid, '/', (err, result) => {
+          expect(err).to.not.exist()
+          expect(result.value).to.eql(data)
+          done()
+        })
+      })
+    })
+  })
+}

test/node.js

@@ -27,6 +27,7 @@ describe('Node.js', () => {
   })
 
   require('./basics')(repo)
+  require('./format-support')(repo)
   require('./ipld-dag-pb')(repo)
   require('./ipld-dag-cbor')(repo)
   require('./ipld-git')(repo)