feat: split .query into .query and .queryKeys

package.json

@@ -45,7 +45,12 @@
   },
   "dependencies": {
     "debug": "^4.1.1",
-    "interface-datastore": "^3.0.1"
+    "interface-datastore": "^4.0.0",
+    "it-filter": "^1.0.2",
+    "it-map": "^1.0.5",
+    "it-merge": "^1.0.1",
+    "it-take": "^1.0.1",
+    "uint8arrays": "^2.1.5"
   },
   "engines": {
     "node": ">=12.0.0"

src/keytransform.js

@@ -1,12 +1,14 @@
 'use strict'
 
-const { Adapter, utils } = require('interface-datastore')
-const map = utils.map
+const { Adapter } = require('interface-datastore')
+const map = require('it-map')
+
 /**
  * @typedef {import('interface-datastore').Datastore} Datastore
  * @typedef {import('interface-datastore').Options} Options
  * @typedef {import('interface-datastore').Batch} Batch
  * @typedef {import('interface-datastore').Query} Query
+ * @typedef {import('interface-datastore').KeyQuery} KeyQuery
  * @typedef {import('interface-datastore').Key} Key
  * @typedef {import('./types').KeyTransform} KeyTransform
  */
@@ -90,9 +92,21 @@ class KeyTransformDatastore extends Adapter {
    * @param {Options} [options]
    */
   query (q, options) {
-    return map(this.child.query(q, options), e => {
-      e.key = this.transform.invert(e.key)
-      return e
+    return map(this.child.query(q, options), ({ key, value }) => {
+      return {
+        key: this.transform.invert(key),
+        value
+      }
+    })
+  }
+
+  /**
+   * @param {KeyQuery} q
+   * @param {Options} [options]
+   */
+  queryKeys (q, options) {
+    return map(this.child.queryKeys(q, options), key => {
+      return this.transform.invert(key)
     })
   }
 

src/mount.js

@@ -3,12 +3,13 @@
 
 const {
   Adapter, Key, Errors, utils: {
-    filter,
-    take,
     sortAll,
     replaceStartWith
   }
 } = require('interface-datastore')
+const filter = require('it-filter')
+const take = require('it-take')
+const merge = require('it-merge')
 
 const Keytransform = require('./keytransform')
 
@@ -17,7 +18,8 @@ const Keytransform = require('./keytransform')
  * @typedef {import('interface-datastore').Options} Options
  * @typedef {import('interface-datastore').Batch} Batch
  * @typedef {import('interface-datastore').Query} Query
- * @typedef {import('interface-datastore').Pair} Pair
+ * @typedef {import('interface-datastore').KeyQuery} KeyQuery
+ * @typedef {import('./types').KeyTransform} KeyTransform
  */
 
 /**
@@ -27,13 +29,12 @@ const Keytransform = require('./keytransform')
 
 /**
  * A datastore that can combine multiple stores inside various
- * key prefixs.
+ * key prefixes
  *
  * @implements {Datastore}
  */
 class MountDatastore extends Adapter {
   /**
-   *
    * @param {Array<{prefix: Key, datastore: Datastore}>} mounts
    */
   constructor (mounts) {
@@ -47,7 +48,7 @@ class MountDatastore extends Adapter {
   }
 
   /**
-   * Lookup the matching datastore for the given key.
+   * Lookup the matching datastore for the given key
    *
    * @private
    * @param {Key} key
@@ -186,12 +187,11 @@ class MountDatastore extends Adapter {
 
       return ks.query({
         prefix: prefix,
-        filters: q.filters,
-        keysOnly: q.keysOnly
+        filters: q.filters
       }, options)
     })
 
-    let it = _many(qs)
+    let it = merge(...qs)
     if (q.filters) q.filters.forEach(f => { it = filter(it, f) })
     if (q.orders) q.orders.forEach(o => { it = sortAll(it, o) })
     if (q.offset != null) {
@@ -202,20 +202,44 @@ class MountDatastore extends Adapter {
 
     return it
   }
-}
 
-/**
- * @param {ArrayLike<AwaitIterable<Pair>>} iterable
- * @returns {AsyncIterable<Pair>}
- */
-function _many (iterable) {
-  return (async function * () {
-    for (let i = 0; i < iterable.length; i++) {
-      for await (const v of iterable[i]) {
-        yield v
+  /**
+   * @param {KeyQuery} q
+   * @param {Options} [options]
+   */
+  queryKeys (q, options) {
+    const qs = this.mounts.map(m => {
+      const ks = new Keytransform(m.datastore, {
+        convert: (key) => {
+          throw new Error('should never be called')
+        },
+        invert: (key) => {
+          return m.prefix.child(key)
+        }
+      })
+
+      let prefix
+      if (q.prefix != null) {
+        prefix = replaceStartWith(q.prefix, m.prefix.toString())
       }
+
+      return ks.queryKeys({
+        prefix: prefix,
+        filters: q.filters
+      }, options)
+    })
+
+    let it = merge(...qs)
+    if (q.filters) q.filters.forEach(f => { it = filter(it, f) })
+    if (q.orders) q.orders.forEach(o => { it = sortAll(it, o) })
+    if (q.offset != null) {
+      let i = 0
+      it = filter(it, () => i++ >= /** @type {number} */ (q.offset))
     }
-  })()
+    if (q.limit != null) it = take(it, q.limit)
+
+    return it
+  }
 }
 
 module.exports = MountDatastore

src/namespace.js

@@ -5,6 +5,7 @@ const KeytransformDatastore = require('./keytransform')
 /**
  * @typedef {import('interface-datastore').Datastore} Datastore
  * @typedef {import('interface-datastore').Query} Query
+ * @typedef {import('interface-datastore').KeyQuery} KeyQuery
  * @typedef {import('interface-datastore').Options} Options
  * @typedef {import('interface-datastore').Batch} Batch
  * @typedef {import('./types').KeyTransform} KeyTransform
@@ -57,6 +58,19 @@ class NamespaceDatastore extends KeytransformDatastore {
     }
     return super.query(q, options)
   }
+
+  /**
+   * @param {KeyQuery} q
+   * @param {Options} [options]
+   */
+  queryKeys (q, options) {
+    if (q.prefix && this.prefix.toString() !== '/') {
+      return super.queryKeys(Object.assign({}, q, {
+        prefix: this.prefix.child(new Key(q.prefix)).toString()
+      }))
+    }
+    return super.queryKeys(q, options)
+  }
 }
 
 module.exports = NamespaceDatastore

src/shard.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { Key, utils: { utf8Decoder } } = require('interface-datastore')
+const { Key } = require('interface-datastore')
 const readme = require('./shard-readme')
 
 /**
@@ -149,7 +149,7 @@ const readShardFun = async (path, store) => {
   // @ts-ignore
   const get = typeof store.getRaw === 'function' ? store.getRaw.bind(store) : store.get.bind(store)
   const res = await get(key)
-  return parseShardFun(utf8Decoder.decode(res || '').trim())
+  return parseShardFun(new TextDecoder().decode(res || '').trim())
 }
 
 module.exports = {

src/sharding.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { Adapter, Key, utils: { utf8Encoder }, Errors } = require('interface-datastore')
+const { Adapter, Key, Errors } = require('interface-datastore')
 const sh = require('./shard')
 const KeytransformStore = require('./keytransform')
 
@@ -11,6 +11,11 @@ const shardReadmeKey = new Key(sh.README_FN)
  * @typedef {import('interface-datastore').Options} Options
  * @typedef {import('interface-datastore').Batch} Batch
  * @typedef {import('interface-datastore').Query} Query
+ * @typedef {import('interface-datastore').QueryFilter} QueryFilter
+ * @typedef {import('interface-datastore').QueryOrder} QueryOrder
+ * @typedef {import('interface-datastore').KeyQuery} KeyQuery
+ * @typedef {import('interface-datastore').KeyQueryFilter} KeyQueryFilter
+ * @typedef {import('interface-datastore').KeyQueryOrder} KeyQueryOrder
  * @typedef {import('interface-datastore').Pair} Pair
  * @typedef {import('./types').Shard} Shard
  *
@@ -107,8 +112,8 @@ class ShardingDatastore extends Adapter {
       // @ts-ignore i have no idea what putRaw is or saw any implementation
       const put = typeof store.putRaw === 'function' ? store.putRaw.bind(store) : store.put.bind(store)
       await Promise.all([
-        put(shardKey, utf8Encoder.encode(shard.toString() + '\n')),
-        put(shardReadmeKey, utf8Encoder.encode(sh.readme))
+        put(shardKey, new TextEncoder().encode(shard.toString() + '\n')),
+        put(shardReadmeKey, new TextEncoder().encode(sh.readme))
       ])
 
       return shard
@@ -167,15 +172,15 @@ class ShardingDatastore extends Adapter {
    */
   query (q, options) {
     const tq = {
-      keysOnly: q.keysOnly,
       offset: q.offset,
       limit: q.limit,
-      /** @type Array<(items: Pair[]) => Await<Pair[]>> */
+      /** @type {QueryOrder[]} */
       orders: [],
+      /** @type {QueryFilter[]} */
       filters: [
-        /** @type {(item: Pair) => boolean} */
+        /** @type {QueryFilter} */
         e => e.key.toString() !== shardKey.toString(),
-        /** @type {(item: Pair) => boolean} */
+        /** @type {QueryFilter} */
         e => e.key.toString() !== shardReadmeKey.toString()
       ]
     }
@@ -188,27 +193,90 @@ class ShardingDatastore extends Adapter {
     }
 
     if (q.filters != null) {
-      // @ts-ignore - can't find a way to easily type this
-      const filters = q.filters.map((f) => (e) => {
-        return f(Object.assign({}, e, {
-          key: this._invertKey(e.key)
-        }))
+      const filters = q.filters.map(f => {
+        /** @type {QueryFilter} */
+        const filter = ({ key, value }) => {
+          return f({
+            key: this._invertKey(key),
+            value
+          })
+        }
+
+        return filter
       })
       tq.filters = tq.filters.concat(filters)
     }
 
     if (q.orders != null) {
-      tq.orders = q.orders.map((o) => async (res) => {
-        res.forEach((e) => { e.key = this._invertKey(e.key) })
-        const ordered = await o(res)
-        ordered.forEach((e) => { e.key = this._convertKey(e.key) })
-        return ordered
+      tq.orders = q.orders.map(o => {
+        /** @type {QueryOrder} */
+        const order = (a, b) => {
+          return o({
+            key: this._invertKey(a.key),
+            value: a.value
+          }, {
+            key: this._invertKey(b.key),
+            value: b.value
+          })
+        }
+
+        return order
       })
     }
 
     return this.child.query(tq, options)
   }
 
+  /**
+   * @param {KeyQuery} q
+   * @param {Options} [options]
+   */
+  queryKeys (q, options) {
+    const tq = {
+      offset: q.offset,
+      limit: q.limit,
+      /** @type {KeyQueryOrder[]} */
+      orders: [],
+      /** @type {KeyQueryFilter[]} */
+      filters: [
+        /** @type {KeyQueryFilter} */
+        key => key.toString() !== shardKey.toString(),
+        /** @type {KeyQueryFilter} */
+        key => key.toString() !== shardReadmeKey.toString()
+      ]
+    }
+
+    const { prefix } = q
+    if (prefix != null) {
+      tq.filters.push((key) => {
+        return this._invertKey(key).toString().startsWith(prefix)
+      })
+    }
+
+    if (q.filters != null) {
+      const filters = q.filters.map(f => {
+        /** @type {KeyQueryFilter} */
+        const filter = (key) => {
+          return f(this._invertKey(key))
+        }
+
+        return filter
+      })
+      tq.filters = tq.filters.concat(filters)
+    }
+
+    if (q.orders != null) {
+      tq.orders = q.orders.map(o => {
+        /** @type {KeyQueryOrder} */
+        const order = (a, b) => o(this._invertKey(a), this._invertKey(b))
+
+        return order
+      })
+    }
+
+    return this.child.queryKeys(tq, options)
+  }
+
   close () {
     return this.child.close()
   }

src/tiered.js

@@ -7,6 +7,7 @@ const log = require('debug')('datastore:core:tiered')
  * @typedef {import('interface-datastore').Options} Options
  * @typedef {import('interface-datastore').Batch} Batch
  * @typedef {import('interface-datastore').Query} Query
+ * @typedef {import('interface-datastore').KeyQuery} KeyQuery
  * @typedef {import('interface-datastore').Key} Key
  */
 
@@ -121,6 +122,14 @@ class TieredDatastore extends Adapter {
   query (q, options) {
     return this.stores[this.stores.length - 1].query(q, options)
   }
+
+  /**
+   * @param {KeyQuery} q
+   * @param {Options} [options]
+   */
+  queryKeys (q, options) {
+    return this.stores[this.stores.length - 1].queryKeys(q, options)
+  }
 }
 
 module.exports = TieredDatastore