Skip to content
This repository was archived by the owner on Mar 23, 2023. It is now read-only.

Commit 6f829db

Browse files
achingbrainachingbrain
authored
feat: split .query into .query and .queryKeys (#59)
Applies changes from ipfs/interface-datastore/pull/87 and removes redundant utilities.
1 parent 7ad48bc commit 6f829db

12 files changed

+205
-66
lines changed

‎package.json

+6-1
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,12 @@
4545
},
4646
"dependencies": {
4747
"debug": "^4.1.1",
48-
"interface-datastore": "^3.0.1"
48+
"interface-datastore": "^4.0.0",
49+
"it-filter": "^1.0.2",
50+
"it-map": "^1.0.5",
51+
"it-merge": "^1.0.1",
52+
"it-take": "^1.0.1",
53+
"uint8arrays": "^2.1.5"
4954
},
5055
"engines": {
5156
"node": ">=12.0.0"

‎src/keytransform.js

+19-5
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
11
'use strict'
22

3-
const { Adapter, utils } = require('interface-datastore')
4-
const map = utils.map
3+
const { Adapter } = require('interface-datastore')
4+
const map = require('it-map')
5+
56
/**
67
* @typedef {import('interface-datastore').Datastore} Datastore
78
* @typedef {import('interface-datastore').Options} Options
89
* @typedef {import('interface-datastore').Batch} Batch
910
* @typedef {import('interface-datastore').Query} Query
11+
* @typedef {import('interface-datastore').KeyQuery} KeyQuery
1012
* @typedef {import('interface-datastore').Key} Key
1113
* @typedef {import('./types').KeyTransform} KeyTransform
1214
*/
@@ -90,9 +92,21 @@ class KeyTransformDatastore extends Adapter {
9092
* @param {Options} [options]
9193
*/
9294
query (q, options) {
93-
return map(this.child.query(q, options), e => {
94-
e.key = this.transform.invert(e.key)
95-
return e
95+
return map(this.child.query(q, options), ({ key, value }) => {
96+
return {
97+
key: this.transform.invert(key),
98+
value
99+
}
100+
})
101+
}
102+
103+
/**
104+
* @param {KeyQuery} q
105+
* @param {Options} [options]
106+
*/
107+
queryKeys (q, options) {
108+
return map(this.child.queryKeys(q, options), key => {
109+
return this.transform.invert(key)
96110
})
97111
}
98112

‎src/mount.js

+44-20
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,13 @@
33

44
const {
55
Adapter, Key, Errors, utils: {
6-
filter,
7-
take,
86
sortAll,
97
replaceStartWith
108
}
119
} = require('interface-datastore')
10+
const filter = require('it-filter')
11+
const take = require('it-take')
12+
const merge = require('it-merge')
1213

1314
const Keytransform = require('./keytransform')
1415

@@ -17,7 +18,8 @@ const Keytransform = require('./keytransform')
1718
* @typedef {import('interface-datastore').Options} Options
1819
* @typedef {import('interface-datastore').Batch} Batch
1920
* @typedef {import('interface-datastore').Query} Query
20-
* @typedef {import('interface-datastore').Pair} Pair
21+
* @typedef {import('interface-datastore').KeyQuery} KeyQuery
22+
* @typedef {import('./types').KeyTransform} KeyTransform
2123
*/
2224

2325
/**
@@ -27,13 +29,12 @@ const Keytransform = require('./keytransform')
2729

2830
/**
2931
* A datastore that can combine multiple stores inside various
30-
* key prefixs.
32+
* key prefixes
3133
*
3234
* @implements {Datastore}
3335
*/
3436
class MountDatastore extends Adapter {
3537
/**
36-
*
3738
* @param {Array<{prefix: Key, datastore: Datastore}>} mounts
3839
*/
3940
constructor (mounts) {
@@ -47,7 +48,7 @@ class MountDatastore extends Adapter {
4748
}
4849

4950
/**
50-
* Lookup the matching datastore for the given key.
51+
* Lookup the matching datastore for the given key
5152
*
5253
* @private
5354
* @param {Key} key
@@ -186,12 +187,11 @@ class MountDatastore extends Adapter {
186187

187188
return ks.query({
188189
prefix: prefix,
189-
filters: q.filters,
190-
keysOnly: q.keysOnly
190+
filters: q.filters
191191
}, options)
192192
})
193193

194-
let it = _many(qs)
194+
let it = merge(...qs)
195195
if (q.filters) q.filters.forEach(f => { it = filter(it, f) })
196196
if (q.orders) q.orders.forEach(o => { it = sortAll(it, o) })
197197
if (q.offset != null) {
@@ -202,20 +202,44 @@ class MountDatastore extends Adapter {
202202

203203
return it
204204
}
205-
}
206205

207-
/**
208-
* @param {ArrayLike<AwaitIterable<Pair>>} iterable
209-
* @returns {AsyncIterable<Pair>}
210-
*/
211-
function _many (iterable) {
212-
return (async function * () {
213-
for (let i = 0; i < iterable.length; i++) {
214-
for await (const v of iterable[i]) {
215-
yield v
206+
/**
207+
* @param {KeyQuery} q
208+
* @param {Options} [options]
209+
*/
210+
queryKeys (q, options) {
211+
const qs = this.mounts.map(m => {
212+
const ks = new Keytransform(m.datastore, {
213+
convert: (key) => {
214+
throw new Error('should never be called')
215+
},
216+
invert: (key) => {
217+
return m.prefix.child(key)
218+
}
219+
})
220+
221+
let prefix
222+
if (q.prefix != null) {
223+
prefix = replaceStartWith(q.prefix, m.prefix.toString())
216224
}
225+
226+
return ks.queryKeys({
227+
prefix: prefix,
228+
filters: q.filters
229+
}, options)
230+
})
231+
232+
let it = merge(...qs)
233+
if (q.filters) q.filters.forEach(f => { it = filter(it, f) })
234+
if (q.orders) q.orders.forEach(o => { it = sortAll(it, o) })
235+
if (q.offset != null) {
236+
let i = 0
237+
it = filter(it, () => i++ >= /** @type {number} */ (q.offset))
217238
}
218-
})()
239+
if (q.limit != null) it = take(it, q.limit)
240+
241+
return it
242+
}
219243
}
220244

221245
module.exports = MountDatastore

‎src/namespace.js

+14
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ const KeytransformDatastore = require('./keytransform')
55
/**
66
* @typedef {import('interface-datastore').Datastore} Datastore
77
* @typedef {import('interface-datastore').Query} Query
8+
* @typedef {import('interface-datastore').KeyQuery} KeyQuery
89
* @typedef {import('interface-datastore').Options} Options
910
* @typedef {import('interface-datastore').Batch} Batch
1011
* @typedef {import('./types').KeyTransform} KeyTransform
@@ -57,6 +58,19 @@ class NamespaceDatastore extends KeytransformDatastore {
5758
}
5859
return super.query(q, options)
5960
}
61+
62+
/**
63+
* @param {KeyQuery} q
64+
* @param {Options} [options]
65+
*/
66+
queryKeys (q, options) {
67+
if (q.prefix && this.prefix.toString() !== '/') {
68+
return super.queryKeys(Object.assign({}, q, {
69+
prefix: this.prefix.child(new Key(q.prefix)).toString()
70+
}))
71+
}
72+
return super.queryKeys(q, options)
73+
}
6074
}
6175

6276
module.exports = NamespaceDatastore

‎src/shard.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
'use strict'
22

3-
const { Key, utils: { utf8Decoder } } = require('interface-datastore')
3+
const { Key } = require('interface-datastore')
44
const readme = require('./shard-readme')
55

66
/**
@@ -149,7 +149,7 @@ const readShardFun = async (path, store) => {
149149
// @ts-ignore
150150
const get = typeof store.getRaw === 'function' ? store.getRaw.bind(store) : store.get.bind(store)
151151
const res = await get(key)
152-
return parseShardFun(utf8Decoder.decode(res || '').trim())
152+
return parseShardFun(new TextDecoder().decode(res || '').trim())
153153
}
154154

155155
module.exports = {

‎src/sharding.js

+85-17
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
'use strict'
22

3-
const { Adapter, Key, utils: { utf8Encoder }, Errors } = require('interface-datastore')
3+
const { Adapter, Key, Errors } = require('interface-datastore')
44
const sh = require('./shard')
55
const KeytransformStore = require('./keytransform')
66

@@ -11,6 +11,11 @@ const shardReadmeKey = new Key(sh.README_FN)
1111
* @typedef {import('interface-datastore').Options} Options
1212
* @typedef {import('interface-datastore').Batch} Batch
1313
* @typedef {import('interface-datastore').Query} Query
14+
* @typedef {import('interface-datastore').QueryFilter} QueryFilter
15+
* @typedef {import('interface-datastore').QueryOrder} QueryOrder
16+
* @typedef {import('interface-datastore').KeyQuery} KeyQuery
17+
* @typedef {import('interface-datastore').KeyQueryFilter} KeyQueryFilter
18+
* @typedef {import('interface-datastore').KeyQueryOrder} KeyQueryOrder
1419
* @typedef {import('interface-datastore').Pair} Pair
1520
* @typedef {import('./types').Shard} Shard
1621
*
@@ -107,8 +112,8 @@ class ShardingDatastore extends Adapter {
107112
// @ts-ignore i have no idea what putRaw is or saw any implementation
108113
const put = typeof store.putRaw === 'function' ? store.putRaw.bind(store) : store.put.bind(store)
109114
await Promise.all([
110-
put(shardKey, utf8Encoder.encode(shard.toString() + '\n')),
111-
put(shardReadmeKey, utf8Encoder.encode(sh.readme))
115+
put(shardKey, new TextEncoder().encode(shard.toString() + '\n')),
116+
put(shardReadmeKey, new TextEncoder().encode(sh.readme))
112117
])
113118

114119
return shard
@@ -167,15 +172,15 @@ class ShardingDatastore extends Adapter {
167172
*/
168173
query (q, options) {
169174
const tq = {
170-
keysOnly: q.keysOnly,
171175
offset: q.offset,
172176
limit: q.limit,
173-
/** @type Array<(items: Pair[]) => Await<Pair[]>> */
177+
/** @type {QueryOrder[]} */
174178
orders: [],
179+
/** @type {QueryFilter[]} */
175180
filters: [
176-
/** @type {(item: Pair) => boolean} */
181+
/** @type {QueryFilter} */
177182
e => e.key.toString() !== shardKey.toString(),
178-
/** @type {(item: Pair) => boolean} */
183+
/** @type {QueryFilter} */
179184
e => e.key.toString() !== shardReadmeKey.toString()
180185
]
181186
}
@@ -188,27 +193,90 @@ class ShardingDatastore extends Adapter {
188193
}
189194

190195
if (q.filters != null) {
191-
// @ts-ignore - can't find a way to easily type this
192-
const filters = q.filters.map((f) => (e) => {
193-
return f(Object.assign({}, e, {
194-
key: this._invertKey(e.key)
195-
}))
196+
const filters = q.filters.map(f => {
197+
/** @type {QueryFilter} */
198+
const filter = ({ key, value }) => {
199+
return f({
200+
key: this._invertKey(key),
201+
value
202+
})
203+
}
204+
205+
return filter
196206
})
197207
tq.filters = tq.filters.concat(filters)
198208
}
199209

200210
if (q.orders != null) {
201-
tq.orders = q.orders.map((o) => async (res) => {
202-
res.forEach((e) => { e.key = this._invertKey(e.key) })
203-
const ordered = await o(res)
204-
ordered.forEach((e) => { e.key = this._convertKey(e.key) })
205-
return ordered
211+
tq.orders = q.orders.map(o => {
212+
/** @type {QueryOrder} */
213+
const order = (a, b) => {
214+
return o({
215+
key: this._invertKey(a.key),
216+
value: a.value
217+
}, {
218+
key: this._invertKey(b.key),
219+
value: b.value
220+
})
221+
}
222+
223+
return order
206224
})
207225
}
208226

209227
return this.child.query(tq, options)
210228
}
211229

230+
/**
231+
* @param {KeyQuery} q
232+
* @param {Options} [options]
233+
*/
234+
queryKeys (q, options) {
235+
const tq = {
236+
offset: q.offset,
237+
limit: q.limit,
238+
/** @type {KeyQueryOrder[]} */
239+
orders: [],
240+
/** @type {KeyQueryFilter[]} */
241+
filters: [
242+
/** @type {KeyQueryFilter} */
243+
key => key.toString() !== shardKey.toString(),
244+
/** @type {KeyQueryFilter} */
245+
key => key.toString() !== shardReadmeKey.toString()
246+
]
247+
}
248+
249+
const { prefix } = q
250+
if (prefix != null) {
251+
tq.filters.push((key) => {
252+
return this._invertKey(key).toString().startsWith(prefix)
253+
})
254+
}
255+
256+
if (q.filters != null) {
257+
const filters = q.filters.map(f => {
258+
/** @type {KeyQueryFilter} */
259+
const filter = (key) => {
260+
return f(this._invertKey(key))
261+
}
262+
263+
return filter
264+
})
265+
tq.filters = tq.filters.concat(filters)
266+
}
267+
268+
if (q.orders != null) {
269+
tq.orders = q.orders.map(o => {
270+
/** @type {KeyQueryOrder} */
271+
const order = (a, b) => o(this._invertKey(a), this._invertKey(b))
272+
273+
return order
274+
})
275+
}
276+
277+
return this.child.queryKeys(tq, options)
278+
}
279+
212280
close () {
213281
return this.child.close()
214282
}

‎src/tiered.js

+9
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ const log = require('debug')('datastore:core:tiered')
77
* @typedef {import('interface-datastore').Options} Options
88
* @typedef {import('interface-datastore').Batch} Batch
99
* @typedef {import('interface-datastore').Query} Query
10+
* @typedef {import('interface-datastore').KeyQuery} KeyQuery
1011
* @typedef {import('interface-datastore').Key} Key
1112
*/
1213

@@ -121,6 +122,14 @@ class TieredDatastore extends Adapter {
121122
query (q, options) {
122123
return this.stores[this.stores.length - 1].query(q, options)
123124
}
125+
126+
/**
127+
* @param {KeyQuery} q
128+
* @param {Options} [options]
129+
*/
130+
queryKeys (q, options) {
131+
return this.stores[this.stores.length - 1].queryKeys(q, options)
132+
}
124133
}
125134

126135
module.exports = TieredDatastore

0 commit comments

Comments
 (0)