Drop support of key & value types other than string and Buffer

Author: vweevers

README.md

@@ -1,6 +1,6 @@
 # level-js
 
-> An [`abstract-leveldown`][abstract-leveldown] compliant store on top of [IndexedDB][indexeddb], which is in turn implemented on top of [LevelDB][leveldb] which brings this whole shebang full circle.
+> An [`abstract-leveldown`][abstract-leveldown] compliant store on top of [IndexedDB][indexeddb], which is ~~in turn implemented on top of [LevelDB][leveldb] which brings this whole shebang full circle~~.
 
 [![level badge][level-badge]][awesome]
 [![npm](https://img.shields.io/npm/v/level-js.svg?label=&logo=npm)](https://www.npmjs.com/package/level-js)
@@ -35,13 +35,11 @@ Here are the goals of `level-js`:
 
 - Store large amounts of data in modern browsers
 - Pass the full [`abstract-leveldown`][abstract-leveldown] test suite
-- Support [`Buffer`][buffer] keys and values
-- Support all key types of IndexedDB Second Edition
-- Support all value types of the [structured clone algorithm][structured-clone-algorithm] except for `null` and `undefined`
+- Support string and [`Buffer`][buffer] keys and values
 - Be as fast as possible
-- Sync with [multilevel](https://github.com/juliangruber/multilevel) over ASCII or binary transports.
+- ~~Sync with [multilevel](https://github.com/juliangruber/multilevel) over ASCII or binary transports.~~
 
-Being `abstract-leveldown` compliant means you can use many of the [Level modules][awesome] on top of this library. For some demos of it working, see [**@brycebaril**](https://github.com/brycebaril)'s presentation [Path of the NodeBases Jedi](http://brycebaril.github.io/nodebase_jedi/#/vanilla).
+Being `abstract-leveldown` compliant means you can use many of the [Level modules][awesome] on top of this library.
 
 ## Example
 
@@ -80,90 +78,63 @@ const value = await db.get('hello')
 
 ## Type Support
 
-Unlike [`leveldown`][leveldown], `level-js` does not stringify keys or values. This means that in addition to strings and Buffers you can store almost any JavaScript type without the need for [`encoding-down`][encoding-down].
+~~Unlike [`leveldown`][leveldown], `level-js` does not stringify keys or values. This means that in addition to strings and Buffers you can store almost any JavaScript type without the need for [`encoding-down`][encoding-down].~~
 
 ### Values
 
-All value types of the [structured clone algorithm][structured-clone-algorithm] are supported except for `null` and `undefined`. Depending on the environment, this includes:
+~~All value types of the [structured clone algorithm][structured-clone-algorithm] are supported except for `null` and `undefined`. Depending on the environment, this includes:~~
 
-- Number, including `NaN`, `Infinity` and `-Infinity`
-- String, Boolean, Date, RegExp, Array, Object
+- ~~Number, including `NaN`, `Infinity` and `-Infinity`~~
+- String, ~~Boolean, Date, RegExp, Array, Object~~
 - ArrayBuffer or a view thereof (typed arrays);
-- Map, Set, Blob, File, FileList, ImageData (limited support).
+- ~~Map, Set, Blob, File, FileList, ImageData (limited support).~~
 
 In addition `level-js` stores [`Buffer`][buffer] values without transformation. This works in all target environments because `Buffer` is a subclass of `Uint8Array`, meaning such values can be passed to `IndexedDB` as-is.
 
-When getting or iterating binary values, regardless of whether they were stored as a `Buffer`, `ArrayBuffer` or a view thereof, values will return as a `Buffer`. This behavior can be disabled, in which case `ArrayBuffer` returns as `ArrayBuffer`, typed arrays return as typed arrays and `Buffer` returns as `Uint8Array`:
+When getting or iterating binary values, regardless of whether they were stored as a `Buffer`, `ArrayBuffer` or a view thereof, values will return as a `Buffer`. This behavior can be disabled, in which case ~~`ArrayBuffer` returns as `ArrayBuffer`, typed arrays return as typed arrays and `Buffer` returns as `Uint8Array`~~:
 
 ```js
 db.get('key', { asBuffer: false })
 db.iterator({ valueAsBuffer: false })
 ```
 
-If the environment does not support a type, it will throw an error which `level-js` catches and passes to the callbacks of `put` or `batch`. For example, IE does not support typed array values. At the time of writing, Chrome is the only browser that supports all types listed above.
+~~If the environment does not support a type, it will throw an error which `level-js` catches and passes to the callbacks of `put` or `batch`. For example, IE does not support typed array values. At the time of writing, Chrome is the only browser that supports all types listed above.~~
 
 ### Keys
 
-All key types of IndexedDB Second Edition are supported. Depending on the environment, this includes:
+~~All key types of IndexedDB Second Edition are supported. Depending on the environment, this includes:~~
 
-- Number, including `Infinity` and `-Infinity`, but not `NaN`
-- Date, except invalid (`NaN`)
+- ~~Number, including `Infinity` and `-Infinity`, but not `NaN`~~
+- ~~Date, except invalid (`NaN`)~~
 - String
 - ArrayBuffer or a view thereof (typed arrays);
-- Array, except cyclical, empty and sparse arrays. Elements must be valid types themselves.
+- ~~Array, except cyclical, empty and sparse arrays. Elements must be valid types themselves.~~
 
-In addition you can use [`Buffer`][buffer] keys, giving `level-js` the same power as implementations like `leveldown` and `memdown`. When iterating binary keys, regardless of whether they were stored as `Buffer`, `ArrayBuffer` or a view thereof, keys will return as a `Buffer`. This behavior can be disabled, in which case binary keys will always return as `ArrayBuffer`:
+In addition you can use [`Buffer`][buffer] keys, giving `level-js` the same power as implementations like `leveldown` and `memdown`. When iterating binary keys, regardless of whether they were stored as `Buffer`, `ArrayBuffer` or a view thereof, keys will return as a `Buffer`. This behavior can be disabled, in which case ~~binary keys will always return as `ArrayBuffer`~~:
 
 ```js
 db.iterator({ keyAsBuffer: false })
 ```
 
 Note that this behavior is slightly different from values due to the way that IndexedDB works. IndexedDB stores binary _values_ using the structured clone algorithm, which preserves views, but it stores binary _keys_ as an array of octets, so that it is able to compare and sort differently typed keys.
 
-If the environment does not support a type, it will throw an error which `level-js` catches and passes to the callbacks of `get`, `put`, `del`, `batch` or an iterator. Exceptions are:
+~~If the environment does not support a type, it will throw an error which `level-js` catches and passes to the callbacks of `get`, `put`, `del`, `batch` or an iterator~~. Exceptions are:
 
 - `null` and `undefined`: rejected early by `abstract-leveldown`
-- Binary and array keys: if not supported by the environment, `level-js` falls back to `String(key)`.
+- Binary keys: if not supported by the environment, `level-js` falls back to `String(key)`. If (one of) your target environments does not support binary keys (like IE11), it is recommended to set the `bufferKeys` option of the constructor to `false` and then only use string keys. Otherwise, you risk losing data once the environment does land support for binary keys.
 
 ### Normalization
 
 If you desire normalization for keys and values (e.g. to stringify numbers), wrap `level-js` with [`encoding-down`][encoding-down]. Alternatively install [`level`][level] which conveniently bundles [`levelup`][levelup], `level-js` and `encoding-down`. Such an approach is also recommended if you want to achieve universal (isomorphic) behavior or to smooth over type differences between browsers. For example, you could have [`leveldown`][leveldown] in a backend and `level-js` in the frontend. The `level` package does exactly that.
 
-Another reason you might want to use `encoding-down` is that the structured clone algorithm, while rich in types, can be slower than `JSON.stringify`.
+~~Another reason you might want to use `encoding-down` is that the structured clone algorithm, while rich in types, can be slower than `JSON.stringify`.~~
 
 ### Sort Order
 
-Unless `level-js` is wrapped with [`encoding-down`][encoding-down], IndexedDB will sort your keys in the following order:
+~~Unless `level-js` is wrapped with [`encoding-down`][encoding-down], IndexedDB will sort your keys in the following order:~~
 
-1. number (numeric)
-2. date (numeric, by epoch offset)
 3. binary (bitwise)
 4. string (lexicographic)
-5. array (componentwise).
-
-You can take advantage of this fact with `levelup` streams. For example, if your keys are dates, you can select everything greater than a specific date (let's be happy and ignore timezones for a moment):
-
-```js
-const db = levelup(leveljs('time-db'))
-
-db.createReadStream({ gt: new Date('2019-01-01') })
-  .pipe(..)
-```
-
-Or if your keys are arrays, you can do things like:
-
-```js
-const db = levelup(leveljs('books-db'))
-
-await db.put(['Roald Dahl', 'Charlie and the Chocolate Factory'], {})
-await db.put(['Roald Dahl', 'Fantastic Mr Fox'], {})
-
-// Select all books by Roald Dahl
-db.createReadStream({ gt: ['Roald Dahl'], lt: ['Roald Dahl', '\xff'] })
-  .pipe(..)
-```
-
-To achieve this on other `abstract-leveldown` implementations, wrap them with [`encoding-down`][encoding-down] and [`charwise`][charwise] (or similar).
 
 #### Known Browser Issues
 
@@ -173,7 +144,7 @@ IE11 and Edge yield incorrect results for `{ gte: '' }` if the database contains
 
 For interoperability it is recommended to use `Buffer` as your binary type. While we recognize that Node.js core modules are moving towards supporting `ArrayBuffer` and views thereof, `Buffer` remains the primary binary type in the Level ecosystem.
 
-That said: if you want to `put()` an `ArrayBuffer` you can! Just know that it will come back as a `Buffer` by default. If you want to `get()` or iterate stored `ArrayBuffer` data as an `ArrayBuffer`, you have a few options. Without `encoding-down`:
+That said: if you want to `put()` an `ArrayBuffer` you can! Just know that it will come back as a `Buffer` by default. ~~If you want to `get()` or iterate stored `ArrayBuffer` data as an `ArrayBuffer`, you have a few options~~. Without `encoding-down`:
 
 ```js
 const db = levelup(leveljs('mydb'))
@@ -220,6 +191,7 @@ The optional `options` argument may contain:
 
 - `prefix` _(string, default: `'level-js-'`)_: Prefix for `IDBDatabase` name.
 - `version` _(string | number, default: `1`)_: The version to open the database with.
+- `bufferKeys`: ..
 
 See [`IDBFactory#open`](https://developer.mozilla.org/en-US/docs/Web/API/IDBFactory/open) for more details.
 
@@ -276,8 +248,6 @@ To sustain [`Level`](https://github.com/Level) and its activities, become a back
 
 [abstract-leveldown]: https://github.com/Level/abstract-leveldown
 
-[charwise]: https://github.com/dominictarr/charwise
-
 [levelup]: https://github.com/Level/levelup
 
 [leveldown]: https://github.com/Level/leveldown

index.js

@@ -7,7 +7,8 @@ module.exports = Level
 var AbstractLevelDOWN = require('abstract-leveldown').AbstractLevelDOWN
 var inherits = require('inherits')
 var Iterator = require('./iterator')
-var mixedToBuffer = require('./util/mixed-to-buffer')
+var serialize = require('./util/serialize')
+var deserialize = require('./util/deserialize')
 var setImmediate = require('./util/immediate')
 var support = require('./util/support')
 
@@ -25,13 +26,24 @@ function Level (location, opts) {
   this.location = location
   this.prefix = opts.prefix || DEFAULT_PREFIX
   this.version = parseInt(opts.version || 1, 10)
+
+  // Experimental, do not externally rely on this object yet.
+  // See Level/community#42.
+  this.supports = {
+    bufferKeys: opts.bufferKeys !== false && Level.binaryKeys,
+    permanence: true,
+    snapshots: true, // For now (#86)
+    seek: false, // #178
+    clear: false, // #175
+    createIfMissing: false,
+    errorIfExists: false
+  }
 }
 
 inherits(Level, AbstractLevelDOWN)
 
 // Detect binary and array key support (IndexedDB Second Edition)
 Level.binaryKeys = support.binaryKeys(indexedDB)
-Level.arrayKeys = support.arrayKeys(indexedDB)
 
 Level.prototype._open = function (options, callback) {
   var req = indexedDB.open(this.prefix + this.location, this.version)
@@ -93,11 +105,7 @@ Level.prototype._get = function (key, options, callback) {
       return callback(new Error('NotFound'))
     }
 
-    if (options.asBuffer) {
-      value = mixedToBuffer(value)
-    }
-
-    callback(null, value)
+    callback(null, deserialize(value, options.asBuffer))
   })
 }
 
@@ -131,27 +139,12 @@ Level.prototype._put = function (key, value, options, callback) {
   this.await(req, callback)
 }
 
-// Valid key types in IndexedDB Second Edition:
-//
-// - Number, except NaN. Includes Infinity and -Infinity
-// - Date, except invalid (NaN)
-// - String
-// - ArrayBuffer or a view thereof (typed arrays). In level-js we also support
-//   Buffer (which is an Uint8Array) (and the primary binary type of Level).
-// - Array, except cyclical and empty (e.g. Array(10)). Elements must be valid
-//   types themselves.
 Level.prototype._serializeKey = function (key) {
-  if (Buffer.isBuffer(key)) {
-    return Level.binaryKeys ? key : key.toString()
-  } else if (Array.isArray(key)) {
-    return Level.arrayKeys ? key.map(this._serializeKey, this) : String(key)
-  } else {
-    return key
-  }
+  return serialize(key, this.supports.bufferKeys)
 }
 
 Level.prototype._serializeValue = function (value) {
-  return value
+  return serialize(value, true)
 }
 
 Level.prototype._iterator = function (options) {

iterator.js

@@ -5,7 +5,7 @@
 var inherits = require('inherits')
 var AbstractIterator = require('abstract-leveldown').AbstractIterator
 var ltgt = require('ltgt')
-var mixedToBuffer = require('./util/mixed-to-buffer')
+var deserialize = require('./util/deserialize')
 var setImmediate = require('./util/immediate')
 var noop = function () {}
 
@@ -23,6 +23,8 @@ function Iterator (db, location, options) {
   this._error = null
   this._transaction = null
 
+  this._keys = options.keys
+  this._values = options.values
   this._keyAsBuffer = options.keyAsBuffer
   this._valueAsBuffer = options.valueAsBuffer
 
@@ -126,8 +128,17 @@ Iterator.prototype._next = function (callback) {
     var key = this._cache.shift()
     var value = this._cache.shift()
 
-    if (this._keyAsBuffer) key = mixedToBuffer(key)
-    if (this._valueAsBuffer) value = mixedToBuffer(value)
+    if (this._keys && key !== undefined) {
+      key = deserialize(key, this._keyAsBuffer)
+    } else {
+      key = undefined
+    }
+
+    if (this._values && value !== undefined) {
+      value = deserialize(value, this._valueAsBuffer)
+    } else {
+      value = undefined
+    }
 
     setImmediate(function () {
       callback(null, key, value)

package.json

@@ -29,8 +29,7 @@
     "abstract-leveldown": "~6.0.1",
     "immediate": "~3.2.3",
     "inherits": "^2.0.3",
-    "ltgt": "^2.1.2",
-    "typedarray-to-buffer": "~3.1.5"
+    "ltgt": "^2.1.2"
   },
   "devDependencies": {
     "airtap": "^2.0.0",

test/custom-test.js

@@ -63,7 +63,8 @@ module.exports = function (leveljs, test, testCommon) {
     })
   })
 
-  test('put Buffer value, get Uint8Array value', function (t) {
+  // TODO: get with { asBuffer: false } should now return a string
+  test.skip('put Buffer value, get Uint8Array value', function (t) {
     var level = testCommon.factory()
     level.open(function (err) {
       t.notOk(err, 'no error')
@@ -96,7 +97,8 @@ module.exports = function (leveljs, test, testCommon) {
     })
   })
 
-  test('put Uint8Array value, get Uint8Array value', function (t) {
+  // TODO: get with { asBuffer: false } should now return a string
+  test.skip('put Uint8Array value, get Uint8Array value', function (t) {
     var level = testCommon.factory()
     level.open(function (err) {
       t.notOk(err, 'no error')
@@ -129,7 +131,8 @@ module.exports = function (leveljs, test, testCommon) {
     })
   })
 
-  test('put ArrayBuffer value, get ArrayBuffer value', function (t) {
+  // TODO: get with { asBuffer: false } should now return a string
+  test.skip('put ArrayBuffer value, get ArrayBuffer value', function (t) {
     var level = testCommon.factory()
     level.open(function (err) {
       t.notOk(err, 'no error')
@@ -145,6 +148,9 @@ module.exports = function (leveljs, test, testCommon) {
     })
   })
 
+  // TODO: test putting+gettting:
+  // Buffer/Uint8Array/ArrayBuffer/string key with keyAsBuffer true/false
+
   // This should be covered by abstract-leveldown tests, but that's
   // prevented by process.browser checks (Level/abstract-leveldown#121).
   // This test is adapted from memdown.
@@ -194,8 +200,8 @@ module.exports = function (leveljs, test, testCommon) {
       t.ifError(err, 'no open error')
 
       db.batch([
-        { type: 'put', key: Buffer.from([0]), value: 0 },
-        { type: 'put', key: Buffer.from([1]), value: 1 }
+        { type: 'put', key: Buffer.from([0]), value: '0' },
+        { type: 'put', key: Buffer.from([1]), value: '1' }
       ], function (err) {
         t.ifError(err, 'no batch error')
 
@@ -204,8 +210,8 @@ module.exports = function (leveljs, test, testCommon) {
           t.ifError(err, 'no iterator error')
 
           t.same(entries, [
-            { key: Buffer.from([0]), value: 0 },
-            { key: Buffer.from([1]), value: 1 }
+            { key: Buffer.from([0]), value: '0' },
+            { key: Buffer.from([1]), value: '1' }
           ], 'keys are Buffers')
 
           db.close(function (err) {

test/index.js

@@ -28,7 +28,3 @@ suite(testCommon)
 
 // Additional tests for this implementation
 require('./custom-test')(leveljs, test, testCommon)
-require('./structured-clone-test')(leveljs, test, testCommon)
-require('./key-type-test')(leveljs, test, testCommon)
-require('./key-type-illegal-test')(leveljs, test, testCommon)
-require('./native-order-test')(leveljs, test, testCommon)