Home

Awesome

level-subkey

The level-subkey is modified from level-sublevel.

The level-subkey use the path to separate sections of levelup, with hooks! these sublevels are called dynamic subkey.

build status

NPM NPM

This module allows you to create a hierarchy data store with levelup-sync database, kinda like tables in an sql database, but hierarchical, evented, and ranged, for real-time changing data.

Main Features different from level-sublevel

todo

Main Concepts

The key is always string only unless it's an index.

Stability

Unstable: Expect patches and features, possible api changes.

This module is working well, but may change in the future as its use is further explored.

Internal Storage Format for Key

The internal key path storage like file path, but the path separator can be customize.

    var precodec = require('level-subkey/lib/codec')
    precodec.SUBKEY_SEPS = ["/|-", "#.+"] //the first char is the default subkey separator, others are customize separator. 
    subkey.put("some", "value", {separator: '|'})
    //list all key/value on separator "|"
    subkey.createReadStream({separator: '.'})
    //it will return all prefixed "|" keys: {key: "|abc", value:....}

var stuff = db.subkey('stuff')
var animal = stuff.subkey('animal')
var plant = stuff.subkey('plant')

animal.put("pig", value, function () {})
// stored raw key is : "/stuff/animal#pig"
// decoded key is: "/stuff/animal/pig"
animal.put("../plant/cucumber", value, function (err) {})
// stored raw key is : "/stuff/plant#cucumber"
// decoded key is: "/stuff/animal/cucumber"
db.put("/stuff/animal/pig/.mouth", value, function(err){})
// stored raw key is : "/stuff/animal/pig*mouth"
// decoded key is: "/stuff/animal/pig/.mouth"
db.put("/stuff/animal/pig/.ear", value, function(err){})
// stored raw key is : "/stuff/animal/pig*ear"
// decoded key is: "/stuff/animal/pig/.ear"
db.put("/stuff/animal/pig/.ear/.type", value, function(err){})
// stored raw key is : "/stuff/animal/pig/.ear*type"
// decoded key is: "/stuff/animal/pig/.ear/.type"

API

Subkey.subkey()/path(keyPath)

Create(or get from a global cache) a new Subkey instance, and load the value if this key is exists on the database

arguments

return

The usages:

Subkey.fullName/path()

arguments

return

Subkey.isAlias()

Get the subkey itself whether is an alias or not.

arguments

return

Subkey.alias()

Create an alias for the keyPath:

Create an alias for itself:

arguments

return

Subkey.readStream/createReadStream([options])

create a read stream to visit the child subkeys of this subkey.

arguments

return

the standard 'data', 'error', 'end' and 'close' events are emitted. the 'last' event will be emitted when the last data arrived, the argument is the last raw key(no decoded). if no more data the last key is undefined.

Examples

filter usage:

db.createReadStream({filter: function(key, value){
    if (/^hit/.test(key))
        return db.FILTER_INCLUDED
    else key == 'endStream'
        return db.FILTER_STOPPED
    else
        return db.FILTER_EXCLUDED
}})
  .on('data', function (data) {
    console.log(data.key, '=', data.value)
  })
  .on('error', function (err) {
    console.log('Oh my!', err)
  })
  .on('close', function () {
    console.log('Stream closed')
  })
  .on('end', function () {
    console.log('Stream closed')
  })

next and last usage for paged data demo:


var callbackStream = require('callback-stream')

var lastKey = null;

function nextPage(db, aLastKey, aPageSize, cb) {
  var stream = db.readStream({next: aLastKey, limit: aPageSize})
  stream.on('last', function(aLastKey){
    lastKey = aLastKey;
  });

  stream.pipe(callbackStream(function(err, data){
    cb(data, lastKey)
  }))

}

var pageNo = 1;
dataCallback = function(data, lastKey) {
    console.log("page:", pageNo);
    console.log(data);
    ++pageNo;
    if (lastKey) {
      nextPage(db, lastKey, 10, dataCallback);
    }
    else
      console.log("no more data");
}
nextPage(db, lastKey, 10, dataCallback);

Examples

Simple Section Usage

var LevelUp = require('levelup')
var Subkey = require('level-subkey')

var db = Subkey(LevelUp('/tmp/sublevel-example'))
var stuff = db.subkey('stuff')
//it is same as stuff = db.sublevel('stuff')  but db.sublevel is deprecated.

//put a key into the main levelup
db.put(key, value, function () {})

//put a key into the sub-section!
stuff.put(key2, value, function () {})

Sublevel prefixes each subsection so that it will not collide with the outer db when saving or reading!

hierarchy data store usage

var LevelUp = require('levelup')
var Subkey = require('level-subkey')

var db = Subkey(LevelUp('/tmp/sublevel-example'))

//old sublevel usage:
var stuff = db.subkey('stuff')      //or stuff = db.path('stuff')
var animal = stuff.subkey('animal') //or animal = stuff.path('animal')
var plant = stuff.subkey('plant')

//put a key into animal!
animal.put("pig", value, function () {})

//new dynamic hierarchy data storage usage:
animal.put("../plant/cucumber", value, function (err) {})
db.put("/stuff/animal/pig", value, function(err){})
db.get("/stuff/animal/pig", function(err, value){})

//put pig's attribute as key/value
db.put("/stuff/animal/pig/.mouth", value, function(err){})
db.put("/stuff/animal/pig/.ear", value, function(err){})

//list all pig's attributes
db.createReadStream({path: "/stuff/animal/pig", separator="."})
//return: {".mouth":value, ".ear":value}

//list all pig's path(excludes the subkeys)
//it will search from "/stuff/\x00" to "/stuff/\uffff"
db.createPathStream({path: "/stuff"}) //= db.createReadStream({separator:'/', separatorRaw: true, start:'0'})
//return:{ 'animal/pig': value, 'animal/pig.ear': value, 'animal/pig.mouth': value, 'plant/cucumber': value}


//list all keys in "/stuff/animal"
db.createReadStream({path: "/stuff/animal"})

//list all keys in "/stuff/plant"
animal.createReadStream({start: "../plant"})


//write by stream
var wsAnimal = animal.createWriteStream()
wsAnimal.on('err', function(err){throw err})
wsAnimal.on('close', function(){})
wsAnimal.write({key: "cow", value:value})
wsAnimal.write({key: "/stuff/animal/cow", value:value})
wsAnimal.write({key: "../plant/tomato", value:value})
wsAnimal.end()

//crazy usage:
//the path will always be absolute key path.
//Warning: setPath will be broken the subkeys cache on nut!!
//  if setPath it will remove itself from cache.
animal.setPath("/stuff/plant")
animal.setPath(plant)
//now the "animal" is plant in fact.
animal.get("cucumber", function(err, value){})

Hooks

Hooks are specially built into Sublevel so that you can do all sorts of clever stuff, like generating views or logs when records are inserted!

Records added via hooks will be atomically inserted with the triggering change.

db.pre/post()

  1. subkey.pre(function(op, add))
  2. subkey.pre(aKeyPattern, function(op, add))
  3. subkey.pre(aRangeObject, function(op, add))
//you should be careful of using the add() function
//maybe endless loop in it. u can disable the trigger
add({
    key:...,
    value:...,
    type:'put' or 'del',
    triggerBefore: false, //defalut is true. whether trigger this key on pre hook.
    triggerAfter: false   //defalut is true. whether trigger this key on post hook.
 
})
add(false): abondon this operation(remove it from the batch).

Hooks Example

Whenever a record is inserted, save an index to it by the time it was inserted.

var sub = db.subkey('SEQ')

db.pre(function (ch, add) {
  add({
    key: ''+Date.now(), 
    value: ch.key, 
    type: 'put',
    // NOTE: pass the destination db to add the value to that subsection!
    path: sub
  })
})

db.put('key', 'VALUE', function (err) {
  // read all the records inserted by the hook!
  sub.createReadStream().on('data', console.log)
})

Notice that the parent property to add() is set to sub, which tells the hook to save the new record in the sub section.

Hooks Another Example

var sub = db.subkey('SEQ')

//Hooks range 
db.pre({gte:"", lte:"", path:""}, function (ch, add) {
  add({
    key: ''+Date.now(), 
    value: ch.key, 
    type: 'put',
    // NOTE: pass the destination db to add the value to that subsection!
    path: sub
  })
})

//hooks a key, and the key can be relative or absolute key path and minimatch supports.
db.pre("a*", function (ch, add) {
  //NOTE: add(false) means do not put this key into storage.
  add({
    key: ''+Date.now(), 
    value: ch.key, 
    type: 'put',
    // NOTE: pass the destination db to add the value to that subsection!
    path: sub
  })
})

Batches

In sublevel batches also support a prefix: subdb property, if set, this row will be inserted into that database section, instead of the current section, similar to the pre hook above.

var sub1 = db.subkey('SUB_1')
var sub2 = db.subkey('SUB_2')

sub2.batch([
  {key: 'key', value: 'Value', type: 'put'},
  {key: 'key', value: 'Value', type: 'put', path: sub2},
  {key: '../SUB_1/key', value: 'Value', type: 'put', path: sub2},
], function (err) {...})

License

MIT