Getting hierchical data from redis with node.js

Question

I was hoping that someone out there could come to my rescue with this. For some reason, I can't get my mind around recursion in node.js. It doesn't even have to be recursion, if there's another way.

I'm using redis sets to store a hierarchy in sets:

SADD parents.<name> <parent1> <parent2>

then, parent1 and parent2 will also have entries, and on up. I want to convert it to a JSON array of objects.

The JSON will look like this:

[
 {
     label: <name>,
     parents: [
         { label: <parent1>,
           parents: [ {label: <grandparent1>}] },
         { label: <parent2> }
     ]
 }
]

And so on and so forth. This should be able to work for any depth, although on average it will only be 4-6 nodes deep.

Here's some code I've been playing with that just gets me the first level:

var redis = require('node-redis');
var r_client = redis.createClient();

function get_parents (name, current, cb) {

        var output = new Array;
        output.push( { label: name, parents: [] } );

        r_client.smembers('parents.' + name, function(err, reply) {
            for (var i = 0; i < reply.length; i++)
            {
                var name = reply[i].toString('utf8');
                output[i].parents.push({label: name, parents: [] });
             }
             cb (output);
        });
}

get_parents( 'bob', function(out) {console.log('Final output: ' + JSON.stringify( out ))} );

I'm basically wanting to do this:

Start at root node. Call redis, get parents.
Build object for root node.
Call same function to build the other objects.
As calls to redis begin to come back null, the calls will return and start to combine objects.

Any help would be greatly appreciated.

EDIT: Updated get_parents (still doesn't work):

function get_parents (name, cb) {
    r_client.smembers('parents.' + name, function(err, reply) {
    for (var i = 0; i < reply.length; i++)
    {
      var name = reply[i].toString('utf8');
      output.push( { label: name, parents: [] } );
      output[i].parents = get_parents (output[i].parents.name, cb);
    }

    cb (output);
  });
}

EDIT: I decided to use Promises, so I pursued that option. Thanks for all the help!

a short way would be to store a json to each parent.name instead of a set — Gntem
– Gntem, Commented Sep 16, 2013 at 20:11
@GeoPhoenix - I agree, but this is part of a larger project, so it makes sense to do it this way. — coding_hero
– coding_hero, Commented Sep 17, 2013 at 16:02
well just keep in mind that redis is not meant to be a database, such as mongoDB or mysql which have build functionality for doing this, or storing in a way that's "making sense", redis, is just a key value store, so IMHO storing something that you have fetched from another source, to Redis, in lets say tree.NAME with expiration.. would make better sense than querying for hierarchical data, i mean, twitter followers/following its ok, but storing something like a tree in such way , kind of misses the point of redis. not saying that's not feasible .. — Gntem
– Gntem, Commented Sep 17, 2013 at 18:33

FGRibreau · Accepted Answer · 2013-09-17 17:21:41Z

6

TL;DR: I made a NodeJS module (RedisTree) and a blog post explaining the implementation.

The above code is the initial implementation

Nice challenge! Here is an implementation that matches your requirement, I added a ".save(tree, f)" method, it uses lodash, async and of course node_redis.

var sampleTree = [{
  label: 'label1',
  parents: [{
    label: 'label2',
    parents: [{
      label: 'label4',
      parents: []
    }]
  }, {
    label: 'label3',
    parents: []
  }]
}];

// Note: it's required to use a set in order to retrieve data
// SET:
// sadd label1 label2 label3
// sadd label2 label4
var _     = require('lodash');
var async = require('async');
var redis = require('redis').createClient();


function RedisTree(){}

/**
 * Iteratively & asynchronously retrieve an item from Redis
 * @param  {String} label label name (set name)
 * @param  {Function} f(err, arrayOfItems)
 */
RedisTree.prototype._getItem = function(label, f) {
  var parent = _.isArray(_.last(arguments)) ? _.last(arguments) : [];

  this.members(label, function(err, cards){
    var item = {
      label: this.label(label),
      parents: []
    };
    parent.push(item);

    if(cards.length === 0){return f(null, parent);}

    async.map(cards, _.partialRight(this._getItem.bind(this), item.parents), function(e){f(e, parent);});
  }.bind(this));
};

RedisTree.prototype._saveItem = function(item, f) {
  var labels = _.pluck(item.parents, 'label');
  if(labels.length === 0){return f();}

  redis.sadd(item.label, labels, function(err){
    if(err){return f(err);}
    this._saveItems(item.parents, f);
  }.bind(this));
};

/**
 *
 * @param  {Array} arrToSave array of items
 * @param  {Function} f(err)
 */
RedisTree.prototype._saveItems = function(arrToSave, f) {
  async.forEach(arrToSave, this._saveItem.bind(this), f);
};

/**
 * Retrieve a name from the label
 * Can be overridden to provide namespace support
 * e.g. return label.substring(2);
 * @return {[type]} [description]
 */
RedisTree.prototype.label = function(label){return label;};

/**
 * Retrieve every members of the `label` set from redis
 * Can be overridden to provide namespace support
 * e.g. redis.smembers('ns.'+label, f);
 * @param {[type]} label [description]
 * @param {[type]} f     [description]
 */
RedisTree.prototype.members = function(label, f) {redis.smembers(label, f);};

/**
 * Load a tree from Redis
 * @param  {String} startLabel Were to start
 * @param  {Function} f(err, tree)
 */
RedisTree.prototype.load = function(startLabel, f){this._getItem(startLabel, f);};

/**
 * Save a tree from Redis
 * @param  {Array} treeArray
 * @param  {Function} f(err, tree)
 */
RedisTree.prototype.save = function(treeArray, f){this._saveItems(treeArray, f);};

Here is how to use it:

var t = new RedisTree();

// Save the sampleTree
t.save(sampleTree, function(err){
  // or ... retrieve it starting at "label1"
  t.load('label1', function(err, tree){
    console.log("sampleTree === ", JSON.stringify(tree, null, 1));
  });
});

edited Sep 17, 2013 at 17:21

answered Sep 17, 2013 at 16:09

FGRibreau

7,2793 gold badges42 silver badges49 bronze badges

Sign up to request clarification or add additional context in comments.

5 Comments

coding_hero Over a year ago

Thanks for the reply. Going over this in my head.

FGRibreau Over a year ago

Do you need anything else or is my answer enough?

coding_hero Over a year ago

I think your answer is enough, but I'm not good enough at Node and JS to fully understand everything. I posted a link on reddit, and someone replied with a solution using promises. I think it makes more sense to me. I'll play with both tonight. And thank you for taking up the challenge!!

Michelle Tilley Over a year ago

@coding_hero I'd love to see the link to the Reddit post. FGRibreau, very nice!

coding_hero Over a year ago

@BrandonTilley - This is what I ended up with. FGRibreau's answer was great, but the promises approach just made more sense to me.

Jason Livesay · Accepted Answer · 2013-09-17 17:42:45Z

0

Are you sure you can't just store it as JSON on disk or with MongoDB? Also why does your desired result list parents instead of children?

answered Sep 17, 2013 at 17:42

Jason Livesay

6,3783 gold badges27 silver badges31 bronze badges

1 Comment

coding_hero Over a year ago

Yes, the rest of the process is already implemented in Redis. That's the way the result needs to be, starting with the root (requested), and all parents and grand-parents following. This isn't for a family tree, this is for another type of hierarchical data.

Collectives™ on Stack Overflow

Getting hierchical data from redis with node.js

2 Answers 2

5 Comments

1 Comment

Your Answer

Linked

Hot Network Questions

Collectives™ on Stack Overflow

2 Answers 2

5 Comments

1 Comment

Your Answer

Sign up or log in

Post as a guest

Linked

Related