diff --git a/Apps/HelloTimeline/img/facebook.png b/Apps/HelloTimeline/img/facebook.png new file mode 100644 index 000000000..355ac84fa Binary files /dev/null and b/Apps/HelloTimeline/img/facebook.png differ diff --git a/Apps/HelloTimeline/img/foursquare.png b/Apps/HelloTimeline/img/foursquare.png new file mode 100644 index 000000000..28ed5e3e4 Binary files /dev/null and b/Apps/HelloTimeline/img/foursquare.png differ diff --git a/Apps/HelloTimeline/img/instagram.png b/Apps/HelloTimeline/img/instagram.png new file mode 100644 index 000000000..d06b4aad3 Binary files /dev/null and b/Apps/HelloTimeline/img/instagram.png differ diff --git a/Apps/HelloTimeline/img/twitter.png b/Apps/HelloTimeline/img/twitter.png new file mode 100644 index 000000000..e07b98665 Binary files /dev/null and b/Apps/HelloTimeline/img/twitter.png differ diff --git a/Apps/HelloTimeline/index.html b/Apps/HelloTimeline/index.html new file mode 100644 index 000000000..d096e6364 --- /dev/null +++ b/Apps/HelloTimeline/index.html @@ -0,0 +1,34 @@ + + + + + + + + Timeline Viewer + + + + + + + +

Forever Timeless

+ + + + + + + + diff --git a/Apps/HelloTimeline/package.json b/Apps/HelloTimeline/package.json new file mode 100644 index 000000000..c6a5a0d27 --- /dev/null +++ b/Apps/HelloTimeline/package.json @@ -0,0 +1,31 @@ +{ + "author": "Singly, Inc. ", + "name": "hellotimeline", + "description": "A simple way to view your timeline.", + "version": "0.0.1", + "homepage": "https://github.com/LockerProject/locker", + "repository": { + "type": "app", + "title": "Hello Timeline", + "status": "stable", + "static": true, + "hidden": true, + "handle": "hellotimeline", + "update": true, + "uses": { + "services": [ + "facebook", + "foursquare", + "instagram", + "twitter" + ] + }, + "github": "https://github.com/LockerProject/locker", + "url": "" + }, + "engines": { + "node": ">=0.4.9" + }, + "dependencies": {}, + "devDependencies": {} +} diff --git a/Apps/HelloTimeline/script.js b/Apps/HelloTimeline/script.js new file mode 100755 index 000000000..457f13c64 --- /dev/null +++ b/Apps/HelloTimeline/script.js @@ -0,0 +1,103 @@ +var baseUrl = false; + +$(document).ready(function() { + if(baseUrl === false) window.alert("Couldn't find your locker, you might need to add a config.js (see dev.singly.com)"); +}); + +var offset=0; +$(function() { + // be careful with the limit, some people have large datasets ;) + loadStuff(); + loadStatus(); + $("#moar").click( function(){ + offset += 50; + loadStuff(); + }); +}); + +var max = 20; +function loadStatus() +{ + $.getJSON(baseUrl + '/Me/timeline/state',{}, function(data) { + if(!data) return $("#status").text("timeline failed :("); + if(data.ready == 1) return $("#status").text(""); + var name = (data.current) ? data.current.type + " at "+data.current.offset : ""; + $("#status").text("timeline is indexing "+name); + if(max-- <= 0) return $("#status").text("plz reload"); + window.setTimeout(loadStatus, 10000); // poll for a bit + }); +} + +function loadStuff(){ + $.getJSON(baseUrl + '/Me/timeline/',{limit:50, offset:offset}, function(data) { + if(!data || !data.length) return; + var html = '

'+ago(data[0].first)+'

'; + for(var i in data) + { + var p = data[i]; + p.refs.forEach(function(ref){ html += icon(ref); }); + var resp = p.comments + p.ups; + html += ""+p.from.name+": " + p.text + ''; + html += ' .js'; + if(resp > 0) html += "
["+resp+"]
"; + html += '
'; + } + $("#test").append(html); + }); +} + +function loadResp(id) +{ + $.getJSON(baseUrl + '/Me/timeline/getResponses',{item:id}, function(data) { + $("#"+id).html(""); + if(!data || !data.length) return; + var ups = " ups: "; + var coms = ""; + for(var i in data) + { + var r = data[i]; + if(r.type == "up") + { + ups += icon(r.ref); + ups += " "+r.from.name+", "; + } + if(r.type == "comment") + { + coms += "
" + coms += icon(r.ref); + coms += " "+r.from.name+": "; + coms += r.text; + } + } + $("#"+id).append(ups+coms); + }); +} + +function icon(ref) +{ + var start = ref.indexOf('//')+2; + var net = ref.substr(start,ref.indexOf('/',start+2)-start); + if(net == "links") return ""; + return ""; + +} + +function ago(at) +{ + var tip = ''; + var timeDiff = Date.now() - at; + if (timeDiff < 60000) { + tip += 'last updated less than a minute ago'; + } else if (timeDiff < 3600000) { + tip += 'last updated ' + Math.floor(timeDiff / 60000) + ' minutes ago'; + } else if (timeDiff < 43200000) { + tip += 'last updated over an hour ago'; + } else if (timeDiff < 43800000) { + tip += 'last updated ' + Math.floor(timeDiff / 3600000) + ' hours ago'; + } else { + var d = new Date; + d.setTime(at); + tip += 'last updated ' + d.toString(); + } + return tip; +} \ No newline at end of file diff --git a/Apps/prolific_posters b/Apps/prolific_posters new file mode 160000 index 000000000..591b54d68 --- /dev/null +++ b/Apps/prolific_posters @@ -0,0 +1 @@ +Subproject commit 591b54d68061810e45b8a0604cc0abcc42463677 diff --git a/Collections/Links/api.js b/Collections/Links/api.js index f25b569d1..6643d4337 100644 --- a/Collections/Links/api.js +++ b/Collections/Links/api.js @@ -156,6 +156,32 @@ module.exports = function(app, lockerInfo) { }); }); + // get just encounters raw! + // expose way to get raw links and encounters + app.get('/encounters', function(req, res) { + var options = {sort:{"_id":-1}}; + if(!req.query["all"]) options.limit = 20; // default 20 unless all is set + if (req.query.limit) { + options.limit = parseInt(req.query.limit); + } + if (req.query.offset) { + options.offset = parseInt(req.query.offset); + } + if(req.query['stream'] == "true") + { + res.writeHead(200, {'content-type' : 'application/jsonstream'}); + } + var results = []; + dataStore.getEncounters(options, function(item) { + if(req.query['stream'] == "true") return res.write(JSON.stringify(item)+'\n'); + results.push(item); + }, function(err){ + if(err) logger.error(err); + if(req.query['stream'] == "true") return res.end(); + return res.send(results); + }); + }); + } function isFull(full) { diff --git a/Collections/Links/dataIn.js b/Collections/Links/dataIn.js index 24b74d899..0271b03b1 100644 --- a/Collections/Links/dataIn.js +++ b/Collections/Links/dataIn.js @@ -109,10 +109,11 @@ var encounterQueue = async.queue(function(e, callback) { async.forEach(urls,function(u,cb){ linkMagic(u,function(link){ // make sure to pass in a new object, asyncutu - dataStore.addEncounter(lutil.extend(true,{orig:u,link:link},e), function(err,doc){ + dataStore.addEncounter(lutil.extend(true,{orig:u,link:link.link},e), function(err,doc){ if(err) return cb(err); dataStore.updateLinkAt(doc.link, doc.at, function(err, obj){ if(err) return cb(err); + obj.encounters = [doc]; // include this encounter! locker.ievent(lutil.idrNew("link","links",obj.id),obj,"update"); // let happen independently cb(); }); @@ -140,6 +141,7 @@ function linkMagic(origUrl, callback){ dataStore.checkUrl(origUrl,function(linkUrl){ if(linkUrl) return callback(linkUrl); // short circuit! // new one, expand it to a full one + var linkUrl; util.expandUrl({url:origUrl},function(u2){linkUrl=u2},function(){ // fallback use orig if errrrr if(!linkUrl) { @@ -149,7 +151,7 @@ function linkMagic(origUrl, callback){ // does this full one already have a link stored? dataStore.getLinks({link:linkUrl,limit:1},function(l){link=l},function(err){ if(link) { - return callback(link.link); // yeah short circuit dos! + return callback(link); // yeah short circuit dos! } // new link!!! link = {link:linkUrl}; @@ -164,11 +166,13 @@ function linkMagic(origUrl, callback){ if (!link.at) link.at = Date.now(); dataStore.addLink(link,function(err, obj){ locker.ievent(lutil.idrNew("link","links",obj.id),obj); // let happen independently - callback(link.link); // TODO: handle when it didn't get stored or is empty better, if even needed + callback(link); // TODO: handle when it didn't get stored or is empty better, if even needed // background fetch oembed and save it on the link if found oembed.fetch({url:link.link, html:html}, function(e){ if(!e) return; - dataStore.updateLinkEmbed(link.link, e, function(){}); + dataStore.updateLinkEmbed(link.link, e, function(err, obj){ + locker.ievent(lutil.idrNew("link","links",obj.id),obj,"update"); + }); }); }); }); @@ -205,7 +209,7 @@ function getEncounterTwitter(event) { var tweet = event.data; var txt = (tweet.retweeted_status && tweet.retweeted_status.text) ? tweet.retweeted_status.text : tweet.text; - var e = {id:tweet.id + var e = {id:tweet.id_str , idr:event.idr , network:"twitter" , text: txt + " " + tweet.user.screen_name diff --git a/Collections/Links/dataStore.js b/Collections/Links/dataStore.js index 833bc236a..725760792 100644 --- a/Collections/Links/dataStore.js +++ b/Collections/Links/dataStore.js @@ -72,7 +72,7 @@ exports.checkUrl = function(origUrl, callback) { if(err) return callback(); cursor.nextObject(function(err, item){ if(err || !item || !item.link) return callback(); - callback(item.link); + callback(item); }); }); } @@ -93,7 +93,7 @@ exports.getLinks = function(arg, cbEach, cbDone) { linkCDS.findWrap(f,arg,cbEach,cbDone); } -exports.getFullLink = function(id, cbDone) { +exports.getLink = function(id, cbDone) { var link = null; exports.getLinks({link:id}, function(l) { link = l; }, function() { cbDone(link); }); } diff --git a/Collections/Timeline/README.md b/Collections/Timeline/README.md new file mode 100644 index 000000000..f482adea1 --- /dev/null +++ b/Collections/Timeline/README.md @@ -0,0 +1,19 @@ +The "feeds" collection is all of the socially shared items across any service, facebook newsfeed, twitter timeline, foursquare recents, etc. + +It consists of a primary type called an "item" which can have an array of "responses" that are comments, likes, retweets, etc from other people. + +The goal is to dedup cross-posts to multiple networks from one person, particularly things like a foursquare checkin. + + +Notes: + + id references and keys + cross-collection reference storage with links + link based dedup of foursquare + dedup uses keys as guids across networks and for text matching variations + /update takes type arg for individual updates + idrs as via, no original storage + prioritization scheme in merging items + generic from object, list of froms + references array of all sources + generalized responses and top-level statistics diff --git a/Collections/Timeline/dataIn.js b/Collections/Timeline/dataIn.js new file mode 100644 index 000000000..58bcf14d9 --- /dev/null +++ b/Collections/Timeline/dataIn.js @@ -0,0 +1,484 @@ +var request = require('request'); +var async = require('async'); +var logger; +var lutil = require('lutil'); +var url = require('url'); +var crypto = require("crypto"); +var path = require('path'); + +var dataStore, locker; + +// internally we need these for happy fun stuff +exports.init = function(l, dStore, callback){ + dataStore = dStore; + locker = l; + logger = l.logger; + callback(); +} + +// take an idr and turn it into a generic network-global key +// this could be network-specific transformation and need data +function idr2key(idr, data) +{ + delete idr.query; delete idr.search; // not account specific + idr.pathname = '/'; // ids are generic across any context + return url.parse(url.format(idr)); +} + +// internal util breakout +function idrHost(r, data) +{ + if(r.host === 'twitter') + { + r.hash = (r.pathname === 'related') ? data.id : data.id_str; + r.protocol = 'tweet'; + } + if(r.host === 'facebook') + { + r.hash = data.id; + r.protocol = 'post'; + } + if(r.host === 'foursquare') + { + r.hash = data.id; + r.protocol = 'checkin'; + } + if(r.host === 'instagram') + { + r.hash = data.id; + r.protocol = 'photo'; + } +} +exports.idrHost = idrHost; + +// useful to get key from raw data directly (like from a via, not from an event) +function getKey(network, data) +{ + var r = {slashes:true}; + r.host = network; + r.pathname = '/'; + idrHost(r, data); + return url.parse(url.format(r)); // make sure it's consistent +} + +// normalize events a bit +exports.processEvent = function(event, callback) +{ + if(!callback) callback = function(err){if(err) logger.error(err);}; + var idr = url.parse(event.idr, true); + if(!idr || !idr.protocol) return callback("don't understand this data"); + // handle links as a special case as we're using them for post-process-deduplication + if(idr.protocol == 'link:') return processLink(event, callback); + masterMaster(idr, event.data, callback); +} + +// some data is incomplete, stupid but WTF do you do! +var profiles = {}; +var mbQ = async.queue(masterBlaster, 1); // only process in series +function masterMaster(idr, data, callback) +{ + if(idr.protocol == 'checkin:' && idr.pathname.indexOf('checkin') != -1 && idr.query.id) + { // foursquare checkins api for a person is the same json format but missing the .user, which we need to store the correct contact idr, so we have to fetch/cache that + var svcId = idr.query.id; + if(profiles[svcId]) + { + data.user = profiles[svcId]; + return mbQ.push({idr:idr, data:data}, callback); + } + var lurl = locker.lockerBase + '/Me/'+svcId+'/getCurrent/profile'; + request.get({uri:lurl, json:true}, function(err, resp, arr){ + if(err || !arr || arr.length == 0) + { + logger.error("couldn't fetch profile for "+svcId+" so have to skip "+url.format(idr)); + return callback(); + } + logger.debug("caching profile for "+svcId); + data.user = profiles[svcId] = arr[0]; + return mbQ.push({idr:idr, data:data}, callback); + }); + return; + } + mbQ.push({idr:idr, data:data}, callback); +} +exports.masterMaster = masterMaster; + +// figure out what to do with any data +function masterBlaster(arg, callback) +{ + var idr = arg.idr; + var data = arg.data; + if(typeof data != 'object') return callback("missing or bad data"); +// logger.debug("MM\t"+url.format(idr)); + var ref = url.format(idr); + var item = {keys:{}, refs:[], froms:{}, from:{}, responses:[], first:new Date().getTime(), last:new Date().getTime()}; + item.ref = ref; + item.refs.push(ref); + item.keys[url.format(idr2key(idr, data))] = item.ref; + if(idr.protocol == 'tweet:'){ + if(data.user && data.text) itemTwitter(item, data); + if(data.related) itemTwitterRelated(item, data.related); + } + if(idr.protocol == 'post:') itemFacebook(item, data); + if(idr.protocol == 'checkin:') itemFoursquare(item, data); + if(idr.host == 'instagram') itemInstagram(item, data); + var dup; + // we're only looking for the first match, if there's more, that's a very odd situation but could be handled here + async.forEach(Object.keys(item.keys), function(key, cb) { + dataStore.getItemByKey(key,function(err, doc){ + if(!err && doc) dup = doc; + cb(); + }); + }, function (err) { + if(!item.pri && !dup) return callback(); // some new items, like twitter related, are only for merging, shouldn't happen but if it does we may need to queue/stash or something + if(dup) item = itemMerge(dup, item); + dataStore.addItem(item, function(err, item){ + if(err) return callback(err); + // all done processing, very useful to pull out some summary stats now! + // feels inefficient to re-query and double-write here, but should be logically safe this way + // TODO: can optimize and special case the first time! + item.comments = item.ups = 0; + dataStore.getResponses({item:item.id}, function(r){ + if(r.type == "comment") item.comments++; + if(r.type == "up") item.ups++; + }, function(){ + dataStore.addItem(item, callback); // finally call back! + }); + }); + }); +} + +// save a reference +function itemRef(item, ref, callback) +{ + var refs = {}; + for(var i = 0; i < item.refs.length; i++) refs[item.refs[i]] = true; + if(refs[ref]) return callback(undefined, item); + refs[ref] = true; + item.refs = Object.keys(refs); + dataStore.addItem(item, callback); +} + +// intelligently merge two items together and return +function itemMerge(older, newer) +{ +// logger.debug("MERGE\t"+JSON.stringify(older)+'\t'+JSON.stringify(newer)); + if(newer.pri > older.pri) + { // replace top level summary stuff if newer is deemed better + older.ref = newer.ref; + older.from = newer.from; + older.text = newer.text; + if(newer.title) older.title = newer.title; + } + // update timestamps to extremes + if(newer.last > older.last) older.last = newer.last; + if(newer.first < older.first) older.first = newer.first; + // update the two searchable objects + for(var k in newer.keys) older.keys[k] = newer.keys[k]; + for(var k in newer.froms) older.froms[k] = newer.froms[k]; + // our array of references for this item, needs to be unique + var refs = {}; + for(var i = 0; i < newer.refs.length; i++) refs[newer.refs[i]]=true; + for(var i = 0; i < older.refs.length; i++) refs[older.refs[i]]=true; + older.refs = Object.keys(refs); + older.responses = newer.responses; // older doesn't have any since they're stored separately + return older; +} + +// merge existing with delete! +function itemMergeHard(a, b, cb) +{ + logger.debug("hard merge of "+a.ref+" to "+b.ref); + var item = itemMerge(a, b); + if(!item.responses) item.responses = []; + // gather old responses + dataStore.getResponses({item:b.id}, function(r){ item.responses.push(r); }, function(){ + // update existing item + dataStore.addItem(item, function(err, item){ + if(err || !item) return cb(err); + // now delete old fully! + dataStore.delItem(b.id, cb); + }); + }); +} + +// when a processed link event comes in, check to see if it's a long url that could help us de-dup +function processLink(event, callback) +{ + if(!event || !event.data || !event.data.encounters) return callback("no encounter"); + // process each encounter if there's multiple + async.forEach(event.data.encounters, function(encounter, cb){ + // first, look up event via/orig key and see if we've processed it yet, if not (for some reason) ignore + var key = url.format(getKey(encounter.network, encounter.via)); + dataStore.getItemByKey(key,function(err, item){ + if(err || !item) return cb(err); + // we tag this item with a ref to the link, super handy for cross-collection mashups + itemRef(item, event.idr, function(err, item){ + if(err || !item) return cb(err); + var u = url.parse(encounter.link); + // if foursquare checkin and from a tweet, generate foursquare key and look for it + if(u.host == 'foursquare.com' && u.pathname.indexOf('/checkin/') > 0) + { + var id = path.basename(u.pathname); + var k2 = 'checkin://foursquare/#'+id; + dataStore.getItemByKey(k2,function(err, item2){ + if(err || !item2) return cb(); + // found a dup! + itemMergeHard(item, item2, cb); + }); + } + // instagram uses the same caption everywhere so link-based dedup isn't needed + }); + }); + }, callback); +} +exports.processLink = processLink; + +// give a bunch of sane defaults +function newResponse(item, type) +{ + return { + type: type, + ref: item.ref, + from: {} + } +} + +// sometimes links can be used as raw keys +// using the hash value here because a ver of mongo will silently barf on some http based keys!!! +function keyUrl(item, link) +{ + var u = url.parse(link); + if(!u || !u.host) return; + // all instagram links across all services seem to be preserved + if(u.host == 'instagr.am') item.keys["url:"+crypto.createHash('md5').update(link).digest('hex')] = item.ref; +} + +// add key(s) based on a timestamp+-window (all in seconds) +function keyTime(item, at, window, prefix) +{ + var keys = {}; + var wlen = window.toString().length; + var alen = at.toString().length; + keys[at.toString().substr(0,alen-wlen)]=true; + at += window; + keys[at.toString().substr(0,alen-wlen)]=true; + at -= (window*2); + keys[at.toString().substr(0,alen-wlen)]=true; + Object.keys(keys).forEach(function(key){ item.keys[prefix+key] = item.ref; }); +} + +// extract info from a tweet +function itemTwitter(item, tweet) +{ + item.pri = 1; // tweets are the lowest priority? + + // since RTs contain the original, add the response first then process the original! + if(tweet.retweeted_status) + { + var resp = newResponse(item, "up"); + resp.from.id = "contact://twitter/#"+tweet.user.screen_name; + resp.from.name = tweet.user.name; + resp.from.icon = tweet.user.profile_image_url; + item.responses.push(resp); + tweet = tweet.retweeted_status; + item.keys['tweet://twitter/#'+tweet.id_str] = item.ref; // tag with the original too + } + + if(tweet.created_at) item.first = item.last = new Date(tweet.created_at).getTime(); + if(tweet.text) item.text = tweet.text; + if(tweet.user) + { + item.from.id = "contact://twitter/#"+tweet.user.screen_name; + item.from.name = tweet.user.name; + item.from.icon = tweet.user.profile_image_url; + item.froms[item.from.id] = item.ref; + } + // add a text based key + if(item.text) + { + var hash = crypto.createHash('md5'); + hash.update(item.text.substr(0,130)); // ignore trimming variations + item.keys['text:'+hash.digest('hex')] = item.ref; + hash = crypto.createHash('md5'); + hash.update(item.text.replace(/ http\:\/\/\S+$/,"")); // cut off appendege links that some apps add (like instagram) + item.keys['text:'+hash.digest('hex')] = item.ref; + } + // check all links + if(tweet.entities && tweet.entities.urls) tweet.entities.urls.forEach(function(link){if(link.expanded_url) keyUrl(item, link.expanded_url);}); + + // if this is also a reply + if(tweet.in_reply_to_status_id_str) + { + item.pri = 0; // demoted + item.keys['tweet://twitter/#'+tweet.in_reply_to_status_id_str] = item.ref; // hopefully this merges with original if it exists + var resp = newResponse(item, "comment"); + // duplicate stuff into response too + resp.text = item.text; + resp.at = item.first; + resp.from.id = item.from.id; + resp.from.name = item.from.name; + resp.from.icon = item.from.icon; + item.responses.push(resp); + } +} + +// extract info from a facebook post +function itemFacebook(item, post) +{ + item.pri = 2; // facebook allows more text, always better? + item.first = post.created_time * 1000; + item.last = post.updated_time * 1000; + if(post.from) + { + item.from.id = 'contact://facebook/#'+post.from.id; + item.from.name = post.from.name; + item.from.icon = 'https://graph.facebook.com/' + post.from.id + '/picture'; + item.froms[item.from.id] = item.ref; + } + if(post.name) item.title = post.name; + if(post.message) item.text = post.message; + if(!item.text && post.description) item.text = post.description; + if(!item.text && post.caption) item.text = post.caption; + if(!item.text && post.story) item.text = post.story; + // should we only add a text key if we can detect it's a tweet? we're de-duping any fb post essentially + if(item.text) + { + var hash = crypto.createHash('md5'); + hash.update(item.text.substr(0,130)); // ignore trimming variations + item.keys['text:'+hash.digest('hex')] = item.ref; + } + if(post.link) keyUrl(item, post.link); + if(!item.text && post.type == "photo") item.text = "New Photo"; + + // process responses! + if(post.comments && post.comments.data) + { + post.comments.data.forEach(function(comment){ + if(!comment.from) return; // anonymous comments skipped + var resp = newResponse(item, "comment"); + resp.at = comment.created_time * 1000; + resp.text = comment.message; + resp.from.id = 'contact://facebook/#'+comment.from.id; + resp.from.name = comment.from.name; + resp.from.icon = 'https://graph.facebook.com/' + comment.from.id + '/picture'; + item.responses.push(resp); + }); + } + if(post.likes && post.likes.data) + { + post.likes.data.forEach(function(like){ + var resp = newResponse(item, "up"); + resp.from.id = 'contact://facebook/#'+like.id; + resp.from.name = like.name; + resp.from.icon = 'https://graph.facebook.com/' + like.id + '/picture'; + item.responses.push(resp); + }); + } +} + +// extract info from a foursquare checkin +function itemFoursquare(item, checkin) +{ + item.pri = 3; // ideally a checkin should source here as the best + item.first = item.last = checkin.createdAt * 1000; + if(checkin.venue) item.text = "Checked in at " + checkin.venue.name; + if(checkin.shout) + { + item.text = checkin.shout; + var hash = crypto.createHash('md5'); + hash.update(item.text.substr(0,130)); // ignore trimming variations + item.keys['text:'+hash.digest('hex')] = item.ref; + } + // foursquare preserves no other metadata than the shout in a cross-post, so we fall back to timestamp window :( + if(checkin.photos && checkin.photos.items) checkin.photos.items.forEach(function(photo){ + if(!photo.source || photo.source.name != "Instagram") return; + keyTime(item, photo.createdAt, 5, "igts:"); + }); + var profile = checkin.user; + item.from.id = 'contact://foursquare/#'+profile.id; + item.from.name = profile.firstName + " " + profile.lastName; + item.from.icon = profile.photo; + item.froms[item.from.id] = item.ref; + if(checkin.comments && checkin.comments.items) + { + checkin.comments.items.forEach(function(comment){ + // TODO: can't find examples of this! + }); + } + +} + +// process twitter's post-tweet related data +function itemTwitterRelated(item, relateds) +{ + relateds.forEach(function(related){ + related.results.forEach(function(result){ + // ReTweet type by default is only the user object + var resp = newResponse(item, "up"); + var user = result; + if(related.resultType == "Tweet") { + resp.type = "comment"; + resp.text = result.value.text; + user = result.value.user; + resp.at = new Date(result.value.created_at).getTime(); + } + resp.from.id = "contact://twitter/#"+user.screen_name; + resp.from.name = user.name; + resp.from.icon = user.profile_image_url; + item.responses.push(resp); + }); + }); +} + +// extract info from an instagram pic +function itemInstagram(item, pic) +{ + item.pri = 4; // the source + item.first = item.last = pic.created_time * 1000; + if(pic.caption && pic.caption.text.length > 0) item.text = pic.caption.text; + if(pic.user) + { + item.from.id = 'contact://instagram/#'+pic.user.id; + item.from.name = pic.user.full_name; + item.from.icon = pic.user.profile_picture; + item.froms[item.from.id] = item.ref; + } + + if(item.text) + { + var hash = crypto.createHash('md5'); + hash.update(item.text.substr(0,130)); // ignore trimming variations + item.keys['text:'+hash.digest('hex')] = item.ref; + }else{ + item.text = "New Picture"; + // if there's no caption, we have to key the timestamp to catch foursquare cross-posting since it has no other metadata :( + keyTime(item, pic.created_time, 5, "igts:"); + } + + if(pic.link) keyUrl(item, pic.link); + + // process responses! + if(pic.comments && pic.comments.data) + { + pic.comments.data.forEach(function(comment){ + var resp = newResponse(item, "comment"); + resp.at = comment.created_time * 1000; + resp.text = comment.text; + resp.from.id = 'contact://instagram/#'+comment.from.id; + resp.from.name = comment.from.full_name; + resp.from.icon = comment.from.profile_picture; + item.responses.push(resp); + }); + } + if(pic.likes && pic.likes.data) + { + pic.likes.data.forEach(function(like){ + var resp = newResponse(item, "up"); + resp.from.id = 'contact://instagram/#'+like.id; + resp.from.name = like.full_name; + resp.from.icon = like.profile_picture; + item.responses.push(resp); + }); + } +} + diff --git a/Collections/Timeline/dataStore.js b/Collections/Timeline/dataStore.js new file mode 100644 index 000000000..59c607113 --- /dev/null +++ b/Collections/Timeline/dataStore.js @@ -0,0 +1,175 @@ +/* +* +* Copyright (C) 2011, The Locker Project +* All rights reserved. +* +* Please see the LICENSE file for more information. +* +*/ +var logger; +var lutil = require("lutil"); +var crypto = require("crypto"); +var async = require('async'); +var lmongoutil = require("lmongoutil"); +var lsql = require('sqlite-cursor'); + + +var itemCol, respCol; +var locker; +var mItem, mResp, mKey; + + +exports.init = function(iCollection, rCollection, l, callback) { + locker = l; + logger = l.logger; + itemCol = iCollection; + itemCol.ensureIndex({"id":1},{unique:true, background:true},function() {}); + itemCol.ensureIndex({"keys":1},{background:true},function() {}); + respCol = rCollection; + respCol.ensureIndex({"id":1},{unique:true, background:true},function() {}); + respCol.ensureIndex({"item":1},{background:true},function() {}); + async.series([ + function(cb) { + lsql.connectToDB("./timeline.sqlite", cb); + }, + function(cb) { + mKey = new lsql.Model("key", { + key:{type:lsql.Types.String, primaryKey:true}, + item:lsql.Types.String + }); + console.error() + mKey.create(cb); + }, + function(cb) { + mItem = new lsql.Model("item", { + item:{type:lsql.Types.String, primaryKey:true}, + json:lsql.Types.String + }); + mItem.create(cb); + }, + function(cb) { + mResp = new lsql.Model("response", { + item:{type:lsql.Types.String}, + json:lsql.Types.String + }); + mResp.create(cb); + }], callback); +} + +exports.clear = function(callback) { + itemCol.drop(function(){respCol.drop(callback)}); +} + +exports.getTotalItems = function(callback) { + itemCol.count(callback); +} +exports.getTotalResponses = function(callback) { + respCol.count(callback); +} + +exports.getAll = function(fields, callback) { + itemCol.find({}, fields, callback); +} + +exports.getLastObjectID = function(cbDone) { + itemCol.find({}, {fields:{_id:1}, limit:1, sort:{_id:-1}}).nextObject(cbDone); +} + +exports.getItemByKey = function(key, callback) { + var item; + var kname = "keys."+key; + var find = {}; + find[kname] = {$exists:true}; + findWrap(find,{limit:1},itemCol,function(i){item=i},function(err){callback(err,item)}); +} + +exports.getItem = function(id, callback) { + var item; + findWrap({id:id},{},itemCol,function(i){item=i},function(err){callback(err,item)}); +} + +// arg takes sort/limit/offset/find +exports.getResponses = function(arg, cbEach, cbDone) { + var f = (arg.item)?{item:arg.item}:{}; + delete arg.item; + if(arg.from) f["from.id"] = arg.from; + findWrap(f,arg,respCol,cbEach,cbDone); +} + +exports.getSince = function(arg, cbEach, cbDone) { + if(!arg || !arg.id) return cbDone("no id given"); + findWrap({"_id":{"$gt":lmongoutil.ObjectID(arg.id)}}, {sort:{_id:-1}}, itemCol, cbEach, cbDone); +} + +// arg takes sort/limit/offset/find +exports.getItems = function(arg, cbEach, cbDone) { + var f = {}; + try { + if(arg.find) f = JSON.parse(arg.find); // optional, can bomb out + }catch(E){ + return cbDone("couldn't parse find"); + } + delete arg.find; + if(arg.from) f["froms."+arg.from] = {$exists:true}; + findWrap(f,arg,itemCol,cbEach,cbDone); +} + +function findWrap(a,b,c,cbEach,cbDone){ +// console.log("a(" + JSON.stringify(a) + ") b("+ JSON.stringify(b) + ")"); + var cursor = c.find(a); + if (b.sort) cursor.sort(parseInt(b.sort)); + if (b.limit) cursor.limit(parseInt(b.limit)); + if (b.offset) cursor.skip(parseInt(b.offset)); + cursor.each(function(err, item) { + if (item != null) { + cbEach(item); + } else { + cbDone(err); + } + }); +} + + +// insert new (fully normalized) item, generate the id here and now +exports.addItem = function(item, callback) { + var etype = "update"; + if(!item.id) + { // first time an item comes in, make a unique id for it + var hash = crypto.createHash('md5'); + for(var i in item.keys) hash.update(i); + item.id = hash.digest('hex'); + etype = "new"; + } + var responses = item.responses; + if(responses) responses.forEach(function(r){ r.item = item.id; }); + delete item.responses; // store responses in their own table + delete item._id; // mongo is miss pissypants + itemCol.findAndModify({"id":item.id}, [['_id','asc']], {$set:item}, {safe:true, upsert:true, new: true}, function(err, doc){ + if(locker) locker.ievent(lutil.idrNew("item","timeline",doc.id),doc,etype); // let happen independently + if(err || !responses) return callback(err, doc); + async.forEach(responses, exports.addResponse, function(err){callback(err, doc);}); // orig caller wants saved item back + }); +} + +// responses are unique by their contents +exports.addResponse = function(response, callback) { + delete response.id; + delete response._id; // mongo is miss pissypants + var item = response.item; + delete response.item; + var hash = crypto.createHash('md5'); + hash.update(JSON.stringify(response)); + response.item = item; + response.id = hash.digest('hex'); + respCol.findAndModify({"id":response.id}, [['_id','asc']], {$set:response}, {safe:true, upsert:true, new: true}, callback); +} + +// so, yeah, and that +exports.delItem = function(id, callback) { + if(!id || id.length < 10) return callback("no or invalid id to del: "+id); + itemCol.remove({id:id}, function(err){ + if(err) return callback(err); + if(locker) locker.ievent(lutil.idrNew("item","timeline",doc.id),doc,"delete"); // let happen independently + respCol.remove({item:id}, callback); + }); +} diff --git a/Collections/Timeline/fixtures/facebook.tweet b/Collections/Timeline/fixtures/facebook.tweet new file mode 100644 index 000000000..20c66c565 --- /dev/null +++ b/Collections/Timeline/fixtures/facebook.tweet @@ -0,0 +1,37 @@ +{ + "action": "update", + "idr": "post://facebook/home?id=facebook#500012126_10150282386392127", + "data": { + "id": "500012126_10150282386392127", + "from": { + "name": "Alex Rainert", + "id": "500012126" + }, + "message": "Dan was the 1st to nail the Google/Motorola news and yesterday he nailed Netflix. Get on the @splatf analysis, people! http://t.co/ZMu8UkyK", + "icon": "http://photos-d.ak.fbcdn.net/photos-ak-snc1/v27562/23/2231777543/app_2_2231777543_9553.gif", + "actions": [ + { + "name": "Comment", + "link": "http://www.facebook.com/500012126/posts/10150282386392127" + }, + { + "name": "Like", + "link": "http://www.facebook.com/500012126/posts/10150282386392127" + }, + { + "name": "@arainert on Twitter", + "link": "http://twitter.com/arainert?utm_source=fb&utm_medium=fb&utm_campaign=arainert&utm_content=116149996419682304" + } + ], + "type": "status", + "application": { + "name": "Twitter", + "id": "2231777543" + }, + "created_time": 1316527292, + "updated_time": 1316527292, + "comments": { + "count": 0 + } + } +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/facebook.update b/Collections/Timeline/fixtures/facebook.update new file mode 100644 index 000000000..75c226222 --- /dev/null +++ b/Collections/Timeline/fixtures/facebook.update @@ -0,0 +1,64 @@ +{ + "action": "update", + "idr": "post://facebook/home?id=facebook#1264086533_2419826536086", + "data": { + "id": "1264086533_2419826536086", + "from": { + "name": "Mindframe Theaters", + "id": "1264086533" + }, + "picture": "http://vthumb.ak.fbcdn.net/hvthumb-ak-snc6/244520_2419869537161_2419826536086_33311_1352_t.jpg", + "link": "http://www.facebook.com/photo.php?v=2419826536086", + "source": "http://video.l3.fbcdn.net/cfs-l3-ash4/342015/380/2419826536086_60821.mp4?oh=115eecc26ea0e77e74bdfb39d3860a33&oe=4E72D400&l3s=20110914043344&l3e=20110916044344&lh=065388ed3f72cef427b4b", + "name": "The Tree of Life - 9/16 [HD]", + "properties": [ + { + "name": "Length", + "text": "2:11" + } + ], + "icon": "http://static.ak.fbcdn.net/rsrc.php/v1/yD/r/DggDhA4z4tO.gif", + "actions": [ + { + "name": "Comment", + "link": "http://www.facebook.com/1264086533/posts/2419826536086" + }, + { + "name": "Like", + "link": "http://www.facebook.com/1264086533/posts/2419826536086" + } + ], + "type": "video", + "object_id": "2419826536086", + "application": { + "name": "Video", + "id": "2392950137" + }, + "created_time": 1315859039, + "updated_time": 1315864735, + "likes": { + "data": [ + { + "name": "Will Kelly", + "id": "760380150" + } + ], + "count": 9 + }, + "comments": { + "data": [ + { + "id": "1264086533_2419826536086_3034673", + "from": { + "name": "Bryan J. Zygmont", + "id": "62106448" + }, + "message": "Thank you. See you soon.", + "created_time": 1315864735 + } + ], + "count": 1 + }, + "_id": "4e6e6d354f7f2ab135d2782b" + } +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/foursquare.recents b/Collections/Timeline/fixtures/foursquare.recents new file mode 100644 index 000000000..020e07810 --- /dev/null +++ b/Collections/Timeline/fixtures/foursquare.recents @@ -0,0 +1,64 @@ +{ + "action": "new", + "idr": "checkin://foursquare/recents?id=foursquare#4e703881d4c03fcb653e0251", + "data": { + "_id": "4e703a41ca17313fd2cccc6b", + "id": "4e703881d4c03fcb653e0251", + "createdAt": 1315977345, + "type": "checkin", + "timeZone": "America/Chicago", + "entities": [], + "user": { + "id": "10598838", + "firstName": "Lis", + "lastName": "Miller", + "photo": "https://playfoursquare.s3.amazonaws.com/userpix_thumbs/ZHSLPGESZ530RZEK.jpg", + "gender": "female", + "homeCity": "Cascade, IA", + "relationship": "friend" + }, + "venue": { + "id": "4e1362df18a84f0f0325cb37", + "name": "Home: 605 Garfield St SW", + "contact": {}, + "location": { + "address": "605 Garfield St. SW", + "lat": 42.292913, + "lng": -91.018424, + "city": "Cascade", + "state": "IA" + }, + "categories": [ + { + "id": "4bf58dd8d48988d103941735", + "name": "Home", + "pluralName": "Homes", + "shortName": "Home", + "icon": "https://foursquare.com/img/categories/building/home.png", + "parents": [ + "Residences" + ], + "primary": true + } + ], + "verified": false, + "stats": { + "checkinsCount": 246, + "usersCount": 4, + "tipCount": 0 + } + }, + "source": { + "name": "foursquare for iPhone", + "url": "https://foursquare.com/download/#/iphone" + }, + "photos": { + "count": 0, + "items": [] + }, + "comments": { + "count": 0, + "items": [] + } + } +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/ig.4sq b/Collections/Timeline/fixtures/ig.4sq new file mode 100644 index 000000000..e0ac671ef --- /dev/null +++ b/Collections/Timeline/fixtures/ig.4sq @@ -0,0 +1,106 @@ +{ + "idr": "checkin://foursquare/recents?id=foursquare#4eb0dfbbe3002b5426f05a51", + "action": "new", + "data": { + "id": "4eb0dfbbe3002b5426f05a51", + "createdAt": 1320214459, + "type": "checkin", + "shout": "11.1.11, 11:11", + "timeZone": "America/Los_Angeles", + "entities": [], + "user": { + "id": "1064267", + "firstName": "Jason", + "lastName": "Cavnar", + "photo": "https://playfoursquare.s3.amazonaws.com/userpix_thumbs/NVPLSYYLJHBHSIEW.jpg", + "gender": "male", + "homeCity": "Palo Alto, CA", + "relationship": "friend" + }, + "venue": { + "id": "4daf691893a037b7b06ee236", + "name": "SINGLY", + "contact": { + "twitter": "singlyinc" + }, + "location": { + "address": "777 Florida St", + "lat": 37.75926, + "lng": -122.410651, + "postalCode": "94110", + "city": "San Francisco", + "state": "CA", + "country": "USA" + }, + "categories": [{ + "id": "4bf58dd8d48988d125941735", + "name": "Tech Startup", + "pluralName": "Tech Startups", + "shortName": "Tech Startup", + "icon": "https://foursquare.com/img/categories/building/default.png", + "parents": ["Professional & Other Places", "Offices"], + "primary": true + }], + "verified": true, + "stats": { + "checkinsCount": 449, + "usersCount": 51, + "tipCount": 1 + }, + "url": "http://singly.com" + }, + "source": { + "name": "Instagram", + "url": "http://instagram.com" + }, + "photos": { + "count": 1, + "items": [{ + "id": "4eb0dfbcdab4a8a58a48dd35", + "createdAt": 1316385389, + "url": "https://playfoursquare.s3.amazonaws.com/pix/PBRGCC2IKYJVJN41TOI1SNQIUJY2U2DUOFTIOP5FNYZ4ZXWG.jpg", + "sizes": { + "count": 4, + "items": [{ + "url": "https://playfoursquare.s3.amazonaws.com/pix/PBRGCC2IKYJVJN41TOI1SNQIUJY2U2DUOFTIOP5FNYZ4ZXWG.jpg", + "width": 612, + "height": 612 + }, + { + "url": "https://playfoursquare.s3.amazonaws.com/derived_pix/PBRGCC2IKYJVJN41TOI1SNQIUJY2U2DUOFTIOP5FNYZ4ZXWG_300x300.jpg", + "width": 300, + "height": 300 + }, + { + "url": "https://playfoursquare.s3.amazonaws.com/derived_pix/PBRGCC2IKYJVJN41TOI1SNQIUJY2U2DUOFTIOP5FNYZ4ZXWG_100x100.jpg", + "width": 100, + "height": 100 + }, + { + "url": "https://playfoursquare.s3.amazonaws.com/derived_pix/PBRGCC2IKYJVJN41TOI1SNQIUJY2U2DUOFTIOP5FNYZ4ZXWG_36x36.jpg", + "width": 36, + "height": 36 + }] + }, + "source": { + "name": "Instagram", + "url": "http://instagram.com" + }, + "user": { + "id": "1064267", + "firstName": "Jason", + "lastName": "Cavnar", + "photo": "https://playfoursquare.s3.amazonaws.com/userpix_thumbs/NVPLSYYLJHBHSIEW.jpg", + "gender": "male", + "homeCity": "Palo Alto, CA", + "relationship": "friend" + }, + "visibility": "public" + }] + }, + "comments": { + "count": 0, + "items": [] + } + } +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/ig.fb b/Collections/Timeline/fixtures/ig.fb new file mode 100644 index 000000000..c4f17ff0c --- /dev/null +++ b/Collections/Timeline/fixtures/ig.fb @@ -0,0 +1,38 @@ +{ + "action": "update", + "idr": "post://facebook/home?id=facebook#4803283_730289247379", + "data": { + "id": "4803283_730289247379", + "from": { + "name": "Julian Missig", + "id": "4803283" + }, + "message": "Model S model", + "picture": "http://platform.ak.fbcdn.net/www/app_full_proxy.php?app=124024574287414&v=1&size=z&cksum=437fa91b29c88f863ebfe460c8aa989b&src=http%3A%2F%2Fimages.instagram.com%2Fmedia%2F2011%2F09%2F18%2F391a26f2de1f47b089f4a2a2215b6bf7_7.jpg", + "link": "http://instagr.am/p/Ne6nr/", + "name": "jmissig's photo", + "caption": "instagr.am", + "description": "jmissig's photo on Instagram", + "icon": "http://photos-e.ak.fbcdn.net/photos-ak-snc1/v27562/10/124024574287414/app_2_124024574287414_6936.gif", + "actions": [ + { + "name": "Comment", + "link": "http://www.facebook.com/4803283/posts/730289247379" + }, + { + "name": "Like", + "link": "http://www.facebook.com/4803283/posts/730289247379" + } + ], + "type": "link", + "application": { + "name": "Instagram", + "id": "124024574287414" + }, + "created_time": 1316385395, + "updated_time": 1316385395, + "comments": { + "count": 0 + } + } +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/ig.instagram b/Collections/Timeline/fixtures/ig.instagram new file mode 100644 index 000000000..21e9f6f94 --- /dev/null +++ b/Collections/Timeline/fixtures/ig.instagram @@ -0,0 +1,61 @@ +{ + "action": "new", + "idr": "photo://instagram/feed?id=instagram#226208235_981179", + "data": { + "tags": [], + "type": "image", + "location": { + "latitude": 37.44355, + "longitude": -122.1706 + }, + "comments": { + "count": 0, + "data": [] + }, + "filter": "Normal", + "created_time": "1316385392", + "link": "http://instagr.am/p/Ne6nr/", + "likes": { + "count": 0, + "data": [] + }, + "images": { + "low_resolution": { + "url": "http://distillery.s3.amazonaws.com/media/2011/09/18/391a26f2de1f47b089f4a2a2215b6bf7_6.jpg", + "width": 306, + "height": 306 + }, + "thumbnail": { + "url": "http://distillery.s3.amazonaws.com/media/2011/09/18/391a26f2de1f47b089f4a2a2215b6bf7_5.jpg", + "width": 150, + "height": 150 + }, + "standard_resolution": { + "url": "http://distillery.s3.amazonaws.com/media/2011/09/18/391a26f2de1f47b089f4a2a2215b6bf7_7.jpg", + "width": 612, + "height": 612 + } + }, + "caption": { + "created_time": "1316385394", + "text": "Model S model", + "from": { + "username": "jmissig", + "profile_picture": "http://images.instagram.com/profiles/profile_981179_75sq_1297666725.jpg", + "id": "981179", + "full_name": "Julian Missig" + }, + "id": "277184261" + }, + "user_has_liked": false, + "id": "226208235_981179", + "user": { + "username": "jmissig", + "website": "", + "bio": "", + "profile_picture": "http://images.instagram.com/profiles/profile_981179_75sq_1297666725.jpg", + "full_name": "Julian Missig", + "id": "981179" + } + } +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/ig.tweet b/Collections/Timeline/fixtures/ig.tweet new file mode 100644 index 000000000..b7267ce07 --- /dev/null +++ b/Collections/Timeline/fixtures/ig.tweet @@ -0,0 +1,80 @@ +{ + "action": "new", + "idr": "tweet://twitter/timeline?id=twitter#115554837076258816", + "data": { + "in_reply_to_status_id_str": null, + "truncated": false, + "in_reply_to_user_id_str": null, + "geo": null, + "retweet_count": 0, + "contributors": null, + "coordinates": null, + "user": { + "statuses_count": 4902, + "favourites_count": 111, + "protected": false, + "profile_text_color": "333333", + "profile_image_url": "http://a1.twimg.com/profile_images/31867902/Julian_Missig_1_normal.png", + "name": "Julian Missig", + "profile_sidebar_fill_color": "A0C5C7", + "listed_count": 22, + "following": true, + "profile_background_tile": false, + "utc_offset": -28800, + "description": "interaction designer & prototyper", + "location": "Redwood City, CA", + "contributors_enabled": false, + "verified": false, + "profile_link_color": "369EAF", + "followers_count": 248, + "url": null, + "profile_sidebar_border_color": "86A4A6", + "screen_name": "jmissig", + "default_profile_image": false, + "notifications": false, + "default_profile": false, + "show_all_inline_media": false, + "geo_enabled": true, + "profile_use_background_image": true, + "friends_count": 186, + "id_str": "9147872", + "is_translator": false, + "lang": "en", + "time_zone": "Pacific Time (US & Canada)", + "created_at": "Fri Sep 28 19:19:58 +0000 2007", + "profile_background_color": "709397", + "id": 9147872, + "follow_request_sent": false, + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme6/bg.gif", + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme6/bg.gif", + "profile_image_url_https": "https://si0.twimg.com/profile_images/31867902/Julian_Missig_1_normal.png" + }, + "favorited": false, + "in_reply_to_screen_name": null, + "source": "Instagram", + "id_str": "115554837076258816", + "entities": { + "hashtags": [], + "urls": [ + { + "indices": [ + 14, + 34 + ], + "url": "http://t.co/PeedUocW", + "display_url": "instagr.am/p/Ne6nr/", + "expanded_url": "http://instagr.am/p/Ne6nr/" + } + ], + "user_mentions": [] + }, + "in_reply_to_status_id": null, + "id": 115554837076258820, + "created_at": "Sun Sep 18 22:36:34 +0000 2011", + "possibly_sensitive": false, + "place": null, + "retweeted": false, + "in_reply_to_user_id": null, + "text": "Model S ... http://t.co/PeedUocW" + } +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/instagram.feed b/Collections/Timeline/fixtures/instagram.feed new file mode 100644 index 000000000..779fd3723 --- /dev/null +++ b/Collections/Timeline/fixtures/instagram.feed @@ -0,0 +1,83 @@ +{ + "action": "new", + "idr": "photo://instagram/feed?id=instagram#294061795_2920312", + "data": { + "tags": [], + "type": "image", + "location": null, + "comments": { + "count": 1, + "data": [ + { + "created_time": "1319651890", + "text": "Great pic, love the colors", + "from": { + "username": "bradl3y", + "profile_picture": "http://images.instagram.com/profiles/profile_11076900_75sq_1319234253.jpg", + "id": "11076900", + "full_name": "Brad Barnett" + }, + "id": "367123985" + } + ] + }, + "filter": "Normal", + "created_time": "1319642091", + "link": "http://instagr.am/p/Rhwbj/", + "likes": { + "count": 2, + "data": [ + { + "username": "seanfeezy", + "profile_picture": "http://images.instagram.com/profiles/profile_8524340_75sq_1314153519.jpg", + "id": "8524340", + "full_name": "Sean Fissel" + }, + { + "username": "bradl3y", + "profile_picture": "http://images.instagram.com/profiles/profile_11076900_75sq_1319234253.jpg", + "id": "11076900", + "full_name": "Brad Barnett" + } + ] + }, + "images": { + "low_resolution": { + "url": "http://images.instagram.com/media/2011/10/26/f47c9ff00aff4ae9a6f8491d24722308_6.jpg", + "width": 306, + "height": 306 + }, + "thumbnail": { + "url": "http://images.instagram.com/media/2011/10/26/f47c9ff00aff4ae9a6f8491d24722308_5.jpg", + "width": 150, + "height": 150 + }, + "standard_resolution": { + "url": "http://images.instagram.com/media/2011/10/26/f47c9ff00aff4ae9a6f8491d24722308_7.jpg", + "width": 612, + "height": 612 + } + }, + "caption": { + "created_time": "1319642158", + "text": "Morning Mission. ", + "from": { + "username": "jasoncavnar", + "profile_picture": "http://images.instagram.com/profiles/profile_2920312_75sq_1316296467.jpg", + "id": "2920312", + "full_name": "Jason Cavnar" + }, + "id": "366806429" + }, + "user_has_liked": false, + "id": "294061795_2920312", + "user": { + "username": "jasoncavnar", + "website": "http://www.singly.com", + "bio": "That guy. Using Instagr.am to try to keep up with my hipster friends and their tight jeans. ", + "profile_picture": "http://images.instagram.com/profiles/profile_2920312_75sq_1316296467.jpg", + "full_name": "Jason Cavnar", + "id": "2920312" + } + } +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/item b/Collections/Timeline/fixtures/item new file mode 100644 index 000000000..c60498ec6 --- /dev/null +++ b/Collections/Timeline/fixtures/item @@ -0,0 +1,23 @@ +{ + "id":"4e703a41ca17313fd2cccc6b", + "first":12345566778, + "last":12356456779, + "keys":{ + "tweet://twitter/#235123423421":"tweet://twitter/timeline?id=twitter#235123423421", + }, + "ref":"tweet://twitter/timeline?id=twitter#235123423421", + "refs":[ + "tweet://twitter/timeline?id=twitter#235123423421", + ], + pri:4, + me:false, + text:"This is my text", + from:{ + "id":"contact://twitter/#jeremie", + "icon":"http://twitter.com/icon.png", + "name":"Jer" + }, + froms:{ + "contact://twitter/#jeremie":"tweet://twitter/timeline?id=twitter#235123423421" + } +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/link b/Collections/Timeline/fixtures/link new file mode 100644 index 000000000..bffc77bc4 --- /dev/null +++ b/Collections/Timeline/fixtures/link @@ -0,0 +1,100 @@ +{ + "idr": "link://links/#FOOOOOOOOO", + "action": "new", + "data": { + "at": 1320220061000, + "id": "FOOOOOOOOO", + "favicon": "http://t.co/favicon.ico", + "link": "http://t.co/QgLsBzHy", + "encounters": [{ + "_hash": "twitter:131638596414353400:http://t.co/QgLsBzHy", + "_id": "4eb0f60f8cc09f2507e66441", + "at": 1320220061000, + "from": "Markus Persson", + "fromID": 63485337, + "id": 131638596414353400, + "link": "https://foursquare.com/reecepacheco/checkin/4e703881d4c03fcb653e0251?s=a0huhS2FUrBtHfHd_oK2hvyvbME&ref=tw", + "network": "twitter", + "orig": "http://t.co/QgLsBzHy", + "text": "http://t.co/QgLsBzHy notch", + "via": { + "_id": "4e702f08bcb6f2f6dd06c849", + "in_reply_to_user_id_str": null, + "retweet_count": 0, + "in_reply_to_status_id": null, + "id_str": "113829905090879488", + "contributors": null, + "truncated": false, + "geo": null, + "coordinates": null, + "favorited": false, + "user": { + "default_profile": false, + "contributors_enabled": false, + "profile_use_background_image": true, + "protected": false, + "id_str": "633", + "time_zone": "Eastern Time (US & Canada)", + "profile_background_color": "1A1B1F", + "name": "danah boyd", + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme9/bg.gif", + "profile_image_url_https": "https://si0.twimg.com/profile_images/890666764/MeAtTalkWithHat-Sq_normal.jpg", + "default_profile_image": false, + "following": true, + "utc_offset": -18000, + "profile_image_url": "http://a0.twimg.com/profile_images/890666764/MeAtTalkWithHat-Sq_normal.jpg", + "description": "social media scholar, youth researcher & advocate | Microsoft Research, Harvard Berkman Center | zephoria@zephoria.org", + "show_all_inline_media": false, + "geo_enabled": false, + "friends_count": 734, + "profile_text_color": "666666", + "location": "Boston, MA", + "is_translator": false, + "profile_sidebar_fill_color": "252429", + "follow_request_sent": false, + "profile_background_tile": false, + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme9/bg.gif", + "url": "http://www.zephoria.org/thoughts/", + "statuses_count": 2471, + "followers_count": 50313, + "screen_name": "zephoria", + "notifications": false, + "profile_link_color": "2FC2EF", + "lang": "en", + "verified": false, + "favourites_count": 44, + "profile_sidebar_border_color": "181A1E", + "id": 633, + "listed_count": 5224, + "created_at": "Thu Jul 13 21:27:23 +0000 2006" + }, + "in_reply_to_user_id": null, + "in_reply_to_screen_name": null, + "possibly_sensitive": false, + "retweeted": false, + "source": "web", + "in_reply_to_status_id_str": null, + "entities": { + "user_mentions": [{ + "name": "Wil Wheaton", + "id_str": "1183041", + "indices": [108, 113], + "screen_name": "wilw", + "id": 1183041 + }], + "hashtags": [], + "urls": [{ + "indices": [116, 135], + "display_url": "amzn.to/qqvlo4", + "url": "http://t.co/zAN4G8S", + "expanded_url": "http://amzn.to/qqvlo4" + }] + }, + "id": 113829905090879490, + "place": null, + "text": "Just finished \"Ready Player One.\" YA dystopia mixed w/ 1980s geek nostalgia. Soooo much fun. (Audio read by @wilw ) http://t.co/zAN4G8S", + "created_at": "Wed Sep 14 04:22:18 +0000 2011" + } + }] + } +} diff --git a/Collections/Timeline/fixtures/response b/Collections/Timeline/fixtures/response new file mode 100644 index 000000000..bedca6e3a --- /dev/null +++ b/Collections/Timeline/fixtures/response @@ -0,0 +1,13 @@ +{ + "id":"hash generated from all KVs below", + "item":"4e703a41ca17313fd2cccc6b", + "at":12345566778, + "ref":"tweet://twitter/timeline?id=twitter#235123423421", + "text":"This is my text", + "type":"comment" + from:{ + "id":"contact://twitter/#jeremie", + "icon":"http://twitter.com/icon.png", + "name":"Jer" + }, +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/tweet.facebook b/Collections/Timeline/fixtures/tweet.facebook new file mode 100644 index 000000000..2d128348b --- /dev/null +++ b/Collections/Timeline/fixtures/tweet.facebook @@ -0,0 +1,91 @@ +{ + "action": "new", + "idr": "tweet://twitter/timeline?id=twitter#116149996419682304", + "data": { + "contributors": null, + "truncated": false, + "geo": null, + "favorited": false, + "coordinates": null, + "user": { + "profile_text_color": "393939", + "protected": false, + "listed_count": 457, + "profile_sidebar_fill_color": "e8e8e8", + "name": "Alex Rainert", + "contributors_enabled": false, + "profile_background_tile": true, + "profile_image_url_https": "https://si0.twimg.com/profile_images/1331003463/arainert_headshot2_supersm_normal.jpg", + "profile_image_url": "http://a0.twimg.com/profile_images/1331003463/arainert_headshot2_supersm_normal.jpg", + "default_profile": false, + "show_all_inline_media": true, + "following": true, + "geo_enabled": true, + "utc_offset": -18000, + "description": "Husband, father & owner of a lovably neurotic vizsla. Obsessed with information, design, emerging tech, sports & food. Head of product at foursquare.", + "profile_link_color": "ff4cb4", + "location": "Brooklyn, NY", + "default_profile_image": false, + "verified": false, + "profile_sidebar_border_color": "ff4cb4", + "statuses_count": 15165, + "friends_count": 511, + "followers_count": 4717, + "url": "http://www.alexrainert.com/", + "is_translator": false, + "profile_use_background_image": true, + "screen_name": "arainert", + "follow_request_sent": false, + "notifications": false, + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/17611084/walken_small.jpg", + "time_zone": "Eastern Time (US & Canada)", + "favourites_count": 68, + "profile_background_color": "352726", + "lang": "en", + "profile_background_image_url": "http://a1.twimg.com/profile_background_images/17611084/walken_small.jpg", + "id": 7482, + "id_str": "7482", + "created_at": "Wed Oct 04 18:33:19 +0000 2006" + }, + "in_reply_to_screen_name": null, + "possibly_sensitive": false, + "retweeted": false, + "in_reply_to_status_id_str": null, + "source": "Tweet Button", + "entities": { + "user_mentions": [ + { + "name": "SplatF", + "indices": [ + 93, + 100 + ], + "screen_name": "splatf", + "id": 174886575, + "id_str": "174886575" + } + ], + "hashtags": [], + "urls": [ + { + "display_url": "splatf.com/2011/09/netfli…", + "indices": [ + 119, + 139 + ], + "expanded_url": "http://www.splatf.com/2011/09/netflix-qwikster-facts/", + "url": "http://t.co/ZMu8UkyK" + } + ] + }, + "in_reply_to_user_id": null, + "id": 116149996419682300, + "in_reply_to_user_id_str": null, + "id_str": "116149996419682304", + "place": null, + "retweet_count": 0, + "in_reply_to_status_id": null, + "text": "Dan was the 1st to nail the Google/Motorola news and yesterday he nailed Netflix. Get on the @splatf analysis, people! http://t.co/ZMu8UkyK", + "created_at": "Tue Sep 20 14:01:31 +0000 2011" + } +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/twitter.related b/Collections/Timeline/fixtures/twitter.related new file mode 100644 index 000000000..cf220dff5 --- /dev/null +++ b/Collections/Timeline/fixtures/twitter.related @@ -0,0 +1,1241 @@ +{ + "action": "new", + "idr": "tweet://twitter/related?id=twitter#109553278831951872", + "data": { + "id": "109553278831951872", + "related": [ + { + "results": [ + { + "score": 1, + "annotations": { + "ConversationRole": "Fork" + }, + "kind": "Tweet", + "value": { + "id_str": "109570326760919041", + "in_reply_to_status_id": 109553278831951870, + "truncated": false, + "user": { + "profile_sidebar_fill_color": "95E8EC", + "protected": false, + "id_str": "79202245", + "default_profile": false, + "notifications": null, + "profile_background_tile": false, + "screen_name": "Tiduilasa", + "name": "Tidu", + "listed_count": 1, + "location": "Brickman Road, S.Fburg. NY/USA", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": -21600, + "profile_link_color": "0099B9", + "description": "YOGI BHAGAVATA-DHARMA\r\n_______________________", + "profile_sidebar_border_color": "5ED4DC", + "url": "http://www.siddhayoga.org/gurumayi-chidvilasananda", + "time_zone": "Central Time (US & Canada)", + "default_profile_image": false, + "statuses_count": 85, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 12, + "friends_count": 5, + "profile_background_color": "0099B9", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme4/bg.gif", + "created_at": "Fri Oct 02 15:01:45 +0000 2009", + "followers_count": 3, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme4/bg.gif", + "id": 79202245, + "profile_text_color": "3C3940", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1522182333/aum-message-artwork_1__normal.jpg", + "profile_image_url": "http://a2.twimg.com/profile_images/1522182333/aum-message-artwork_1__normal.jpg" + }, + "favorited": false, + "possibly_sensitive": false, + "in_reply_to_status_id_str": "109553278831951872", + "geo": null, + "in_reply_to_screen_name": "DalaiLama", + "in_reply_to_user_id_str": "20609518", + "coordinates": null, + "in_reply_to_user_id": 20609518, + "source": "web", + "created_at": "Fri Sep 02 10:16:16 +0000 2011", + "contributors": null, + "retweeted": false, + "retweet_count": 0, + "id": 109570326760919040, + "place": null, + "text": "@DalaiLama ~ Fowers to Mr, Dalai; (was called \"Delair\" - my papy)\nhttp://t.co/MXYIaaS" + } + }, + { + "score": 1, + "annotations": { + "ConversationRole": "Fork" + }, + "kind": "Tweet", + "value": { + "id_str": "109569700891082752", + "in_reply_to_status_id": 109553278831951870, + "truncated": false, + "user": { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "114698131", + "default_profile": false, + "notifications": null, + "profile_background_tile": false, + "screen_name": "jeshvik77", + "name": "rajesh", + "listed_count": 1, + "location": "KNR.HYD.GHY..", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": -36000, + "profile_link_color": "0084B4", + "description": "Hypo r Hyper n still trying for Normo, 'll bit narcissistic, as of nw Hurt-Locker, pasion n EM,belivin fnds r lyf...", + "profile_sidebar_border_color": "C0DEED", + "url": null, + "time_zone": "Hawaii", + "default_profile_image": false, + "statuses_count": 2136, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 4, + "friends_count": 37, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a2.twimg.com/profile_background_images/308608018/PIX_094-58.jpg", + "created_at": "Tue Feb 16 09:30:33 +0000 2010", + "followers_count": 49, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/308608018/PIX_094-58.jpg", + "id": 114698131, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1452584974/uu_normal.jpg", + "profile_image_url": "http://a1.twimg.com/profile_images/1452584974/uu_normal.jpg" + }, + "favorited": false, + "in_reply_to_status_id_str": "109553278831951872", + "geo": null, + "in_reply_to_screen_name": "DalaiLama", + "in_reply_to_user_id_str": "20609518", + "coordinates": null, + "in_reply_to_user_id": 20609518, + "source": "web", + "created_at": "Fri Sep 02 10:13:47 +0000 2011", + "contributors": null, + "retweeted": false, + "retweet_count": 0, + "id": 109569700891082750, + "place": null, + "text": "@DalaiLama \"@jeshvik77 The practice of altruism is the authentic way of living a human life; it is not limited only to the religious\"" + } + }, + { + "score": 1, + "annotations": { + "ConversationRole": "Fork" + }, + "kind": "Tweet", + "value": { + "id_str": "109569691311288321", + "in_reply_to_status_id": 109553278831951870, + "truncated": false, + "user": { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "351589243", + "default_profile": true, + "notifications": null, + "profile_background_tile": false, + "screen_name": "nukmi", + "name": "nukmi", + "listed_count": 0, + "location": "", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": null, + "profile_link_color": "0084B4", + "description": "human being from south east london, a real tyrant :)", + "profile_sidebar_border_color": "C0DEED", + "url": null, + "time_zone": null, + "default_profile_image": true, + "statuses_count": 24, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 0, + "friends_count": 11, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Tue Aug 09 13:26:35 +0000 2011", + "followers_count": 3, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 351589243, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/sticky/default_profile_images/default_profile_5_normal.png", + "profile_image_url": "http://a0.twimg.com/sticky/default_profile_images/default_profile_5_normal.png" + }, + "favorited": false, + "in_reply_to_status_id_str": "109553278831951872", + "geo": null, + "in_reply_to_screen_name": "DalaiLama", + "in_reply_to_user_id_str": "20609518", + "coordinates": null, + "in_reply_to_user_id": 20609518, + "source": "web", + "created_at": "Fri Sep 02 10:13:44 +0000 2011", + "contributors": null, + "retweeted": false, + "retweet_count": 0, + "id": 109569691311288320, + "place": null, + "text": "@DalaiLama does HHDL have any thoughts to share on the role of cinema & film in creating a moral/altruistic society?" + } + }, + { + "score": 1, + "annotations": { + "ConversationRole": "Fork" + }, + "kind": "Tweet", + "value": { + "id_str": "109568108846530560", + "in_reply_to_status_id": 109553278831951870, + "truncated": false, + "user": { + "profile_sidebar_fill_color": "efefef", + "protected": false, + "id_str": "96293863", + "default_profile": false, + "notifications": null, + "profile_background_tile": true, + "screen_name": "SofistaPrateado", + "name": "Renato(Rewah Hameth)", + "listed_count": 1, + "location": "Ilha da Rainha da Morte - RJ", + "show_all_inline_media": true, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": -10800, + "profile_link_color": "009999", + "description": "Bancário, gamer, lutador de Kung-fu, escritor de contos, tarólogo, acupuntor, leitor, futurista e pseudo-humorista. Assim sou/estou eu.", + "profile_sidebar_border_color": "eeeeee", + "url": "http://sofistaprateado.tumblr.com", + "time_zone": "Brasilia", + "default_profile_image": false, + "statuses_count": 11251, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 76, + "friends_count": 123, + "profile_background_color": "131516", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme14/bg.gif", + "created_at": "Sat Dec 12 06:54:04 +0000 2009", + "followers_count": 123, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme14/bg.gif", + "id": 96293863, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1514569915/aquiles_normal.JPG", + "profile_image_url": "http://a3.twimg.com/profile_images/1514569915/aquiles_normal.JPG" + }, + "favorited": false, + "in_reply_to_status_id_str": "109553278831951872", + "geo": null, + "in_reply_to_screen_name": "DalaiLama", + "in_reply_to_user_id_str": "20609518", + "coordinates": null, + "in_reply_to_user_id": 20609518, + "source": "web", + "created_at": "Fri Sep 02 10:07:27 +0000 2011", + "contributors": null, + "retweeted": false, + "retweet_count": 0, + "id": 109568108846530560, + "place": null, + "text": "A prática de altruísmo é o modo autêntico de viver uma vida humana; não é limitada apenas aos religiosos. (via @DalaiLama) #fb" + } + }, + { + "score": 1, + "annotations": { + "ConversationRole": "Fork" + }, + "kind": "Tweet", + "value": { + "id_str": "109567960246530048", + "in_reply_to_status_id": 109553278831951870, + "truncated": false, + "user": { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "76316278", + "default_profile": true, + "notifications": null, + "profile_background_tile": false, + "screen_name": "DrMradul", + "name": "Mradul Kaushik", + "listed_count": 0, + "location": null, + "show_all_inline_media": false, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": null, + "profile_link_color": "0084B4", + "description": null, + "profile_sidebar_border_color": "C0DEED", + "url": null, + "time_zone": null, + "default_profile_image": true, + "statuses_count": 21, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 1, + "friends_count": 26, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Tue Sep 22 12:11:22 +0000 2009", + "followers_count": 9, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 76316278, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/sticky/default_profile_images/default_profile_3_normal.png", + "profile_image_url": "http://a2.twimg.com/sticky/default_profile_images/default_profile_3_normal.png" + }, + "favorited": false, + "in_reply_to_status_id_str": "109553278831951872", + "geo": null, + "in_reply_to_screen_name": "DalaiLama", + "in_reply_to_user_id_str": "20609518", + "coordinates": null, + "in_reply_to_user_id": 20609518, + "source": "Twitter for BlackBerry®", + "created_at": "Fri Sep 02 10:06:52 +0000 2011", + "contributors": null, + "retweeted": false, + "retweet_count": 0, + "id": 109567960246530050, + "place": null, + "text": "@DalaiLama whatever is to be followed righteously is the true religion" + } + }, + { + "score": 1, + "annotations": { + "ConversationRole": "Fork" + }, + "kind": "Tweet", + "value": { + "id_str": "109567497652543488", + "in_reply_to_status_id": 109553278831951870, + "truncated": false, + "user": { + "profile_sidebar_fill_color": "7AC3EE", + "protected": false, + "id_str": "186321654", + "default_profile": false, + "notifications": null, + "profile_background_tile": true, + "screen_name": "antdigdogdagity", + "name": "Antony Blake Munro", + "listed_count": 0, + "location": "New Zealand, Auckland", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": 43200, + "profile_link_color": "FF0000", + "description": "I like magic, witchcraft, voodoo, buddhism, any thing goes, entertain me.", + "profile_sidebar_border_color": "65B0DA", + "url": "http://www.facebook.com/antony.munro", + "time_zone": "Auckland", + "default_profile_image": false, + "statuses_count": 79, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 1, + "friends_count": 173, + "profile_background_color": "642D8B", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme10/bg.gif", + "created_at": "Fri Sep 03 03:51:49 +0000 2010", + "followers_count": 26, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme10/bg.gif", + "id": 186321654, + "profile_text_color": "3D1957", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1509387282/antdog_normal.jpg", + "profile_image_url": "http://a0.twimg.com/profile_images/1509387282/antdog_normal.jpg" + }, + "favorited": false, + "in_reply_to_status_id_str": "109553278831951872", + "geo": null, + "in_reply_to_screen_name": "DalaiLama", + "in_reply_to_user_id_str": "20609518", + "coordinates": null, + "in_reply_to_user_id": 20609518, + "source": "web", + "created_at": "Fri Sep 02 10:05:01 +0000 2011", + "contributors": null, + "retweeted": false, + "retweet_count": 0, + "id": 109567497652543490, + "place": null, + "text": "@DalaiLama altruism is very interesting ill think about that." + } + }, + { + "score": 1, + "annotations": { + "ConversationRole": "Fork" + }, + "kind": "Tweet", + "value": { + "id_str": "109566564193402880", + "in_reply_to_status_id": 109553278831951870, + "truncated": false, + "user": { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "261139101", + "default_profile": true, + "notifications": null, + "profile_background_tile": false, + "screen_name": "JohnGeoMatthews", + "name": "John George Matthews", + "listed_count": 0, + "location": "Margate", + "show_all_inline_media": true, + "contributors_enabled": false, + "following": null, + "geo_enabled": true, + "utc_offset": 0, + "profile_link_color": "0084B4", + "description": "Morris Minor wedding car driver, www.johngmatthews.co.uk, car restorer, guitarist a band:- http://www.mentalblock.info, guitar teacher, mobile recordist. ", + "profile_sidebar_border_color": "C0DEED", + "url": "http://www.johngmatthews.co.uk/", + "time_zone": "London", + "default_profile_image": false, + "statuses_count": 314, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 0, + "friends_count": 86, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Sat Mar 05 09:37:20 +0000 2011", + "followers_count": 20, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 261139101, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1262832286/Twitter_pic_normal.jpg", + "profile_image_url": "http://a1.twimg.com/profile_images/1262832286/Twitter_pic_normal.jpg" + }, + "favorited": false, + "in_reply_to_status_id_str": "109553278831951872", + "geo": null, + "in_reply_to_screen_name": "DalaiLama", + "in_reply_to_user_id_str": "20609518", + "coordinates": null, + "in_reply_to_user_id": 20609518, + "source": "web", + "created_at": "Fri Sep 02 10:01:19 +0000 2011", + "contributors": null, + "retweeted": false, + "retweet_count": 0, + "id": 109566564193402880, + "place": null, + "text": "@DalaiLama- altruism- If only more people would live by this, then the world would be a much better place. altruism = all true ism" + } + }, + { + "score": 1, + "annotations": { + "ConversationRole": "Fork" + }, + "kind": "Tweet", + "value": { + "id_str": "109566508916670464", + "in_reply_to_status_id": 109553278831951870, + "truncated": false, + "user": { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "362947094", + "default_profile": true, + "notifications": null, + "profile_background_tile": false, + "screen_name": "KARANHIRWAT", + "name": "KARANHIRWAT", + "listed_count": 0, + "location": "", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": null, + "profile_link_color": "0084B4", + "description": "", + "profile_sidebar_border_color": "C0DEED", + "url": null, + "time_zone": null, + "default_profile_image": false, + "statuses_count": 90, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 1, + "friends_count": 76, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Sat Aug 27 07:41:23 +0000 2011", + "followers_count": 16, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 362947094, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1515548535/karan_new_pic_normal.jpg", + "profile_image_url": "http://a0.twimg.com/profile_images/1515548535/karan_new_pic_normal.jpg" + }, + "favorited": false, + "in_reply_to_status_id_str": "109553278831951872", + "geo": null, + "in_reply_to_screen_name": "DalaiLama", + "in_reply_to_user_id_str": "20609518", + "coordinates": null, + "in_reply_to_user_id": 20609518, + "source": "web", + "created_at": "Fri Sep 02 10:01:06 +0000 2011", + "contributors": null, + "retweeted": false, + "retweet_count": 0, + "id": 109566508916670460, + "place": null, + "text": "@DalaiLama The tongue like a sharp knife... Kills without drawing blood" + } + } + ], + "resultType": "Tweet", + "groupName": "TweetsWithConversation", + "annotations": { + "FromUser": "DalaiLama" + }, + "score": 1 + }, + { + "results": [ + { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "348062004", + "notifications": null, + "profile_background_tile": false, + "screen_name": "seshdaherbaliss", + "name": "seshi", + "listed_count": 0, + "location": "tema", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": null, + "profile_link_color": "0084B4", + "description": "last killer", + "profile_sidebar_border_color": "C0DEED", + "url": null, + "default_profile": true, + "time_zone": null, + "default_profile_image": true, + "statuses_count": 60, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 0, + "friends_count": 13, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Wed Aug 03 20:26:46 +0000 2011", + "followers_count": 7, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 348062004, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/sticky/default_profile_images/default_profile_3_normal.png", + "profile_image_url": "http://a2.twimg.com/sticky/default_profile_images/default_profile_3_normal.png" + }, + { + "profile_sidebar_fill_color": "99CC33", + "protected": false, + "id_str": "107930046", + "notifications": false, + "profile_background_tile": true, + "screen_name": "paolaandrea25", + "name": "Paola Andrea☆", + "listed_count": 0, + "location": "Bogota", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": true, + "utc_offset": -18000, + "profile_link_color": "D02B55", + "description": "Adm. de Empresas, Bogotana!!! desarrollando mi proyecto SHAROVA DAY SPA, Me gusta Soñar en cada momento... Dios siempre cumple mis sueños !) 100% Feliz. ", + "profile_sidebar_border_color": "829D5E", + "url": null, + "time_zone": "Bogota", + "default_profile_image": false, + "statuses_count": 74, + "profile_use_background_image": true, + "default_profile": false, + "verified": false, + "favourites_count": 5, + "friends_count": 278, + "profile_background_color": "352726", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/profile_background_images/294286892/kjhkjk.jpg", + "created_at": "Sun Jan 24 06:39:53 +0000 2010", + "followers_count": 75, + "follow_request_sent": false, + "lang": "es", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/294286892/kjhkjk.jpg", + "id": 107930046, + "profile_text_color": "3E4415", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1431039138/IMG01029-20100603-1450_normal.jpg", + "profile_image_url": "http://a0.twimg.com/profile_images/1431039138/IMG01029-20100603-1450_normal.jpg" + }, + { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "273380258", + "notifications": false, + "profile_background_tile": false, + "screen_name": "Positivestarts", + "name": "Positive Starts", + "listed_count": 0, + "location": null, + "show_all_inline_media": true, + "contributors_enabled": false, + "following": false, + "geo_enabled": false, + "utc_offset": -18000, + "profile_link_color": "0084B4", + "description": "Life is a short ride. Negativity can lead to stress. Reflect on the positive to drive away the negative.", + "profile_sidebar_border_color": "C0DEED", + "url": null, + "time_zone": "Quito", + "default_profile_image": false, + "statuses_count": 395, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 18, + "friends_count": 15, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Mon Mar 28 11:29:47 +0000 2011", + "followers_count": 28, + "default_profile": true, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 273380258, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1290603834/image_normal.jpg", + "profile_image_url": "http://a2.twimg.com/profile_images/1290603834/image_normal.jpg" + }, + { + "profile_sidebar_fill_color": "5bda1b", + "protected": false, + "id_str": "36706744", + "notifications": false, + "profile_background_tile": false, + "screen_name": "Herrardo", + "name": "Gerardo Torres", + "listed_count": 15, + "location": "Mexico City", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": false, + "utc_offset": -21600, + "profile_link_color": "1203fc", + "description": "I love playin' guitar! Or bass! Having fun! To party! Read, watch movies, learn something interesting!", + "profile_sidebar_border_color": "000000", + "url": "http://www.facebook.com/Gerrytn", + "time_zone": "Mexico City", + "default_profile_image": false, + "statuses_count": 4483, + "profile_use_background_image": true, + "default_profile": false, + "verified": false, + "favourites_count": 17, + "friends_count": 266, + "profile_background_color": "25a290", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/profile_background_images/13762946/DSC00295.JPG", + "created_at": "Thu Apr 30 16:38:47 +0000 2009", + "followers_count": 156, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/13762946/DSC00295.JPG", + "id": 36706744, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1303959010/yop_normal.jpg", + "profile_image_url": "http://a3.twimg.com/profile_images/1303959010/yop_normal.jpg" + }, + { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "199273449", + "notifications": false, + "profile_background_tile": false, + "screen_name": "sitarabeling", + "name": "sita rabeling", + "listed_count": 0, + "location": "Netherlands", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": false, + "utc_offset": 3600, + "profile_link_color": "0084B4", + "description": "", + "profile_sidebar_border_color": "C0DEED", + "url": null, + "time_zone": "Amsterdam", + "default_profile_image": false, + "statuses_count": 120, + "profile_use_background_image": true, + "default_profile": true, + "verified": false, + "favourites_count": 0, + "friends_count": 47, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Wed Oct 06 13:17:11 +0000 2010", + "followers_count": 11, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 199273449, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1379852945/IMG_0195_normal.jpg", + "profile_image_url": "http://a2.twimg.com/profile_images/1379852945/IMG_0195_normal.jpg" + }, + { + "profile_sidebar_fill_color": "E6F6F9", + "protected": false, + "id_str": "47444972", + "notifications": false, + "profile_background_tile": false, + "screen_name": "MooreofMary", + "name": "Mary Moore", + "listed_count": 8, + "location": "Oklahoma", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": false, + "utc_offset": -21600, + "profile_link_color": "CC3366", + "description": "Morning Radio Show Cohost. Sports Reporter. Emcee. Workaholic. ", + "profile_sidebar_border_color": "DBE9ED", + "url": "http://www.facebook.com/pages/Mary-MooreTULSA/160802983954993", + "time_zone": "Central Time (US & Canada)", + "default_profile_image": false, + "statuses_count": 3331, + "profile_use_background_image": false, + "verified": false, + "favourites_count": 3, + "friends_count": 245, + "profile_background_color": "DBE9ED", + "is_translator": false, + "default_profile": false, + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme17/bg.gif", + "created_at": "Mon Jun 15 21:07:55 +0000 2009", + "followers_count": 455, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme17/bg.gif", + "id": 47444972, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1505124747/image_normal.jpg", + "profile_image_url": "http://a2.twimg.com/profile_images/1505124747/image_normal.jpg" + }, + { + "profile_sidebar_fill_color": "252429", + "protected": false, + "id_str": "28393220", + "notifications": false, + "profile_background_tile": false, + "screen_name": "picyadri", + "name": "Adrienn Szikszay", + "listed_count": 0, + "location": "Hungary", + "show_all_inline_media": true, + "contributors_enabled": false, + "following": false, + "geo_enabled": false, + "utc_offset": 3600, + "profile_link_color": "edc345", + "description": "Optimistic. Loves a bunch of stuff. Always smiling.", + "profile_sidebar_border_color": "f1bd0e", + "url": null, + "time_zone": "Budapest", + "default_profile_image": false, + "statuses_count": 65, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 0, + "friends_count": 114, + "profile_background_color": "1A1B1F", + "is_translator": false, + "default_profile": false, + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme9/bg.gif", + "created_at": "Thu Apr 02 18:43:08 +0000 2009", + "followers_count": 53, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme9/bg.gif", + "id": 28393220, + "profile_text_color": "aba6a6", + "profile_image_url_https": "https://si0.twimg.com/profile_images/121702135/adrisapiban_normal.jpg", + "profile_image_url": "http://a1.twimg.com/profile_images/121702135/adrisapiban_normal.jpg" + }, + { + "profile_sidebar_fill_color": "bd77bd", + "protected": false, + "id_str": "97671153", + "notifications": false, + "profile_background_tile": true, + "screen_name": "purple9dragon", + "name": "purple dragon", + "listed_count": 2, + "location": "天空の城ラプタ", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": false, + "utc_offset": 32400, + "profile_link_color": "0055ff", + "description": "民間人 フォローミー、フォローユー 趣味は盆栽", + "profile_sidebar_border_color": "6cd966", + "url": null, + "time_zone": "Osaka", + "default_profile_image": false, + "statuses_count": 1556, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 6, + "friends_count": 112, + "profile_background_color": "000000", + "is_translator": false, + "default_profile": false, + "profile_background_image_url": "http://a1.twimg.com/profile_background_images/63728857/the-eye-dl.jpg", + "created_at": "Fri Dec 18 13:55:27 +0000 2009", + "followers_count": 62, + "follow_request_sent": false, + "lang": "ja", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/63728857/the-eye-dl.jpg", + "id": 97671153, + "profile_text_color": "000000", + "profile_image_url_https": "https://si0.twimg.com/profile_images/938357687/exit_normal.jpg", + "profile_image_url": "http://a1.twimg.com/profile_images/938357687/exit_normal.jpg" + }, + { + "profile_sidebar_fill_color": "4f468c", + "protected": false, + "id_str": "23290095", + "notifications": false, + "profile_background_tile": true, + "screen_name": "RachelCappucci", + "name": "Rachel Cappucci", + "listed_count": 2, + "location": "New York City", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": false, + "utc_offset": -18000, + "profile_link_color": "d911cf", + "description": " The person who risks nothing, does nothing, has nothing, is nothing. ", + "profile_sidebar_border_color": "d678d6", + "url": null, + "time_zone": "Eastern Time (US & Canada)", + "default_profile_image": false, + "statuses_count": 1810, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 5, + "friends_count": 335, + "profile_background_color": "3e416b", + "is_translator": false, + "default_profile": false, + "profile_background_image_url": "http://a1.twimg.com/profile_background_images/87659392/6040_143426809917_610109917_3324938_5333854_n.jpg", + "created_at": "Sun Mar 08 08:30:14 +0000 2009", + "followers_count": 121, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/87659392/6040_143426809917_610109917_3324938_5333854_n.jpg", + "id": 23290095, + "profile_text_color": "ed9aed", + "profile_image_url_https": "https://si0.twimg.com/profile_images/391846681/twitter_this_normal.jpg", + "profile_image_url": "http://a1.twimg.com/profile_images/391846681/twitter_this_normal.jpg" + }, + { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "209990451", + "notifications": false, + "profile_background_tile": false, + "screen_name": "penfoldholes", + "name": "Richard Barker", + "listed_count": 0, + "location": "Shefford", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": true, + "utc_offset": null, + "profile_link_color": "0084B4", + "description": "kidney cancer survivor - former footballer and cricketer - wine , dog and music lover.", + "profile_sidebar_border_color": "C0DEED", + "url": null, + "time_zone": null, + "default_profile_image": false, + "statuses_count": 1608, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 7, + "friends_count": 66, + "profile_background_color": "C0DEED", + "is_translator": false, + "default_profile": true, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Sat Oct 30 09:40:15 +0000 2010", + "followers_count": 52, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 209990451, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1157214189/Richard_normal.jpg", + "profile_image_url": "http://a0.twimg.com/profile_images/1157214189/Richard_normal.jpg" + }, + { + "profile_sidebar_fill_color": "F6F6F6", + "protected": false, + "id_str": "90737592", + "notifications": false, + "profile_background_tile": false, + "screen_name": "coti_rodriguez", + "name": "Constanza Rodriguez", + "listed_count": 0, + "location": "Monte Hermoso", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": true, + "utc_offset": -10800, + "profile_link_color": "038543", + "description": "", + "profile_sidebar_border_color": "EEEEEE", + "url": null, + "time_zone": "Buenos Aires", + "default_profile_image": false, + "default_profile": false, + "statuses_count": 276, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 2, + "friends_count": 153, + "profile_background_color": "ACDED6", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme18/bg.gif", + "created_at": "Tue Nov 17 22:36:14 +0000 2009", + "followers_count": 48, + "follow_request_sent": false, + "lang": "es", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme18/bg.gif", + "id": 90737592, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1098075067/DSCN1664_normal.JPG", + "profile_image_url": "http://a0.twimg.com/profile_images/1098075067/DSCN1664_normal.JPG" + }, + { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "111380497", + "notifications": false, + "profile_background_tile": false, + "screen_name": "PROFAMS", + "name": "ANTONIO MARCOS", + "listed_count": 0, + "location": "São Paulo", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": true, + "utc_offset": -10800, + "profile_link_color": "0084B4", + "description": "Gerente Nacional de Marketing e Professor nas áreas de adminisração e marketing.", + "profile_sidebar_border_color": "C0DEED", + "url": null, + "time_zone": "Brasilia", + "default_profile_image": false, + "default_profile": true, + "statuses_count": 124, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 5, + "friends_count": 44, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Thu Feb 04 18:34:41 +0000 2010", + "followers_count": 45, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 111380497, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1309327114/AMS_3_normal.JPG", + "profile_image_url": "http://a2.twimg.com/profile_images/1309327114/AMS_3_normal.JPG" + }, + { + "profile_sidebar_fill_color": "333333", + "protected": false, + "id_str": "19216042", + "notifications": false, + "profile_background_tile": true, + "screen_name": "anaverena", + "name": "Ana Verena Menezes", + "listed_count": 14, + "location": "Salvador,Bahia,Brasil", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": true, + "utc_offset": -10800, + "profile_link_color": "b36868", + "description": "all my colours turn to clouds.", + "profile_sidebar_border_color": "b6b8b6", + "url": "http://www.anaverena.tumblr.com", + "time_zone": "Brasilia", + "default_profile_image": false, + "default_profile": false, + "statuses_count": 8298, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 35, + "friends_count": 149, + "profile_background_color": "ffffff", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/profile_background_images/290833153/ikea_lena_2.jpg", + "created_at": "Tue Jan 20 02:45:47 +0000 2009", + "followers_count": 271, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/290833153/ikea_lena_2.jpg", + "id": 19216042, + "profile_text_color": "b8b2b2", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1445007999/IMG_7694_copie2_normal.jpg", + "profile_image_url": "http://a2.twimg.com/profile_images/1445007999/IMG_7694_copie2_normal.jpg" + }, + { + "profile_sidebar_fill_color": "fa89c9", + "protected": false, + "id_str": "192627716", + "notifications": false, + "profile_background_tile": true, + "screen_name": "soouzalaura", + "name": "Laura Souza", + "listed_count": 2, + "location": "Brasil", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": true, + "utc_offset": -10800, + "profile_link_color": "942394", + "description": "Welcome to real world. It's suck, you're gonna love it! (Friends)", + "profile_sidebar_border_color": "ed26bf", + "url": "http://laurapriscila.tumblr.com/", + "time_zone": "Brasilia", + "default_profile_image": false, + "statuses_count": 2218, + "profile_use_background_image": true, + "default_profile": false, + "verified": false, + "favourites_count": 2, + "friends_count": 119, + "profile_background_color": "ffffff", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/profile_background_images/237284023/bg.gif", + "created_at": "Sun Sep 19 18:12:21 +0000 2010", + "followers_count": 61, + "follow_request_sent": false, + "lang": "pt", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/237284023/bg.gif", + "id": 192627716, + "profile_text_color": "524646", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1422000163/SAM_0539_normal.JPG", + "profile_image_url": "http://a0.twimg.com/profile_images/1422000163/SAM_0539_normal.JPG" + }, + { + "profile_sidebar_fill_color": "efefef", + "protected": false, + "id_str": "58000577", + "notifications": false, + "profile_background_tile": true, + "screen_name": "colchica", + "name": "C Orne", + "listed_count": 0, + "location": "Lyon - France", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": true, + "utc_offset": 3600, + "profile_link_color": "2d66ad", + "description": "addicted to caffeine and a little bit greener... ", + "default_profile": false, + "profile_sidebar_border_color": "eeeeee", + "url": null, + "time_zone": "Paris", + "default_profile_image": false, + "statuses_count": 232, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 1, + "friends_count": 45, + "profile_background_color": "ebeef0", + "is_translator": false, + "profile_background_image_url": "http://a3.twimg.com/profile_background_images/154201528/20100912103242_2010-09-11_eboulis_05.jpg", + "created_at": "Sat Jul 18 18:44:29 +0000 2009", + "followers_count": 14, + "follow_request_sent": false, + "lang": "fr", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/154201528/20100912103242_2010-09-11_eboulis_05.jpg", + "id": 58000577, + "profile_text_color": "994399", + "profile_image_url_https": "https://si0.twimg.com/profile_images/336483751/samantha_normal.jpg", + "profile_image_url": "http://a0.twimg.com/profile_images/336483751/samantha_normal.jpg" + }, + { + "profile_sidebar_fill_color": "c9c9c9", + "protected": false, + "id_str": "67218346", + "notifications": null, + "profile_background_tile": true, + "screen_name": "GrandyManFTW14", + "name": "Valerie", + "listed_count": 39, + "location": "Yankee Stadium", + "show_all_inline_media": true, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": -18000, + "profile_link_color": "2138B4", + "description": "im a nerd thats infatuated with sports... mostly #yankees baseball. I should reside in a bubble. I'm 28 and speak my mind.", + "profile_sidebar_border_color": "bfbfbf", + "url": "http://www.facebook.com/yankeesangel21", + "default_profile": false, + "time_zone": "Eastern Time (US & Canada)", + "default_profile_image": false, + "statuses_count": 8744, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 11, + "friends_count": 653, + "profile_background_color": "07090b", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/profile_background_images/202039528/x1f1841c56244072a549acc188477e1e.jpg", + "created_at": "Thu Aug 20 03:43:17 +0000 2009", + "followers_count": 500, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/202039528/x1f1841c56244072a549acc188477e1e.jpg", + "id": 67218346, + "profile_text_color": "183050", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1520181363/image_normal.jpg", + "profile_image_url": "http://a2.twimg.com/profile_images/1520181363/image_normal.jpg" + }, + { + "profile_sidebar_fill_color": "95E8EC", + "protected": false, + "id_str": "185014625", + "default_profile": false, + "notifications": false, + "profile_background_tile": false, + "screen_name": "chiquip35", + "name": "Paula Klimowitz", + "listed_count": 1, + "location": "Miami", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": false, + "utc_offset": null, + "profile_link_color": "0099B9", + "description": "I love the positive energy of the people of the world!!", + "profile_sidebar_border_color": "5ED4DC", + "url": null, + "time_zone": null, + "default_profile_image": false, + "statuses_count": 48, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 0, + "friends_count": 102, + "profile_background_color": "0099B9", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme4/bg.gif", + "created_at": "Tue Aug 31 01:15:36 +0000 2010", + "followers_count": 9, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme4/bg.gif", + "id": 185014625, + "profile_text_color": "3C3940", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1521529755/174758_normal.jpg", + "profile_image_url": "http://a3.twimg.com/profile_images/1521529755/174758_normal.jpg" + } + ], + "resultType": "ReTweet" + } + ] + } +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/twitter.reply b/Collections/Timeline/fixtures/twitter.reply new file mode 100644 index 000000000..f8044b273 --- /dev/null +++ b/Collections/Timeline/fixtures/twitter.reply @@ -0,0 +1,142 @@ +{ + "action": "new", + "idr": "tweet://twitter/timeline?id=twitter#132147322749599744", + "data": { + "place": { + "country_code": "US", + "place_type": "city", + "name": "Cascade", + "bounding_box": { + "type": "Polygon", + "coordinates": [ + [ + [ + -91.02358, + 42.289324 + ], + [ + -90.991574, + 42.289324 + ], + [ + -90.991574, + 42.308435 + ], + [ + -91.02358, + 42.308435 + ] + ] + ] + }, + "attributes": {}, + "full_name": "Cascade, IA", + "country": "United States", + "url": "http://api.twitter.com/1/geo/id/85905f44ae0575e5.json", + "id": "85905f44ae0575e5" + }, + "retweet_count": 0, + "in_reply_to_screen_name": "FrancescoC", + "created_at": "Thu Nov 03 17:29:11 +0000 2011", + "retweeted": false, + "in_reply_to_status_id_str": "132133925953875969", + "in_reply_to_user_id_str": "15592821", + "user": { + "time_zone": "Central Time (US & Canada)", + "profile_link_color": "0000ff", + "protected": false, + "default_profile": false, + "created_at": "Wed Mar 21 02:44:51 +0000 2007", + "profile_image_url_https": "https://si0.twimg.com/profile_images/565991047/jer_normal.jpg", + "name": "Jeremie Miller", + "geo_enabled": true, + "favourites_count": 9, + "profile_background_color": "ffffff", + "default_profile_image": false, + "notifications": null, + "profile_background_tile": false, + "utc_offset": -21600, + "description": "Currently building the Locker Project, TeleHash, and Singly with a focus on personal data + distributed protocols. Helped found Jabber/XMPP, open platforms FTW!", + "statuses_count": 891, + "following": null, + "verified": false, + "profile_sidebar_fill_color": "ffffff", + "followers_count": 1375, + "profile_image_url": "http://a0.twimg.com/profile_images/565991047/jer_normal.jpg", + "follow_request_sent": null, + "profile_sidebar_border_color": "000000", + "location": "Cascade, IA", + "is_translator": false, + "contributors_enabled": false, + "screen_name": "jeremie", + "profile_use_background_image": false, + "url": "http://jeremie.com/-", + "id_str": "1704421", + "show_all_inline_media": true, + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "profile_text_color": "000000", + "id": 1704421, + "listed_count": 141, + "lang": "en", + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "friends_count": 161 + }, + "contributors": null, + "in_reply_to_status_id": 132133925953875970, + "source": "Twitter for iPhone", + "id_str": "132147322749599744", + "geo": { + "type": "Point", + "coordinates": [ + 42.29882394, + -91.01307985 + ] + }, + "favorited": false, + "id": 132147322749599740, + "entities": { + "urls": [], + "user_mentions": [ + { + "name": "Francesco Cesarini", + "indices": [ + 6, + 17 + ], + "screen_name": "FrancescoC", + "id_str": "15592821", + "id": 15592821 + }, + { + "name": "DizzyD", + "indices": [ + 34, + 41 + ], + "screen_name": "dizzyd", + "id_str": "7206052", + "id": 7206052 + } + ], + "hashtags": [ + { + "indices": [ + 123, + 131 + ], + "text": "euc2011" + } + ] + }, + "coordinates": { + "type": "Point", + "coordinates": [ + -91.01307985, + 42.29882394 + ] + }, + "in_reply_to_user_id": 15592821, + "truncated": false, + "text": "Nice! @FrancescoC: Congratulating @dizzyd for winning the Erlang User of the year award for giving rebar to the community! #euc2011" + } +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/twitter.reply.orig b/Collections/Timeline/fixtures/twitter.reply.orig new file mode 100644 index 000000000..3e2c1f3d2 --- /dev/null +++ b/Collections/Timeline/fixtures/twitter.reply.orig @@ -0,0 +1,174 @@ +{ + "action": "new", + "idr": "tweet://twitter/timeline?id=twitter#132138590636478464", + "data": { + "place": null, + "retweet_count": 41, + "in_reply_to_screen_name": null, + "created_at": "Thu Nov 03 16:54:29 +0000 2011", + "retweeted": false, + "in_reply_to_status_id_str": null, + "in_reply_to_user_id_str": null, + "user": { + "time_zone": null, + "profile_link_color": "2FC2EF", + "protected": false, + "created_at": "Fri Mar 13 13:15:23 +0000 2009", + "profile_image_url_https": "https://si0.twimg.com/profile_images/95456179/cortex_normal.png", + "name": "Jon Meredith", + "geo_enabled": false, + "favourites_count": 1, + "profile_background_color": "1A1B1F", + "default_profile_image": false, + "notifications": false, + "profile_background_tile": false, + "utc_offset": null, + "description": "", + "statuses_count": 85, + "following": true, + "verified": false, + "profile_sidebar_fill_color": "252429", + "followers_count": 127, + "profile_image_url": "http://a0.twimg.com/profile_images/95456179/cortex_normal.png", + "follow_request_sent": false, + "profile_sidebar_border_color": "181A1E", + "location": "Denver, CO, USA", + "is_translator": false, + "contributors_enabled": false, + "screen_name": "jon_meredith", + "default_profile": false, + "profile_use_background_image": true, + "url": null, + "id_str": "24168124", + "show_all_inline_media": false, + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme9/bg.gif", + "profile_text_color": "666666", + "id": 24168124, + "listed_count": 12, + "lang": "en", + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme9/bg.gif", + "friends_count": 69 + }, + "contributors": null, + "retweeted_status": { + "place": null, + "retweet_count": 41, + "in_reply_to_screen_name": null, + "created_at": "Thu Nov 03 16:35:57 +0000 2011", + "retweeted": false, + "in_reply_to_status_id_str": null, + "in_reply_to_user_id_str": null, + "user": { + "time_zone": "London", + "profile_link_color": "D02B55", + "protected": false, + "created_at": "Fri Jul 25 02:14:58 +0000 2008", + "profile_image_url_https": "https://si0.twimg.com/profile_images/337579346/francescoJulianCash_normal.jpg", + "name": "Francesco Cesarini", + "geo_enabled": false, + "favourites_count": 16, + "profile_background_color": "352726", + "default_profile_image": false, + "notifications": false, + "profile_background_tile": false, + "utc_offset": 0, + "description": "Founded Erlang Solutions, co-authored Erlang Programming (O'Reilly), part time uncle & godfather and full time pointy haired boss. ", + "statuses_count": 3127, + "following": false, + "verified": false, + "profile_sidebar_fill_color": "99CC33", + "followers_count": 1480, + "profile_image_url": "http://a1.twimg.com/profile_images/337579346/francescoJulianCash_normal.jpg", + "follow_request_sent": false, + "profile_sidebar_border_color": "829D5E", + "location": "London, United Kingdom", + "is_translator": false, + "contributors_enabled": false, + "screen_name": "FrancescoC", + "default_profile": false, + "profile_use_background_image": true, + "url": "http://www.erlang-solutions.com", + "id_str": "15592821", + "show_all_inline_media": false, + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme5/bg.gif", + "profile_text_color": "3E4415", + "id": 15592821, + "listed_count": 122, + "lang": "en", + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme5/bg.gif", + "friends_count": 365 + }, + "contributors": null, + "in_reply_to_status_id": null, + "source": "Twitter for Android", + "id_str": "132133925953875969", + "geo": null, + "favorited": false, + "id": 132133925953875970, + "entities": { + "urls": [], + "user_mentions": [ + { + "name": "DizzyD", + "indices": [ + 15, + 22 + ], + "screen_name": "dizzyd", + "id_str": "7206052", + "id": 7206052 + } + ], + "hashtags": [ + { + "indices": [ + 121, + 129 + ], + "text": "euc2011" + } + ] + }, + "coordinates": null, + "in_reply_to_user_id": null, + "truncated": false, + "text": "Congratulating @dizzyd for winning the Erlang User of the year award for giving rebar to the community! Well done Dizzy. #euc2011" + }, + "in_reply_to_status_id": null, + "source": "web", + "id_str": "132138590636478464", + "geo": null, + "favorited": false, + "id": 132138590636478460, + "entities": { + "urls": [], + "user_mentions": [ + { + "name": "Francesco Cesarini", + "indices": [ + 3, + 14 + ], + "screen_name": "FrancescoC", + "id_str": "15592821", + "id": 15592821 + }, + { + "name": "DizzyD", + "indices": [ + 31, + 38 + ], + "screen_name": "dizzyd", + "id_str": "7206052", + "id": 7206052 + } + ], + "hashtags": [] + }, + "coordinates": null, + "in_reply_to_user_id": null, + "truncated": true, + "text": "RT @FrancescoC: Congratulating @dizzyd for winning the Erlang User of the year award for giving rebar to the community! Well done Dizzy. ..." + } +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/twitter.rt b/Collections/Timeline/fixtures/twitter.rt new file mode 100644 index 000000000..8ea409f8d --- /dev/null +++ b/Collections/Timeline/fixtures/twitter.rt @@ -0,0 +1,145 @@ +{ + "action": "new", + "idr": "tweet://twitter/timeline?id=twitter#132508288435752960", + "data": { + "place": null, + "retweet_count": 6, + "in_reply_to_screen_name": null, + "created_at": "Fri Nov 04 17:23:32 +0000 2011", + "retweeted": false, + "in_reply_to_status_id_str": null, + "in_reply_to_user_id_str": null, + "user": { + "time_zone": "Pacific Time (US & Canada)", + "profile_link_color": "0000ff", + "protected": false, + "created_at": "Tue Mar 27 01:14:05 +0000 2007", + "profile_image_url_https": "https://si0.twimg.com/profile_images/941827802/IMG_3811_v4_normal.jpg", + "name": "Tim O'Reilly", + "geo_enabled": true, + "favourites_count": 52, + "profile_background_color": "9ae4e8", + "default_profile_image": false, + "notifications": false, + "profile_background_tile": false, + "utc_offset": -28800, + "description": "Founder and CEO, O'Reilly Media. Watching the alpha geeks, sharing their stories, helping the future unfold.", + "statuses_count": 16827, + "following": true, + "verified": true, + "profile_sidebar_fill_color": "e0ff92", + "followers_count": 1508525, + "profile_image_url": "http://a1.twimg.com/profile_images/941827802/IMG_3811_v4_normal.jpg", + "follow_request_sent": false, + "profile_sidebar_border_color": "87bc44", + "location": "Sebastopol, CA", + "is_translator": false, + "default_profile": false, + "contributors_enabled": false, + "screen_name": "timoreilly", + "profile_use_background_image": true, + "url": "http://radar.oreilly.com", + "id_str": "2384071", + "show_all_inline_media": false, + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/3587880/notes.gif", + "profile_text_color": "000000", + "id": 2384071, + "listed_count": 20459, + "lang": "en", + "profile_background_image_url": "http://a1.twimg.com/profile_background_images/3587880/notes.gif", + "friends_count": 784 + }, + "contributors": null, + "retweeted_status": { + "place": null, + "retweet_count": 6, + "in_reply_to_screen_name": null, + "created_at": "Fri Nov 04 15:36:13 +0000 2011", + "retweeted": false, + "in_reply_to_status_id_str": null, + "in_reply_to_user_id_str": null, + "user": { + "time_zone": "Pacific Time (US & Canada)", + "profile_link_color": "0084B4", + "protected": false, + "created_at": "Tue Jan 27 02:59:48 +0000 2009", + "profile_image_url_https": "https://si0.twimg.com/profile_images/80487785/Michael_Chui_face_square_normal.jpg", + "name": "Michael Chui", + "geo_enabled": false, + "favourites_count": 0, + "profile_background_color": "C0DEED", + "default_profile_image": false, + "notifications": false, + "profile_background_tile": false, + "utc_offset": -28800, + "description": "Senior Fellow at McKinsey Global Institute (@McKinsey_MGI) leading research on the impact of technology trends. Opinions expressed are my own.", + "statuses_count": 1787, + "following": false, + "verified": false, + "profile_sidebar_fill_color": "DDEEF6", + "followers_count": 1326, + "profile_image_url": "http://a0.twimg.com/profile_images/80487785/Michael_Chui_face_square_normal.jpg", + "follow_request_sent": false, + "profile_sidebar_border_color": "C0DEED", + "location": "", + "is_translator": false, + "default_profile": true, + "contributors_enabled": false, + "screen_name": "mchui", + "profile_use_background_image": true, + "url": null, + "id_str": "19573968", + "show_all_inline_media": false, + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "profile_text_color": "333333", + "id": 19573968, + "listed_count": 85, + "lang": "en", + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "friends_count": 157 + }, + "contributors": null, + "in_reply_to_status_id": null, + "source": "Twitter for BlackBerry®", + "id_str": "132481280091754496", + "geo": null, + "favorited": false, + "id": 132481280091754500, + "entities": { + "urls": [], + "user_mentions": [], + "hashtags": [] + }, + "coordinates": null, + "in_reply_to_user_id": null, + "truncated": false, + "text": "Quoth Niall Ferguson: We no longer have the rule of law; we have the rule of lawyers and that's different." + }, + "in_reply_to_status_id": null, + "source": "Twitter for Android", + "id_str": "132508288435752960", + "geo": null, + "favorited": false, + "id": 132508288435752960, + "entities": { + "urls": [], + "user_mentions": [ + { + "name": "Michael Chui", + "indices": [ + 3, + 9 + ], + "screen_name": "mchui", + "id_str": "19573968", + "id": 19573968 + } + ], + "hashtags": [] + }, + "coordinates": null, + "in_reply_to_user_id": null, + "truncated": false, + "text": "RT @mchui: Quoth Niall Ferguson: We no longer have the rule of law; we have the rule of lawyers and that's different." + } +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/twitter.timeline b/Collections/Timeline/fixtures/twitter.timeline new file mode 100644 index 000000000..f840662df --- /dev/null +++ b/Collections/Timeline/fixtures/twitter.timeline @@ -0,0 +1,92 @@ +{ + "action": "new", + "idr": "tweet://twitter/timeline?id=twitter#113829905090879488", + "data": { + "_id": "4e702f08bcb6f2f6dd06c849", + "in_reply_to_user_id_str": null, + "retweet_count": 0, + "in_reply_to_status_id": null, + "id_str": "113829905090879488", + "contributors": null, + "truncated": false, + "geo": null, + "coordinates": null, + "favorited": false, + "user": { + "default_profile": false, + "contributors_enabled": false, + "profile_use_background_image": true, + "protected": false, + "id_str": "633", + "time_zone": "Eastern Time (US & Canada)", + "profile_background_color": "1A1B1F", + "name": "danah boyd", + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme9/bg.gif", + "profile_image_url_https": "https://si0.twimg.com/profile_images/890666764/MeAtTalkWithHat-Sq_normal.jpg", + "default_profile_image": false, + "following": true, + "utc_offset": -18000, + "profile_image_url": "http://a0.twimg.com/profile_images/890666764/MeAtTalkWithHat-Sq_normal.jpg", + "description": "social media scholar, youth researcher & advocate | Microsoft Research, Harvard Berkman Center | zephoria@zephoria.org", + "show_all_inline_media": false, + "geo_enabled": false, + "friends_count": 734, + "profile_text_color": "666666", + "location": "Boston, MA", + "is_translator": false, + "profile_sidebar_fill_color": "252429", + "follow_request_sent": false, + "profile_background_tile": false, + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme9/bg.gif", + "url": "http://www.zephoria.org/thoughts/", + "statuses_count": 2471, + "followers_count": 50313, + "screen_name": "zephoria", + "notifications": false, + "profile_link_color": "2FC2EF", + "lang": "en", + "verified": false, + "favourites_count": 44, + "profile_sidebar_border_color": "181A1E", + "id": 633, + "listed_count": 5224, + "created_at": "Thu Jul 13 21:27:23 +0000 2006" + }, + "in_reply_to_user_id": null, + "in_reply_to_screen_name": null, + "possibly_sensitive": false, + "retweeted": false, + "source": "web", + "in_reply_to_status_id_str": null, + "entities": { + "user_mentions": [ + { + "name": "Wil Wheaton", + "id_str": "1183041", + "indices": [ + 108, + 113 + ], + "screen_name": "wilw", + "id": 1183041 + } + ], + "hashtags": [], + "urls": [ + { + "indices": [ + 116, + 135 + ], + "display_url": "amzn.to/qqvlo4", + "url": "http://t.co/zAN4G8S", + "expanded_url": "http://amzn.to/qqvlo4" + } + ] + }, + "id": 113829905090879490, + "place": null, + "text": "Just finished \"Ready Player One.\" YA dystopia mixed w/ 1980s geek nostalgia. Soooo much fun. (Audio read by @wilw ) http://t.co/zAN4G8S", + "created_at": "Wed Sep 14 04:22:18 +0000 2011" + } +} \ No newline at end of file diff --git a/Collections/Timeline/package.json b/Collections/Timeline/package.json new file mode 100644 index 000000000..184996ebe --- /dev/null +++ b/Collections/Timeline/package.json @@ -0,0 +1,33 @@ +{ + "author": "Singly ", + "name": "timeline", + "description": "Timeline Collection", + "version": "0.0.1", + "repository": { + "title": "Timeline", + "run": "node timeline.js", + "author": "nerds", + "update": "auto", + "github": "https://github.com/LockerProject/Locker", + "type": "collection", + "handle": "timeline", + "provides": [ + "item" + ], + "mongoCollections": [ + "item", "response" + ], + "events":[["checkin/foursquare","/events"] + ,["tweet/twitter","/events"] + ,["photo/instagram","/events"] + ,["related/twitter","/events"] + ,["post/facebook","/events"] + ,["link","/events"] + ] + }, + "engines": { + "node": ">=0.4.9" + }, + "dependencies": {}, + "devDependencies": {} +} diff --git a/Collections/Timeline/sync.js b/Collections/Timeline/sync.js new file mode 100644 index 000000000..f98d08c57 --- /dev/null +++ b/Collections/Timeline/sync.js @@ -0,0 +1,129 @@ +var async = require('async'); +var logger; +var lutil = require('lutil'); +var url = require('url'); +var fs = require('fs'); + +var dataStore, dataIn, locker, state; + +// internally we need these for happy fun stuff +exports.init = function(l, dStore, dIn, callback){ + dataStore = dStore; + dataIn = dIn; + locker = l; + logger = l.logger; + callback(); + loadState(function(){ + if(state.ready == 1) return; // nothing to do + locker.listen("work/me", "/work"); + exports.sync(); + }); +} + +var stopped; +exports.work = function(type) +{ + stopped = false; + if(type == "start") { + state.sleep = 500; + exports.sync(); + } + if(type == "stop") stopped = true; + if(type == "warn") state.sleep = 10000; +} + +function loadState(callback) +{ + try { state = JSON.parse(fs.readFileSync('state.json')); } catch(E) { logger.error("state.json failed ",E); } + if(state) return callback(); + // starting from scratch, make sure clear, set initial list + logger.error("syncing from a new state"); + dataStore.clear(function(){ + state = {sleep:500, types:[ + "facebook/getCurrent/home", + "twitter/getCurrent/tweets", "twitter/getCurrent/timeline", "twitter/getCurrent/mentions", "twitter/getCurrent/related", + "foursquare/getCurrent/recents", "foursquare/getCurrent/checkin", + "instagram/getCurrent/photo", "instagram/getCurrent/feed", + "links/encounters" + ]}; + saveState(); + callback(); + }); +} + +function saveState() +{ + lutil.atomicWriteFileSync("state.json", JSON.stringify(state)); +} + +// see if there's any syncing work to do in the background, only running one at a time +var syncTimeout; +var running; +exports.sync = function() { + // if any timer for future sync, zap it + if(syncTimeout) clearTimeout(syncTimeout); + syncTimeout = false; + // don't run if told not to + if(stopped) return; + // don't run if we're already running + if(running) return; + running = true; + // load fresh just to be safe + loadState(function(){ + // if nothing to sync, go away + if(!state) return logger.error("invalid state!"); + if(state.ready == 1) return; + // see if we're all done! + if(state.types.length == 0 && !state.current) + { + running = false; + state.ready = 1; + return saveState(); + } + if(!state.current) + { + state.current = {}; + state.current.type = state.types.shift(); + state.current.offset = 0; + } + var cnt = 0; + var lurl = locker.lockerBase + '/Me/' + state.current.type + "?stream=true&limit=250&offset="+state.current.offset; + logger.error("syncing "+lurl); + lutil.streamFromUrl(lurl, function(a, cb){ + cnt++; + (state.current.type.indexOf("link/") == 0) ? dataIn.processLink({data:a}, cb) : dataIn.masterMaster(getIdr(state.current.type, a), a, cb); + }, function(err){ + running = false; + if(err) return logger.error("sync failed, stopping: ",err); + // no data n no error, we hit the end, move on! + state.current.offset += 250; + if(cnt == 0) delete state.current; + saveState(); + // come back again darling + syncTimeout = setTimeout(exports.sync, state.sleep); + }) + }); +} + + +// generate unique id for any item based on it's event +//> u.parse("type://network/context?id=account#46234623456",true); +//{ protocol: 'type:', +// slashes: true, +// host: 'network', +// hostname: 'network', +// href: 'type://network/context?id=account#46234623456', +// hash: '#46234623456', +// search: '?id=account', +// query: { id: 'account' }, +// pathname: '/context' } +function getIdr(type, data) +{ + var r = {slashes:true}; + r.host = type.substr(0, type.indexOf('/')); + r.pathname = type.substr(type.lastIndexOf('/')+1); + r.query = {id: r.host}; // best proxy of account id right now + dataIn.idrHost(r, data); + return url.parse(url.format(r),true); // make sure it's consistent +} + diff --git a/Collections/Timeline/test.js b/Collections/Timeline/test.js new file mode 100644 index 000000000..6826ce751 --- /dev/null +++ b/Collections/Timeline/test.js @@ -0,0 +1,176 @@ +// BWARE, this is a scratch area used while dev'ing and is pretty ugly as it does a bunch of support work to simulate a working env +// TODO convert to a real unit testing framework +try{ + var lconfig = require('lconfig'); +}catch(E){ + console.error("export NODE_PATH=../../Common/node/"); + process.exit(); +} +var fs = require('fs'); +var url = require('url'); +var request = require('request'); +lconfig.load('../../Config/config.json'); +var lmongoclient = require('../../Common/node/lmongoclient')(lconfig.mongo.host, lconfig.mongo.port, "timelinetest", ["item", "response"]); +var logger = require('../../Common/node/logger'); +var async = require("async"); +var url = require("url"); + +var dataIn = require('./dataIn'); // for processing incoming twitter/facebook/etc data types +var dataStore = require("./dataStore"); // storage/retreival of raw items and responses + +boot(function(){dataStore.clear(false,testFB)}); + +function testFB() +{ + dataIn.processEvent(fixture("facebook.update"), function(err){ + if(err) console.error(err); + count(function(i1, r1){ + dataIn.processEvent(fixture("facebook.update"), function(err){ + if(err) console.error(err); + count(function(i2, r2){ + if(i2 != i1) return console.error("didn't dedup facebook.update"); + if(r2 != r1) return console.error("didn't dedup facebook.update responses"); + console.error("facebook.update OK"); + testFBTW(); + }); + }) + }); + }) +} + +function testFBTW() +{ + dataIn.processEvent(fixture("facebook.tweet"), function(err){ + if(err) console.error(err); + count(function(i1, r1){ + dataIn.processEvent(fixture("tweet.facebook"), function(err){ + if(err) console.error(err); + count(function(i2, r2){ + if(i2 != i1) return console.error("didn't dedup facebook.tweet"); + if(r2 != r1) return console.error("didn't dedup facebook.tweet responses"); + console.error("facebook<->twitter OK"); + testRT(); + }); + }) + }); + }) +} + +function testRT() +{ + count(function(i1, r1){ + dataIn.processEvent(fixture("twitter.rt"), function(err){ + if(err) console.error(err); + count(function(i2, r2){ + if(i2 - i1 != 1) return console.error("didn't process twitter.rt"); + if(r2 - r1 != 1) return console.error("didn't process twitter.rt response"); + console.error("ReTweet OK"); + testTWReply() + }); + }) + }); +} + +function testTWReply() +{ + count(function(i0, r0){ + dataIn.processEvent(fixture("twitter.reply.orig"), function(err){ + if(err) console.error(err); + count(function(i1, r1){ + dataIn.processEvent(fixture("twitter.reply"), function(err){ + if(err) console.error(err); + count(function(i2, r2){ + if(i2 != i1 && i2 - i0 != 1) return console.error("didn't process twitter.reply properly"); + if(r2 - r1 != 1 || r2 - r0 != 2) return console.error("didn't dedup twitter.reply responses"); + console.error("twitter reply OK"); + test4sqLink(); + }); + }) + }); + }) + }); +} + +function test4sqLink() +{ + count(function(i0, r0){ + dataIn.processEvent(fixture("foursquare.recents"), function(err){ + if(err) console.error(err); + count(function(i1, r1){ + dataIn.processEvent(fixture("twitter.timeline"), function(err){ + if(err) console.error(err); + count(function(i2, r2){ + dataIn.processEvent(fixture("link"), function(err){ + if(err) console.error(err); + count(function(i3, r3){ + if(i2 - i0 != 2 || i2 - i3 != 1) return console.error("didn't dedup with link properly"); + if(r3 - r0 != 0) return console.error("found responses and shouldn't have"); + console.error("link based dedup OK"); + testIG(); + }); + }) + }); + }) + }); + }) + }); +} + +function testIG() +{ + count(function(i0, r0){ + dataIn.processEvent(fixture("ig.instagram"), function(err){ + if(err) console.error(err); + count(function(i1, r1){ + dataIn.processEvent(fixture("ig.fb"), function(err){ + if(err) console.error(err); + count(function(i2, r2){ + dataIn.processEvent(fixture("ig.tweet"), function(err){ + if(err) console.error(err); + dataIn.processEvent(fixture("ig.4sq"), function(err){ + if(err) console.error(err); + count(function(i3, r3){ + if(i3 - i0 != 1) return console.error("didn't dedup instagram properly"); + if(r3 - r0 != 0) return console.error("found responses and shouldn't have"); + console.error("instagram dedup OK"); + }); + }) + }) + }); + }) + }); + }) + }); +} + + + +function count(callback) +{ + dataStore.getTotalItems(function(err, items){ + dataStore.getTotalResponses(function(err, resp){ +// console.error("Items: "+items+" Responses: "+resp); + callback(items, resp); + }) + }) +} +function fixture(name) +{ + return JSON.parse(fs.readFileSync(__dirname+"/fixtures/"+name)); +} + +function boot(callback){ + var tdir = __dirname+"/test"; + try{ + fs.mkdirSync(tdir,0755); + }catch(E){} + process.chdir(tdir); + lmongoclient.connect(function(mongo) { + // initialize all our libs + dataStore.init(mongo.collections.item,mongo.collections.response); + dataIn.init({}, dataStore, function(){ + console.error("booted up"); + callback(); + }); + }); +} diff --git a/Collections/Timeline/timeline.js b/Collections/Timeline/timeline.js new file mode 100644 index 000000000..c8e732abf --- /dev/null +++ b/Collections/Timeline/timeline.js @@ -0,0 +1,174 @@ +/* +* +* Copyright (C) 2011, The Locker Project +* All rights reserved. +* +* Please see the LICENSE file for more information. +* +*/ + +// merge newsfeed style data and any gather any responses + +var fs = require('fs'), + url = require('url'), + request = require('request'), + locker = require('locker.js'), + lconfig = require('lconfig'); +var logger; +var async = require("async"); +var url = require("url"); + +var dataIn = require('./dataIn'); // for processing incoming twitter/facebook/etc data types +var dataStore = require("./dataStore"); // storage/retreival of raw items and responses +var sync = require('./sync'); // for manual updating/resyncing of data from synclets + +var lockerInfo; +var express = require('express'), + connect = require('connect'); +var app = express.createServer(connect.bodyParser()); + +app.set('views', __dirname); + +app.get('/', function(req, res) { + var fields = {}; + if (req.query.fields) { + try { fields = JSON.parse(req.query.fields); } catch(E) {} + } + dataStore.getAll(fields, function(err, cursor) { + if(!req.query["all"]) cursor.limit(20); // default 20 unless all is set + if(req.query["limit"]) cursor.limit(parseInt(req.query["limit"])); + if(req.query["offset"]) cursor.skip(parseInt(req.query["offset"])); + var sorter = {"first":-1}; + if(req.query["sort"]) { + sorter = {}; + if(req.query["order"]) { + sorter[req.query["sort"]] = +req.query["order"]; + } else { + sorter[req.query["sort"]]= 1; + } + } + var ndx = {}; + cursor.sort(sorter).toArray(function(err, items) { + if(req.query["all"] || !req.query.full) return res.send(items); // default not include responses, forced if all + items.forEach(function(item){ ndx[item.id] = item; item.responses = []; }); // build index + var arg = {"item":{$in: Object.keys(ndx)}}; + arg.fields = fields; + dataStore.getResponses(arg, function(response) { + ndx[response.item].responses.push(response); + }, function() { + res.send(items); + }); + }); + }); +}); + +app.get('/state', function(req, res) { + dataStore.getTotalItems(function(err, countInfo) { + if(err) return res.send(err, 500); + var updated = new Date().getTime(); + var js; + try { + js = JSON.parse(fs.readFileSync('state.json')); + } catch(E) {} + if(!js) js = {ready:0}; + js.count = countInfo; + res.send(js); + }); +}); + +// expose way to get the list of responses from an item id +app.get('/responses/:id', function(req, res) { + var results = []; + dataStore.getResponses({item:req.param('id')}, function(item){results.push(item);},function(err){ + if(err) return res.send(err,500); + res.send(results); + }); +}); + +app.get('/id/:id', function(req, res) { + dataStore.getItem(req.param('id'), function(err, doc) { return (err != null || !doc) ? res.send(err, 500) : res.send(doc); }); +}); + +// this would only be useful if something changes or removes state.json +app.get('/sync', function(req, res) { + res.send(true); + sync.sync(); // happens async +}); + +// sync.init() optionally listens for work events, funnel back +app.post('/work', function(req, res) { + if (!req.body.idr || !req.body.data){ + logger.error('bad work data: ',JSON.stringify(req.body)); + return res.send('bad data', 500); + } + res.send('ok'); + sync.work(req.body.data.work); +}); + +app.get('/ref', function(req, res) { + var idr = url.parse(req.query.id); + if(!idr || !idr.hash) return res.send("missing or invalid id",500); + var lurl = locker.lockerBase + '/Me/' + idr.host + idr.pathname + '/id/' + idr.hash.substr(1); + request.get({url:lurl, json:true}, function(err, res2, body){ + if(err || !body) return res.send(err, 500); + res.send(body); + }); +}); + +app.post('/events', function(req, res) { + if (!req.body.idr || !req.body.data){ + logger.error('5 HUNDO bad data:',JSON.stringify(req.body)); + res.writeHead(500); + res.end('bad data'); + return; + } + res.send('ok'); + // handle asyncadilly + dataIn.processEvent(req.body, function(err){ + if(err) logger.error(err); + }); +}); + +function genericApi(name,f) +{ + app.get(name,function(req,res){ + var results = []; + f(req.query,function(item){results.push(item);},function(err){ + if(err) return res.send(err,500); + res.send(results); + }); + }); +} + +genericApi('/getItems', dataStore.getItems); +genericApi('/getResponses', dataStore.getResponses); +genericApi('/getSince', dataStore.getSince); + +// Process the startup JSON object +process.stdin.resume(); +process.stdin.on('data', function(data) { + lockerInfo = JSON.parse(data); + locker.initClient(lockerInfo); + locker.lockerBase = lockerInfo.lockerUrl; + if (!lockerInfo || !lockerInfo['workingDirectory']) { + process.stderr.write('Was not passed valid startup information.'+data+'\n'); + process.exit(1); + } + process.chdir(lockerInfo.workingDirectory); + lconfig.load('../../Config/config.json'); + locker.logger = logger = require('logger'); + + locker.connectToMongo(function(mongo) { + // initialize all our libs + dataStore.init(mongo.collections.item,mongo.collections.response, locker, function(err){ + if(err) logger.error("datastore init failed: ",err); + dataIn.init(locker, dataStore, function(){ + sync.init(locker, dataStore, dataIn, function(){ + app.listen(lockerInfo.port, 'localhost', function() { + process.stdout.write(data); + }); + }); + }); + }); + }); +}); diff --git a/Common/node/lconfig.js b/Common/node/lconfig.js index c00ba1801..54cdc990b 100644 --- a/Common/node/lconfig.js +++ b/Common/node/lconfig.js @@ -10,6 +10,7 @@ //just a place for lockerd.js to populate config info var fs = require('fs'); var path = require('path'); +var os = require('os'); exports.load = function(filepath) { var config = {}; @@ -90,6 +91,8 @@ exports.load = function(filepath) { config.dashboard = config.dashboard || {}; config.dashboard.lockerName = config.dashboard.customLockerName || 'locker'; exports.dashboard = config.dashboard; + exports.workWarn = config.workWarn || os.cpus().length; + exports.workStop = config.workStop || os.cpus().length * 3; // load trusted public keys var kdir = path.join(path.dirname(filepath), "keys"); diff --git a/Common/node/ldatastore.js b/Common/node/ldatastore.js index 0c0f3e9c8..5248bec45 100644 --- a/Common/node/ldatastore.js +++ b/Common/node/ldatastore.js @@ -16,6 +16,7 @@ var IJOD = require('ijod').IJOD , colls = {} , mongoIDs = {} ; +var svcMan = require('lservicemanager'); exports.init = function(owner, callback) { if (mongo[owner]) return callback(); @@ -114,14 +115,21 @@ exports.getEachCurrent = function(owner, type, callback, options) { } exports.getCurrent = function(owner, type, id, callback) { - try { if (!(id && (typeof id === 'string' || typeof id === 'number'))) return callback(new Error('bad id:' + id), null); var m = getMongo(owner, type); - var query = {_id: mongo[owner].db.bson_serializer.ObjectID(id)}; - m.findOne(query, callback); - } catch(err) { - return callback(err); - } + var or = []; + try { + or.push({_id: mongo[owner].db.bson_serializer.ObjectID(id)}); + }catch(E){} + var parts = type.split("_"); + var idname = "id"; + // BEWARE THE mongoId s appendage WEIRDNESS! #techdebt + if(parts.length == 2 && svcMan.map(parts[0]) && svcMan.map(parts[0]).mongoId && svcMan.map(parts[0]).mongoId[parts[1]+'s']) idname = svcMan.map(parts[0]).mongoId[parts[1]+'s']; + var id2 = {}; + id2[idname] = id; + or.push(id2); + console.error("querying "+JSON.stringify(or)); + m.findOne({$or: or}, callback); } exports.getCurrentId = function(owner, type, id, callback) { diff --git a/Common/node/levents.js b/Common/node/levents.js index 97000d452..16fa479ad 100644 --- a/Common/node/levents.js +++ b/Common/node/levents.js @@ -89,7 +89,7 @@ exports.fireEvent = function(idr, action, obj) { } exports.displayListeners = function(type) { - return eventListeners[type]; + return eventListeners[type] || []; } function findListenerPosition(type, id, cb) { diff --git a/Common/node/lservicemanager.js b/Common/node/lservicemanager.js index fa88050d6..a85c9bfe9 100644 --- a/Common/node/lservicemanager.js +++ b/Common/node/lservicemanager.js @@ -9,6 +9,7 @@ var fs = require("fs"); var path = require("path"); +var os = require("os"); var lconfig = require("lconfig"); var crypto = require("crypto"); var util = require("util"); @@ -479,6 +480,25 @@ function checkForShutdown() { shuttingDown = null; } +// create signal events for services to listen to +var workLast = null; +var workSig = ""; // need to re-signal if listeners change +function workCheck() +{ + var newSig = levents.displayListeners("work/me").join(" "); + if(newSig != workSig) workLast = null; + workSig = newSig; + if(newSig == "") return; // no listeners! + var work = "start"; + var load = os.loadavg(); + if(load[0] > lconfig.workWarn) work = "warn"; + if(load[0] > lconfig.workStop) work = "stop"; + if(work == workLast) return; // no changes! + workLast = work; + levents.fireEvent("work://me/#"+work,"new",{work:work}); +} +setInterval(workCheck, 10000); // 10s granularity + exports.getCollectionApis = function() { var collectionApis = {}; for (var i in serviceMap) { diff --git a/Common/node/synclet/dataaccess.js b/Common/node/synclet/dataaccess.js index 71c727f9e..1b35bd9bb 100644 --- a/Common/node/synclet/dataaccess.js +++ b/Common/node/synclet/dataaccess.js @@ -15,6 +15,15 @@ module.exports = function(app) { var options = {}; if(req.query['limit']) options.limit = parseInt(req.query['limit']); if(req.query['offset']) options.skip = parseInt(req.query['offset']); + if(req.query["sort"]) { + var sorter = {} + if(req.query["order"]) { + sorter[req.query["sort"]] = +req.query["order"]; + } else { + sorter[req.query["sort"]] = 1; + } + options.sort = sorter; + } if(req.query['stream'] == "true") { @@ -95,7 +104,7 @@ module.exports = function(app) { res.end(JSON.stringify(doc)); } else { res.writeHead(404); - res.end(); + res.end("not found"); } }); });