From aec0e730d7472c906835b6b411875fe65bcac882 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Sun, 2 Oct 2011 11:49:29 -0500 Subject: [PATCH 01/40] attempting a clean merge of Timeline, some very funky things happened w/ git so not sure --- .../migrations/1316810745000_update.js | 3 + Collections/Timeline/README.md | 5 + Collections/Timeline/dataIn.js | 244 ++++++++++++ Collections/Timeline/dataStore.js | 78 ++++ Collections/Timeline/fixtures/facebook.update | 63 ++++ .../Timeline/fixtures/foursquare.recents | 65 ++++ Collections/Timeline/fixtures/item | 23 ++ Collections/Timeline/fixtures/response | 13 + Collections/Timeline/fixtures/twitter.related | 355 ++++++++++++++++++ .../Timeline/fixtures/twitter.timeline | 88 +++++ Collections/Timeline/timeline.collection | 15 + Collections/Timeline/timeline.js | 133 +++++++ Common/node/lsyncmanager.js | 20 + 13 files changed, 1105 insertions(+) create mode 100644 Collections/Contacts/migrations/1316810745000_update.js create mode 100644 Collections/Timeline/README.md create mode 100644 Collections/Timeline/dataIn.js create mode 100644 Collections/Timeline/dataStore.js create mode 100644 Collections/Timeline/fixtures/facebook.update create mode 100644 Collections/Timeline/fixtures/foursquare.recents create mode 100644 Collections/Timeline/fixtures/item create mode 100644 Collections/Timeline/fixtures/response create mode 100644 Collections/Timeline/fixtures/twitter.related create mode 100644 Collections/Timeline/fixtures/twitter.timeline create mode 100644 Collections/Timeline/timeline.collection create mode 100644 Collections/Timeline/timeline.js diff --git a/Collections/Contacts/migrations/1316810745000_update.js b/Collections/Contacts/migrations/1316810745000_update.js new file mode 100644 index 000000000..45489804c --- /dev/null +++ b/Collections/Contacts/migrations/1316810745000_update.js @@ -0,0 +1,3 @@ +module.exports = function(dir) { + return "update"; +}; diff --git a/Collections/Timeline/README.md b/Collections/Timeline/README.md new file mode 100644 index 000000000..651072522 --- /dev/null +++ b/Collections/Timeline/README.md @@ -0,0 +1,5 @@ +The "feeds" collection is all of the socially shared items across any service, facebook newsfeed, twitter timeline, foursquare recents, etc. + +It consists of a primary type called an "item" which can have an array of "responses" that are comments, likes, retweets, etc from other people. + +The goal is to dedup cross-posts to multiple networks from one person, particularly things like a foursquare checkin. \ No newline at end of file diff --git a/Collections/Timeline/dataIn.js b/Collections/Timeline/dataIn.js new file mode 100644 index 000000000..05b2e6bcf --- /dev/null +++ b/Collections/Timeline/dataIn.js @@ -0,0 +1,244 @@ +var request = require('request'); +var async = require('async'); +var logger = require(__dirname + "/../../Common/node/logger").logger; +var lutil = require('lutil'); +var url = require('url'); +var crypto = require("crypto"); + +var dataStore, locker; + +// internally we need these for happy fun stuff +exports.init = function(l, dStore){ + dataStore = dStore; + locker = l; +} + +// manually walk and reindex all possible link sources +exports.update = function(locker, callback) { + dataStore.clear(function(){ + callback(); + locker.providers(['link/facebook', 'status/twitter', 'checkin/foursquare'], function(err, services) { + if (!services) return; + services.forEach(function(svc) { + if(svc.provides.indexOf('link/facebook') >= 0) { + getData("home/facebook", svc.id); + } else if(svc.provides.indexOf('status/twitter') >= 0) { + getData("tweets/twitter", svc.id); + getData("timeline/twitter", svc.id); + getData("mentions/twitter", svc.id); + } else if(svc.provides.indexOf('checkin/foursquare') >= 0) { + getData("recents/foursquare", svc.id); + getData("checkin/foursquare", svc.id); + } + }); + }); + }); +} + +// go fetch data from sources to bulk process +function getData(type, svcId) +{ + var subtype = type.substr(0, type.indexOf('/')); + var lurl = locker.lockerBase + '/Me/' + svcId + '/getCurrent/' + subtype; + request.get({uri:lurl, json:true}, function(err, resp, arr) { + async.forEachSeries(arr,function(a,cb){ + var idr = getIdr(type, svcId, a); + masterMaster(idr, a, cb); + },function(err){ + logger.debug("processed "+arr.length+" items from "+lurl+" "+(err)?err:""); + }); + }); +} + +// generate unique id for any item based on it's event +//> u.parse("type://network/context?id=account#46234623456",true); +//{ protocol: 'type:', +// slashes: true, +// host: 'network', +// hostname: 'network', +// href: 'type://network/context?id=account#46234623456', +// hash: '#46234623456', +// search: '?id=account', +// query: { id: 'account' }, +// pathname: '/context' } +function getIdr(type, via, data) +{ + var r = {slashes:true}; + r.host = type.substr(type.indexOf('/')+1); + r.pathname = type.substr(0, type.indexOf('/')); + r.query = {id: via}; // best proxy of account id right now + if(r.host === 'twitter') + { + r.hash = (r.pathname === 'related') ? data.id : data.id_str; + r.protocol = 'tweet'; + } + if(r.host === 'facebook') + { + r.hash = data.id; + r.protocol = 'post'; + } + if(r.host === 'foursquare') + { + r.hash = data.id; + r.protocol = 'checkin'; + } + return url.parse(url.format(r)); // make sure it's consistent +} + +// take an idr and turn it into a generic network-global key +// this could be network-specific transformation and need data +function idr2key(idr, data) +{ + delete idr.query; delete idr.search; // not account specific + idr.pathname = '/'; // ids are generic across any context + return url.parse(url.format(idr)); +} + +// normalize events a bit +exports.processEvent = function(event, callback) +{ + if(!callback) callback = function(){}; + // handle links as a special case as we're using them for post-process-deduplication + if(event.type == 'link') return processLink(event, callback); + + var idr = getIdr(event.type, event.via, event.obj.data); + masterMaster(idr, event.obj.data, callback); +} + +function isItMe(idr) +{ + if(idr.protocol == 'tweet:' && idr.pathname == '/tweets') return true; + if(idr.protocol == 'checkin:' && idr.pathname == '/checkin') return true; + return false; +} + +// figure out what to do with any data +function masterMaster(idr, data, callback) +{ + if(typeof data != 'object') return callback(); +// logger.debug("MM\t"+url.format(idr)); + var ref = url.format(idr); + var item = {keys:{}, refs:[], froms:{}, from:{}}; + item.ref = ref; + item.refs.push(ref); + item.keys[url.format(idr2key(idr, data))] = item.ref; + item.me = isItMe(idr); + if(idr.protocol == 'tweet:') itemTwitter(item, data); + if(idr.protocol == 'post:') itemFacebook(item, data); + if(idr.protocol == 'checkin:') itemFoursquare(item, data); + var dup; + // we're only looking for the first match, if there's more, that's a very odd situation but could be handled here + async.forEach(Object.keys(item.keys), function(key, cb) { + dataStore.getItemByKey(key,function(err, doc){ + dup = doc; + cb(true); + }); + }, function (err) { + if(dup) item = itemMerge(dup, item); + dataStore.addItem(item, function(err, doc){ +// logger.debug("ADDED\t"+JSON.stringify(doc)); + callback(); + }); + }); +} + +// intelligently merge two items together and return +function itemMerge(older, newer) +{ + logger.debug("MERGE\t"+older.ref+'\t'+newer.ref); + if(newer.pri > older.pri) + { // replace top level summary stuff if newer is deemed better + older.ref = newer.ref; + older.from = newer.from; + older.text = newer.text; + if(newer.title) older.title = newer.title; + } + // update timestamps to extremes + if(newer.last > older.last) older.last = newer.last; + if(newer.first < older.first) older.first = newer.first; + // update the two searchable objects + for(var k in newer.keys) older.keys[k] = newer.keys[k]; + for(var k in newer.froms) older.froms[k] = newer.froms[k]; + // our array of references for this item, needs to be unique + var refs = {}; + for(var i = 0; i < newer.refs.length; i++) refs[newer.refs[i]]=true; + for(var i = 0; i < older.refs.length; i++) refs[older.refs[i]]=true; + older.refs = Object.keys(refs); + return older; +} + +// when a processed link event comes in, check to see if it's a long url that could help us de-dup +function processLink(event, callback) +{ + // look up event via/orig key and see if we've processed it yet, if not (for some reason) ignore + // if foursquare checkin and from a tweet, generate foursquare key and look for it + // if from instagram, generate instagram key and look for it + + // by here should have the item with the link, and the item for which the link is a target that is a duplicate + // merge them, merge any responses, delete the lesser + callback(); +} + +// extract info from a tweet +function itemTwitter(item, tweet) +{ + item.pri = 1; // tweets are the lowest priority? + if(tweet.created_at) item.first = item.last = new Date(tweet.created_at).getTime(); + if(tweet.text) item.text = tweet.text; + if(tweet.user) + { + item.from.id = "twitter:"+tweet.user.id; + item.from.name = tweet.user.name; + item.from.icon = tweet.user.profile_image_url; + item.froms[item.from.id] = item.ref; + } + // add a text based key + if(item.text) + { + var hash = crypto.createHash('md5'); + hash.update(item.text.substr(0,130)); // ignore trimming variations + item.keys['text:'+hash.digest('hex')] = item.ref; + } + // if it's tracking a reply, key it too + if(tweet.in_reply_to_status_id_str) item.keys['tweet://twitter/#'+tweet.in_reply_to_status_id_str] = item.ref; + // TODO track links for link matching +} + +// extract info from a facebook post +function itemFacebook(item, post) +{ + item.pri = 2; // facebook allows more text, always better? + item.first = post.created_time * 1000; + item.last = post.updated_time * 1000; + if(post.from) + { + item.from.id = 'facebook:'+post.from.id; + item.from.name = post.from.name; + item.from.icon = 'https://graph.facebook.com/' + post.from.id + '/picture'; + item.froms[item.from.id] = item.ref; + } + if(post.name) item.title = post.name; + if(post.message) item.text = post.message; + if(!post.message && post.caption) item.text = post.caption; + // should we only add a text key if we can detect it's a tweet? we're de-duping any fb post essentially + if(item.text) + { + var hash = crypto.createHash('md5'); + hash.update(item.text.substr(0,130)); // ignore trimming variations + item.keys['text:'+hash.digest('hex')] = item.ref; + } +} + +// extract info from a foursquare checkin +function itemFoursquare(item, checkin) +{ + item.pri = 3; // ideally a checkin should source here as the best + item.first = item.last = checkin.createdAt * 1000; + if(checkin.venue) item.text = "Checked in at " + checkin.venue.name; + if(checkin.user) + { + item.from.id = 'foursquare:'+checkin.user.id; + item.from.name = checkin.user.firstName + " " + checkin.user.lastName; + item.from.icon = checkin.user.photo; + } +} diff --git a/Collections/Timeline/dataStore.js b/Collections/Timeline/dataStore.js new file mode 100644 index 000000000..85f994359 --- /dev/null +++ b/Collections/Timeline/dataStore.js @@ -0,0 +1,78 @@ +/* +* +* Copyright (C) 2011, The Locker Project +* All rights reserved. +* +* Please see the LICENSE file for more information. +* +*/ +var logger = require(__dirname + "/../../Common/node/logger").logger; +var crypto = require("crypto"); + +// in the future we'll probably need a visitCollection too +var itemCol, respCol; + +exports.init = function(iCollection, rCollection) { + itemCol = iCollection; +// itemCol.ensureIndex({"item":1},{unique:true},function() {}); + respCol = rCollection; +} + +exports.clear = function(callback) { + itemCol.drop(function(){respCol.drop(callback)}); +} + +exports.getTotalItems = function(callback) { + itemCol.count(callback); +} +exports.getTotalResponses = function(callback) { + respColl.count(callback); +} + +exports.getItemByKey = function(key, callback) { + var item; + var kname = "keys."+key; + var find = {}; + find[kname] = {$exists:true}; + findWrap(find,{limit:1},itemCol,function(i){item=i},function(err){callback(err,item)}); +} + +exports.getItem = function(id, callback) { + var item; + findWrap({id:id},{},itemCol,function(i){item=i},function(err){callback(err,item)}); +} + +// arg takes sort/limit/offset/find +exports.getItems = function(arg, cbEach, cbDone) { + var f = {}; + try { + f = JSON.parse(arg.find); // optional, can bomb out + }catch(E){} + delete arg.find; + findWrap(f,arg,itemCol,cbEach,cbDone); +} + +function findWrap(a,b,c,cbEach,cbDone){ +// console.log("a(" + JSON.stringify(a) + ") b("+ JSON.stringify(b) + ")"); + var cursor = c.find(a); + if (b.sort) cursor.sort(parseInt(b.sort)); + if (b.limit) cursor.limit(parseInt(b.limit)); + if (b.offset) cursor.skip(parseInt(b.offset)); + cursor.each(function(err, item) { + if (item != null) { + cbEach(item); + } else { + cbDone(); + } + }); +} + + +// insert new (fully normalized) item, generate the id here and now +exports.addItem = function(item, callback) { + var hash = crypto.createHash('md5'); + for(var i in item.keys) hash.update(i); + item.id = hash.digest('hex'); +// logger.debug("addItem: "+JSON.stringify(item)); + itemCol.findAndModify({"id":item.id}, [['_id','asc']], {$set:item}, {safe:true, upsert:true, new: true}, callback); +} diff --git a/Collections/Timeline/fixtures/facebook.update b/Collections/Timeline/fixtures/facebook.update new file mode 100644 index 000000000..fa8c0a03f --- /dev/null +++ b/Collections/Timeline/fixtures/facebook.update @@ -0,0 +1,63 @@ +{ + "type": "home/facebook", + "via": "synclet/facebook", + "timestamp": 1315976215195, + "action": "update", + "obj": { + "source": "facebook_home", + "type": "update", + "data": { + "id": "1264086533_2419826536086", + "from": { + "name": "Mindframe Theaters", + "id": "1264086533" + }, + "picture": "http://vthumb.ak.fbcdn.net/hvthumb-ak-snc6/244520_2419869537161_2419826536086_33311_1352_t.jpg", + "link": "http://www.facebook.com/photo.php?v=2419826536086", + "source": "http://video.l3.fbcdn.net/cfs-l3-ash4/342015/380/2419826536086_60821.mp4?oh=115eecc26ea0e77e74bdfb39d3860a33&oe=4E72D400&l3s=20110914043344&l3e=20110916044344&lh=065388ed3f72cef427b4b", + "name": "The Tree of Life - 9/16 [HD]", + "properties": [{ + "name": "Length", + "text": "2:11" + }], + "icon": "http://static.ak.fbcdn.net/rsrc.php/v1/yD/r/DggDhA4z4tO.gif", + "actions": [{ + "name": "Comment", + "link": "http://www.facebook.com/1264086533/posts/2419826536086" + }, + { + "name": "Like", + "link": "http://www.facebook.com/1264086533/posts/2419826536086" + }], + "type": "video", + "object_id": "2419826536086", + "application": { + "name": "Video", + "id": "2392950137" + }, + "created_time": 1315859039, + "updated_time": 1315864735, + "likes": { + "data": [{ + "name": "Will Kelly", + "id": "760380150" + }], + "count": 9 + }, + "comments": { + "data": [{ + "id": "1264086533_2419826536086_3034673", + "from": { + "name": "Bryan J. Zygmont", + "id": "62106448" + }, + "message": "Thank you. See you soon.", + "created_time": 1315864735 + }], + "count": 1 + }, + "_id": "4e6e6d354f7f2ab135d2782b" + } + } +} +Ó \ No newline at end of file diff --git a/Collections/Timeline/fixtures/foursquare.recents b/Collections/Timeline/fixtures/foursquare.recents new file mode 100644 index 000000000..9e7087fa8 --- /dev/null +++ b/Collections/Timeline/fixtures/foursquare.recents @@ -0,0 +1,65 @@ +{ + "type": "recents/foursquare", + "via": "synclet/foursquare", + "timestamp": 1315977793913, + "action": "new", + "obj": { + "source": "foursquare_recents", + "data": { + "_id": "4e703a41ca17313fd2cccc6b", + "id": "4e703881d4c03fcb653e0251", + "createdAt": 1315977345, + "type": "checkin", + "timeZone": "America/Chicago", + "entities": [], + "user": { + "id": "10598838", + "firstName": "Lis", + "lastName": "Miller", + "photo": "https://playfoursquare.s3.amazonaws.com/userpix_thumbs/ZHSLPGESZ530RZEK.jpg", + "gender": "female", + "homeCity": "Cascade, IA", + "relationship": "friend" + }, + "venue": { + "id": "4e1362df18a84f0f0325cb37", + "name": "Home: 605 Garfield St SW", + "contact": {}, + "location": { + "address": "605 Garfield St. SW", + "lat": 42.292913, + "lng": -91.018424, + "city": "Cascade", + "state": "IA" + }, + "categories": [{ + "id": "4bf58dd8d48988d103941735", + "name": "Home", + "pluralName": "Homes", + "shortName": "Home", + "icon": "https://foursquare.com/img/categories/building/home.png", + "parents": ["Residences"], + "primary": true + }], + "verified": false, + "stats": { + "checkinsCount": 246, + "usersCount": 4, + "tipCount": 0 + } + }, + "source": { + "name": "foursquare for iPhone", + "url": "https://foursquare.com/download/#/iphone" + }, + "photos": { + "count": 0, + "items": [] + }, + "comments": { + "count": 0, + "items": [] + } + } + } +} diff --git a/Collections/Timeline/fixtures/item b/Collections/Timeline/fixtures/item new file mode 100644 index 000000000..c60498ec6 --- /dev/null +++ b/Collections/Timeline/fixtures/item @@ -0,0 +1,23 @@ +{ + "id":"4e703a41ca17313fd2cccc6b", + "first":12345566778, + "last":12356456779, + "keys":{ + "tweet://twitter/#235123423421":"tweet://twitter/timeline?id=twitter#235123423421", + }, + "ref":"tweet://twitter/timeline?id=twitter#235123423421", + "refs":[ + "tweet://twitter/timeline?id=twitter#235123423421", + ], + pri:4, + me:false, + text:"This is my text", + from:{ + "id":"contact://twitter/#jeremie", + "icon":"http://twitter.com/icon.png", + "name":"Jer" + }, + froms:{ + "contact://twitter/#jeremie":"tweet://twitter/timeline?id=twitter#235123423421" + } +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/response b/Collections/Timeline/fixtures/response new file mode 100644 index 000000000..bedca6e3a --- /dev/null +++ b/Collections/Timeline/fixtures/response @@ -0,0 +1,13 @@ +{ + "id":"hash generated from all KVs below", + "item":"4e703a41ca17313fd2cccc6b", + "at":12345566778, + "ref":"tweet://twitter/timeline?id=twitter#235123423421", + "text":"This is my text", + "type":"comment" + from:{ + "id":"contact://twitter/#jeremie", + "icon":"http://twitter.com/icon.png", + "name":"Jer" + }, +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/twitter.related b/Collections/Timeline/fixtures/twitter.related new file mode 100644 index 000000000..405292557 --- /dev/null +++ b/Collections/Timeline/fixtures/twitter.related @@ -0,0 +1,355 @@ +{ + "type": "related/twitter", + "via": "synclet/twitter", + "timestamp": 1315976039292, + "action": "new", + "obj": { + "source": "twitter_related", + "type": "new", + "data": { + "_id": "4e703367bcb6f2f6dd06c867", + "id": "113792617996759041", + "related": [{ + "results": [{ + "contributors_enabled": false, + "profile_use_background_image": true, + "protected": false, + "id_str": "264083385", + "time_zone": "London", + "profile_background_color": "ffffff", + "name": "Barni", + "default_profile": false, + "profile_background_image_url": "http://a3.twimg.com/profile_background_images/307873995/Copy_of_Lion_Profile.jpg", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1493346358/Copy_of_DSCN3225_normal.JPG", + "default_profile_image": false, + "following": false, + "utc_offset": 0, + "profile_image_url": "http://a1.twimg.com/profile_images/1493346358/Copy_of_DSCN3225_normal.JPG", + "description": " Whatever I did not know, I was not ashamed to inquire about, so I aquired knowledge.\r\n\r\n", + "show_all_inline_media": false, + "geo_enabled": false, + "friends_count": 80, + "profile_text_color": "7e7aff", + "location": "Global ", + "is_translator": false, + "profile_sidebar_fill_color": "000000", + "status": { + "in_reply_to_user_id_str": null, + "retweet_count": 6, + "in_reply_to_status_id": null, + "id_str": "113813170862301184", + "contributors": null, + "truncated": false, + "retweeted_status": { + "in_reply_to_user_id_str": null, + "retweet_count": 6, + "in_reply_to_status_id": null, + "id_str": "113792617996759041", + "contributors": null, + "truncated": false, + "geo": null, + "coordinates": null, + "favorited": false, + "in_reply_to_user_id": null, + "in_reply_to_screen_name": null, + "possibly_sensitive": false, + "retweeted": false, + "source": "bitly", + "in_reply_to_status_id_str": null, + "id": 113792617996759040, + "place": null, + "text": "From March: @BarackObama explains how HuffPo would report the Emancipation Proclamation: http://t.co/kBD07wN", + "created_at": "Wed Sep 14 01:54:08 +0000 2011" + }, + "geo": null, + "coordinates": null, + "favorited": false, + "in_reply_to_user_id": null, + "in_reply_to_screen_name": null, + "possibly_sensitive": false, + "retweeted": false, + "source": "web", + "in_reply_to_status_id_str": null, + "id": 113813170862301180, + "place": null, + "text": "RT @anildash: From March: @BarackObama explains how HuffPo would report the Emancipation Proclamation: http://t.co/kBD07wN", + "created_at": "Wed Sep 14 03:15:49 +0000 2011" + }, + "follow_request_sent": false, + "profile_background_tile": true, + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/307873995/Copy_of_Lion_Profile.jpg", + "url": "http://iphotosbyab.tumblr.com/", + "statuses_count": 428, + "followers_count": 50, + "screen_name": "iJauntygrapher", + "notifications": false, + "profile_link_color": "ff802b", + "lang": "en", + "verified": false, + "favourites_count": 5, + "profile_sidebar_border_color": "ffffff", + "id": 264083385, + "listed_count": 1, + "created_at": "Fri Mar 11 09:02:42 +0000 2011" + }, + { + "contributors_enabled": false, + "profile_use_background_image": true, + "protected": false, + "id_str": "25013495", + "default_profile": false, + "time_zone": "Pacific Time (US & Canada)", + "profile_background_color": "030302", + "name": "Max Benavidez, PhD", + "profile_background_image_url": "http://a2.twimg.com/profile_background_images/237835820/xad508adf00907e14e42fd292e1a7ff0.jpg", + "profile_image_url_https": "https://si0.twimg.com/profile_images/996977116/Benavidez_Photo_normal.jpg", + "default_profile_image": false, + "following": false, + "utc_offset": -28800, + "profile_image_url": "http://a0.twimg.com/profile_images/996977116/Benavidez_Photo_normal.jpg", + "description": "PhD, news analyst, new media consultant, author", + "show_all_inline_media": false, + "geo_enabled": false, + "friends_count": 1886, + "profile_text_color": "786a55", + "location": "Los Angeles, California", + "is_translator": false, + "profile_sidebar_fill_color": "12291f", + "status": { + "in_reply_to_user_id_str": null, + "retweet_count": 38, + "in_reply_to_status_id": null, + "id_str": "113799556541587457", + "contributors": null, + "truncated": false, + "retweeted_status": { + "in_reply_to_user_id_str": null, + "retweet_count": 38, + "in_reply_to_status_id": null, + "id_str": "113799495749337089", + "contributors": null, + "truncated": false, + "geo": null, + "coordinates": null, + "favorited": false, + "in_reply_to_user_id": null, + "in_reply_to_screen_name": null, + "possibly_sensitive": false, + "retweeted": false, + "source": "SocialFlow", + "in_reply_to_status_id_str": null, + "id": 113799495749337090, + "place": null, + "text": "What happens to fornicating passengers who get caught? http://t.co/MZkK67k #mysteries #explained", + "created_at": "Wed Sep 14 02:21:28 +0000 2011" + }, + "geo": null, + "coordinates": null, + "favorited": false, + "in_reply_to_user_id": null, + "in_reply_to_screen_name": null, + "possibly_sensitive": false, + "retweeted": false, + "source": "web", + "in_reply_to_status_id_str": null, + "id": 113799556541587460, + "place": null, + "text": "RT @Slate: What happens to fornicating passengers who get caught? http://t.co/MZkK67k #mysteries #explained", + "created_at": "Wed Sep 14 02:21:43 +0000 2011" + }, + "follow_request_sent": false, + "profile_background_tile": false, + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/237835820/xad508adf00907e14e42fd292e1a7ff0.jpg", + "url": "http://theamericanshow.com/", + "statuses_count": 1749, + "followers_count": 685, + "screen_name": "MaxBenavidez", + "notifications": false, + "profile_link_color": "547b61", + "lang": "en", + "verified": false, + "favourites_count": 1, + "profile_sidebar_border_color": "3c5449", + "id": 25013495, + "listed_count": 15, + "created_at": "Wed Mar 18 02:45:11 +0000 2009" + }, + { + "verified": false, + "favourites_count": 12481, + "profile_sidebar_border_color": "dfe394", + "protected": false, + "listed_count": 46, + "name": "Dinah Sanders", + "default_profile": false, + "contributors_enabled": false, + "profile_use_background_image": true, + "id_str": "6722", + "following": false, + "time_zone": "Pacific Time (US & Canada)", + "utc_offset": -28800, + "profile_background_color": "4c8018", + "description": "Writer. Productivity and happiness coach. Drinker of classic cocktails. Pre-blogs blogger.", + "profile_background_image_url": "http://a0.twimg.com/profile_background_images/303847553/beedleumbum_edgewoodmeadow.jpg", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1002780733/201006dpsbytecgirl_lookup_normal.png", + "location": "San Francisco, CA, USA", + "default_profile_image": false, + "profile_image_url": "http://a1.twimg.com/profile_images/1002780733/201006dpsbytecgirl_lookup_normal.png", + "show_all_inline_media": true, + "geo_enabled": false, + "friends_count": 95, + "profile_text_color": "333333", + "url": "http://www.metagrrrl.com", + "is_translator": false, + "profile_sidebar_fill_color": "bbf08d", + "screen_name": "MetaGrrrl", + "follow_request_sent": false, + "notifications": false, + "profile_background_tile": false, + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/303847553/beedleumbum_edgewoodmeadow.jpg", + "lang": "en", + "statuses_count": 7093, + "followers_count": 965, + "id": 6722, + "created_at": "Sat Sep 23 18:20:44 +0000 2006", + "profile_link_color": "0084b4" + }, + { + "contributors_enabled": false, + "profile_use_background_image": true, + "protected": false, + "id_str": "14367955", + "time_zone": "Pacific Time (US & Canada)", + "profile_background_color": "333333", + "name": "Kevin Cantù", + "default_profile": false, + "profile_background_image_url": "http://a3.twimg.com/profile_background_images/232144494/3517286539_06a527be68_o.jpg", + "profile_image_url_https": "https://si0.twimg.com/profile_images/202909645/2430137243_0223b18eab_o_normal.jpg", + "default_profile_image": false, + "following": false, + "utc_offset": -28800, + "profile_image_url": "http://a3.twimg.com/profile_images/202909645/2430137243_0223b18eab_o_normal.jpg", + "description": "I occasionally say interesting things!", + "show_all_inline_media": false, + "geo_enabled": true, + "friends_count": 712, + "profile_text_color": "333333", + "location": "Santa Barbara, California", + "is_translator": false, + "profile_sidebar_fill_color": "f6ffd1", + "status": { + "in_reply_to_user_id_str": "47951511", + "retweet_count": 0, + "in_reply_to_status_id": 113834332413624320, + "id_str": "113836727860346881", + "contributors": null, + "truncated": false, + "geo": null, + "coordinates": null, + "favorited": false, + "in_reply_to_user_id": 47951511, + "in_reply_to_screen_name": "zunguzungu", + "retweeted": false, + "source": "web", + "in_reply_to_status_id_str": "113834332413624320", + "id": 113836727860346880, + "place": { + "name": "Santa Barbara", + "attributes": {}, + "full_name": "Santa Barbara, CA", + "bounding_box": { + "type": "Polygon", + "coordinates": [[[ - 119.859784, 34.336029], [ - 119.639922, 34.336029], [ - 119.639922, 34.460896], [ - 119.859784, 34.460896]]] + }, + "place_type": "city", + "url": "http://api.twitter.com/1/geo/id/f6ebc676e5cde864.json", + "country_code": "US", + "country": "United States", + "id": "f6ebc676e5cde864" + }, + "text": "@zunguzungu Atta is the Bolshevik to Bouazizi's serfdom: they are at two corners of a larger problem", + "created_at": "Wed Sep 14 04:49:25 +0000 2011" + }, + "follow_request_sent": false, + "profile_background_tile": true, + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/232144494/3517286539_06a527be68_o.jpg", + "url": "http://kevincantu.org", + "statuses_count": 12240, + "followers_count": 316, + "screen_name": "killerswan", + "notifications": false, + "profile_link_color": "98b302", + "lang": "en", + "verified": false, + "favourites_count": 7951, + "profile_sidebar_border_color": "ffffad", + "id": 14367955, + "listed_count": 24, + "created_at": "Sat Apr 12 09:25:23 +0000 2008" + }, + { + "contributors_enabled": false, + "profile_use_background_image": true, + "protected": false, + "id_str": "6299142", + "time_zone": "New Delhi", + "profile_background_color": "8B542B", + "name": "Firas", + "profile_background_image_url": "http://a1.twimg.com/profile_background_images/88254516/palmetto_t.gif", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1314697710/cropprofile1_normal.png", + "default_profile_image": false, + "default_profile": false, + "following": null, + "utc_offset": 19800, + "profile_image_url": "http://a1.twimg.com/profile_images/1314697710/cropprofile1_normal.png", + "description": "I like you already.", + "show_all_inline_media": false, + "geo_enabled": true, + "friends_count": 322, + "profile_text_color": "333333", + "location": "Delhi, India", + "is_translator": false, + "profile_sidebar_fill_color": "EADEAA", + "status": { + "in_reply_to_user_id_str": null, + "retweet_count": 0, + "in_reply_to_status_id": null, + "id_str": "113817061733576705", + "contributors": null, + "truncated": false, + "geo": null, + "coordinates": null, + "favorited": false, + "in_reply_to_user_id": null, + "in_reply_to_screen_name": null, + "possibly_sensitive": false, + "retweeted": false, + "source": "web", + "in_reply_to_status_id_str": null, + "id": 113817061733576700, + "place": null, + "text": "that’s what I’m talking about → RT @Leffot: Photo: Hand-sewing shell cordovan http://t.co/rcxrU20", + "created_at": "Wed Sep 14 03:31:16 +0000 2011" + }, + "follow_request_sent": null, + "profile_background_tile": true, + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/88254516/palmetto_t.gif", + "url": "http://firasd.org", + "statuses_count": 2821, + "followers_count": 215, + "screen_name": "firasd", + "notifications": null, + "profile_link_color": "9D582E", + "lang": "en", + "verified": false, + "favourites_count": 3, + "profile_sidebar_border_color": "D9B17E", + "id": 6299142, + "listed_count": 16, + "created_at": "Fri May 25 00:02:48 +0000 2007" + }], + "resultType": "ReTweet" + }] + } + } +} diff --git a/Collections/Timeline/fixtures/twitter.timeline b/Collections/Timeline/fixtures/twitter.timeline new file mode 100644 index 000000000..89bd37d1c --- /dev/null +++ b/Collections/Timeline/fixtures/twitter.timeline @@ -0,0 +1,88 @@ +{ + "type": "timeline/twitter", + "via": "synclet/twitter", + "timestamp": 1315974920913, + "action": "new", + "obj": { + "source": "twitter_timeline", + "type": "new", + "data": { + "_id": "4e702f08bcb6f2f6dd06c849", + "in_reply_to_user_id_str": null, + "retweet_count": 0, + "in_reply_to_status_id": null, + "id_str": "113829905090879488", + "contributors": null, + "truncated": false, + "geo": null, + "coordinates": null, + "favorited": false, + "user": { + "default_profile": false, + "contributors_enabled": false, + "profile_use_background_image": true, + "protected": false, + "id_str": "633", + "time_zone": "Eastern Time (US & Canada)", + "profile_background_color": "1A1B1F", + "name": "danah boyd", + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme9/bg.gif", + "profile_image_url_https": "https://si0.twimg.com/profile_images/890666764/MeAtTalkWithHat-Sq_normal.jpg", + "default_profile_image": false, + "following": true, + "utc_offset": -18000, + "profile_image_url": "http://a0.twimg.com/profile_images/890666764/MeAtTalkWithHat-Sq_normal.jpg", + "description": "social media scholar, youth researcher & advocate | Microsoft Research, Harvard Berkman Center | zephoria@zephoria.org", + "show_all_inline_media": false, + "geo_enabled": false, + "friends_count": 734, + "profile_text_color": "666666", + "location": "Boston, MA", + "is_translator": false, + "profile_sidebar_fill_color": "252429", + "follow_request_sent": false, + "profile_background_tile": false, + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme9/bg.gif", + "url": "http://www.zephoria.org/thoughts/", + "statuses_count": 2471, + "followers_count": 50313, + "screen_name": "zephoria", + "notifications": false, + "profile_link_color": "2FC2EF", + "lang": "en", + "verified": false, + "favourites_count": 44, + "profile_sidebar_border_color": "181A1E", + "id": 633, + "listed_count": 5224, + "created_at": "Thu Jul 13 21:27:23 +0000 2006" + }, + "in_reply_to_user_id": null, + "in_reply_to_screen_name": null, + "possibly_sensitive": false, + "retweeted": false, + "source": "web", + "in_reply_to_status_id_str": null, + "entities": { + "user_mentions": [{ + "name": "Wil Wheaton", + "id_str": "1183041", + "indices": [108, 113], + "screen_name": "wilw", + "id": 1183041 + }], + "hashtags": [], + "urls": [{ + "indices": [116, 135], + "display_url": "amzn.to/qqvlo4", + "url": "http://t.co/zAN4G8S", + "expanded_url": "http://amzn.to/qqvlo4" + }] + }, + "id": 113829905090879490, + "place": null, + "text": "Just finished \"Ready Player One.\" YA dystopia mixed w/ 1980s geek nostalgia. Soooo much fun. (Audio read by @wilw ) http://t.co/zAN4G8S", + "created_at": "Wed Sep 14 04:22:18 +0000 2011" + } + } +} diff --git a/Collections/Timeline/timeline.collection b/Collections/Timeline/timeline.collection new file mode 100644 index 000000000..fb052ce52 --- /dev/null +++ b/Collections/Timeline/timeline.collection @@ -0,0 +1,15 @@ +{ + "title":"Timeline", + "desc":"A collection of socially shared items from various networks.", + "run":"node timeline.js", + "status":"stable", + "handle":"timeline", + "provides":["item"], + "mongoCollections": ["item","response"], + "events":[["recents/foursquare","/events"] + ,["timeline/twitter","/events"] + ,["related/twitter","/events"] + ,["home/facebook","/events"] + ,["link","/events"] + ] +} diff --git a/Collections/Timeline/timeline.js b/Collections/Timeline/timeline.js new file mode 100644 index 000000000..616e3100c --- /dev/null +++ b/Collections/Timeline/timeline.js @@ -0,0 +1,133 @@ +/* +* +* Copyright (C) 2011, The Locker Project +* All rights reserved. +* +* Please see the LICENSE file for more information. +* +*/ + +// merge newsfeed style data and any gather any responses + +var fs = require('fs'), + url = require('url'), + request = require('request'), + locker = require('../../Common/node/locker.js'); +var async = require("async"); + +var dataIn = require('./dataIn'); // for processing incoming twitter/facebook/etc data types +var dataStore = require("./dataStore"); // storage/retreival of raw items and responses + +var lockerInfo; +var express = require('express'), + connect = require('connect'); +var app = express.createServer(connect.bodyParser()); + +app.set('views', __dirname); + +app.get('/', function(req, res) { + res.writeHead(200, { + 'Content-Type': 'text/html' + }); + dataStore.getTotalItems(function(err, countInfo) { + res.write('

Found '+ countInfo +' items

'); + res.end(); + }); +}); + +app.get('/state', function(req, res) { + dataStore.getTotalItems(function(err, countInfo) { + if(err) return res.send(err, 500); + var updated = new Date().getTime(); + try { + var js = JSON.parse(fs.readFileSync('state.json')); + if(js && js.updated) updated = js.updated; + } catch(E) {} + res.send({ready:1, count:countInfo, updated:updated}); + }); +}); + + +app.get('/update', function(req, res) { + dataIn.update(locker, function(){ + res.writeHead(200); + res.end('Extra mince!'); + }); +}); + +app.post('/events', function(req, res) { + if (!req.body.type || !req.body.obj || !req.body.obj.data){ + console.log('5 HUNDO bad data:',JSON.stringify(req.body)); + res.writeHead(500); + res.end('bad data'); + return; + } + + // handle asyncadilly + dataIn.processEvent(req.body); + res.writeHead(200); + res.end('ok'); +}); + +function genericApi(name,f) +{ + app.get(name,function(req,res){ + var results = []; + f(req.query,function(item){results.push(item);},function(err){ + if(err) return res.send(err,500); + res.send(results); + }); + }); +} + +// expose way to get items and responses in one +app.get('/getItemsFull', function(req, res) { + var fullResults = []; + var results = []; + var options = {sort:{"at":-1}}; + if (req.query.limit) { + options.limit = parseInt(req.query.limit); + }else{ + options.limit = 100; + } + if (req.query.offset) { + options.offset = parseInt(req.query.offset); + } + dataStore.getItems(options, function(item) { results.push(item); }, function(err) { + async.forEach(results, function(item, callback) { + item.responses = []; + dataStore.getResponses({"item":item.idr}, function(response) { + item.responses.push(response); + }, function() { + fullResults.push(item); + callback(); + }); + }, function() { + res.send(results); + }); + }); +}); +genericApi('/getItems', dataStore.getItems); +genericApi('/getResponses',dataStore.getResponses); + +// Process the startup JSON object +process.stdin.resume(); +process.stdin.on('data', function(data) { + lockerInfo = JSON.parse(data); + locker.initClient(lockerInfo); + locker.lockerBase = lockerInfo.lockerUrl; + if (!lockerInfo || !lockerInfo['workingDirectory']) { + process.stderr.write('Was not passed valid startup information.'+data+'\n'); + process.exit(1); + } + process.chdir(lockerInfo.workingDirectory); + + locker.connectToMongo(function(mongo) { + // initialize all our libs + dataStore.init(mongo.collections.item,mongo.collections.response); + dataIn.init(locker, dataStore); + app.listen(lockerInfo.port, 'localhost', function() { + process.stdout.write(data); + }); + }); +}); diff --git a/Common/node/lsyncmanager.js b/Common/node/lsyncmanager.js index 26d0702a8..7fe69bd13 100644 --- a/Common/node/lsyncmanager.js +++ b/Common/node/lsyncmanager.js @@ -76,6 +76,26 @@ exports.findInstalled = function (callback) { } } +// combined an installed me.json info with any of it's manifest changes, a stopgap until these things can be separated cleanly +function mergeManifest(js) +{ + var serviceInfo = {}; + synclets.available.some(function(svcInfo) { + if (svcInfo.srcdir == js.srcdir) { + for(var a in svcInfo){serviceInfo[a]=svcInfo[a];} + return true; + } + return false; + }); + if (serviceInfo && serviceInfo.manifest) { + var fullInfo = JSON.parse(fs.readFileSync(serviceInfo.manifest)); + return lutil.extend(js, fullInfo); + } else { + return js; + } + +} + exports.scanDirectory = function(dir) { datastore = require('./synclet/datastore'); var files = fs.readdirSync(dir); From 98ce09e6c8e0f95000a016a503c2275d39b3bcb7 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Sun, 2 Oct 2011 11:56:23 -0500 Subject: [PATCH 02/40] cleaning up merge tirds --- .../migrations/1316810745000_update.js | 3 --- Common/node/lsyncmanager.js | 20 ------------------- 2 files changed, 23 deletions(-) delete mode 100644 Collections/Contacts/migrations/1316810745000_update.js diff --git a/Collections/Contacts/migrations/1316810745000_update.js b/Collections/Contacts/migrations/1316810745000_update.js deleted file mode 100644 index 45489804c..000000000 --- a/Collections/Contacts/migrations/1316810745000_update.js +++ /dev/null @@ -1,3 +0,0 @@ -module.exports = function(dir) { - return "update"; -}; diff --git a/Common/node/lsyncmanager.js b/Common/node/lsyncmanager.js index 7fe69bd13..26d0702a8 100644 --- a/Common/node/lsyncmanager.js +++ b/Common/node/lsyncmanager.js @@ -76,26 +76,6 @@ exports.findInstalled = function (callback) { } } -// combined an installed me.json info with any of it's manifest changes, a stopgap until these things can be separated cleanly -function mergeManifest(js) -{ - var serviceInfo = {}; - synclets.available.some(function(svcInfo) { - if (svcInfo.srcdir == js.srcdir) { - for(var a in svcInfo){serviceInfo[a]=svcInfo[a];} - return true; - } - return false; - }); - if (serviceInfo && serviceInfo.manifest) { - var fullInfo = JSON.parse(fs.readFileSync(serviceInfo.manifest)); - return lutil.extend(js, fullInfo); - } else { - return js; - } - -} - exports.scanDirectory = function(dir) { datastore = require('./synclet/datastore'); var files = fs.readdirSync(dir); From 2ffc000ecb5ca5f7c633dd46299c575ace20e018 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Wed, 2 Nov 2011 04:23:30 -0500 Subject: [PATCH 03/40] lots of fixes, pretty solid response support, and early link-based deduping --- Collections/Timeline/dataIn.js | 167 +- Collections/Timeline/dataStore.js | 49 +- Collections/Timeline/fixtures/facebook.update | 1 - Collections/Timeline/fixtures/link | 105 ++ Collections/Timeline/fixtures/twitter.related | 1438 +++++++++++++---- 5 files changed, 1460 insertions(+), 300 deletions(-) create mode 100644 Collections/Timeline/fixtures/link diff --git a/Collections/Timeline/dataIn.js b/Collections/Timeline/dataIn.js index 05b2e6bcf..4b3eb359c 100644 --- a/Collections/Timeline/dataIn.js +++ b/Collections/Timeline/dataIn.js @@ -4,6 +4,7 @@ var logger = require(__dirname + "/../../Common/node/logger").logger; var lutil = require('lutil'); var url = require('url'); var crypto = require("crypto"); +var path = require('path'); var dataStore, locker; @@ -67,6 +68,13 @@ function getIdr(type, via, data) r.host = type.substr(type.indexOf('/')+1); r.pathname = type.substr(0, type.indexOf('/')); r.query = {id: via}; // best proxy of account id right now + idrHost(r, data); + return url.parse(url.format(r)); // make sure it's consistent +} + +// internal util breakout +function idrHost(r, data) +{ if(r.host === 'twitter') { r.hash = (r.pathname === 'related') ? data.id : data.id_str; @@ -82,7 +90,6 @@ function getIdr(type, via, data) r.hash = data.id; r.protocol = 'checkin'; } - return url.parse(url.format(r)); // make sure it's consistent } // take an idr and turn it into a generic network-global key @@ -94,10 +101,20 @@ function idr2key(idr, data) return url.parse(url.format(idr)); } +// useful to get key from raw data directly directly (like from a via, not from an event) +function getKey(network, data) +{ + var r = {slashes:true}; + r.host = network; + r.pathname = '/'; + idrHost(r, data); + return url.parse(url.format(r)); // make sure it's consistent +} + // normalize events a bit exports.processEvent = function(event, callback) { - if(!callback) callback = function(){}; + if(!callback) callback = function(err){if(err) console.error(err);}; // handle links as a special case as we're using them for post-process-deduplication if(event.type == 'link') return processLink(event, callback); @@ -118,12 +135,15 @@ function masterMaster(idr, data, callback) if(typeof data != 'object') return callback(); // logger.debug("MM\t"+url.format(idr)); var ref = url.format(idr); - var item = {keys:{}, refs:[], froms:{}, from:{}}; + var item = {keys:{}, refs:[], froms:{}, from:{}, responses:[], first:new Date().getTime(), last:new Date().getTime()}; item.ref = ref; item.refs.push(ref); item.keys[url.format(idr2key(idr, data))] = item.ref; item.me = isItMe(idr); - if(idr.protocol == 'tweet:') itemTwitter(item, data); + if(idr.protocol == 'tweet:'){ + if(data.user && data.text) itemTwitter(item, data); + if(data.related) itemTwitterRelated(item, data.related); + } if(idr.protocol == 'post:') itemFacebook(item, data); if(idr.protocol == 'checkin:') itemFoursquare(item, data); var dup; @@ -135,9 +155,17 @@ function masterMaster(idr, data, callback) }); }, function (err) { if(dup) item = itemMerge(dup, item); - dataStore.addItem(item, function(err, doc){ -// logger.debug("ADDED\t"+JSON.stringify(doc)); - callback(); + dataStore.addItem(item, function(err, item){ + if(err) return callback(err); + // all done processing, very useful to pull out some summary stats now! + // feels inefficient to re-query and double-write here, possible optimization, but should be logically safe this way + item.comments = item.ups = 0; + dataStore.getResponses({item:item.id}, function(r){ + if(r.type == "comment") item.comments++; + if(r.type == "up") item.ups++; + }, function(){ + dataStore.addItem(item, callback); // finally call back! + }); }); }); } @@ -145,7 +173,7 @@ function masterMaster(idr, data, callback) // intelligently merge two items together and return function itemMerge(older, newer) { - logger.debug("MERGE\t"+older.ref+'\t'+newer.ref); + logger.debug("MERGE\t"+JSON.stringify(older)+'\t'+JSON.stringify(newer)); if(newer.pri > older.pri) { // replace top level summary stuff if newer is deemed better older.ref = newer.ref; @@ -164,21 +192,70 @@ function itemMerge(older, newer) for(var i = 0; i < newer.refs.length; i++) refs[newer.refs[i]]=true; for(var i = 0; i < older.refs.length; i++) refs[older.refs[i]]=true; older.refs = Object.keys(refs); + older.responses = newer.responses; // older doesn't have any since they're stored separately return older; } +// merge existing with delete! +function itemMergeHard(a, b, cb) +{ + var item = itemMerge(a, b); + if(!item.responses) item.responses = []; + // gather old responses + dataStore.getResponses({item:b.id}, function(r){ item.responses.push(r); }, function(){ + // update existing item + dataStore.addItem(item, function(err, item){ + if(err || !item) return cb(err); + // now delete old fully! + dataStore.delItem(b.id, cb); + }); + }); +} + // when a processed link event comes in, check to see if it's a long url that could help us de-dup function processLink(event, callback) { - // look up event via/orig key and see if we've processed it yet, if not (for some reason) ignore - // if foursquare checkin and from a tweet, generate foursquare key and look for it - // if from instagram, generate instagram key and look for it - + var encounter; + try { + encounter = event.obj.data.encounters[0]; + }catch(E){ + return callback(E); + } + // first, look up event via/orig key and see if we've processed it yet, if not (for some reason) ignore + var key = url.format(getKey(encounter.network, encounter.via)); + dataStore.getItemByKey(key,function(err, item){ + if(err || !item) return callback(err); + var u = url.parse(encounter.link); + // if foursquare checkin and from a tweet, generate foursquare key and look for it + if(u.host == 'foursquare.com' && u.pathname.indexOf('/checkin/') > 0) + { + var id = path.basename(u.pathname); + var k2 = 'checkin://foursquare/#'+id; + dataStore.getItemByKey(k2,function(err, item2){ + if(err || !item2) return callback(); + // found a dup! + itemMergeHard(item, item2); + }) + + } + // if from instagram, generate instagram key and look for it + }); + // by here should have the item with the link, and the item for which the link is a target that is a duplicate - // merge them, merge any responses, delete the lesser + // merge them, merge any responses, delete the lesser item and it's responses callback(); } +// give a bunch of sane defaults +function newResponse(item, type) +{ + return { + type: type, + ref: item.ref, + from: {} + } +} + // extract info from a tweet function itemTwitter(item, tweet) { @@ -187,7 +264,7 @@ function itemTwitter(item, tweet) if(tweet.text) item.text = tweet.text; if(tweet.user) { - item.from.id = "twitter:"+tweet.user.id; + item.from.id = "contact://twitter/#"+tweet.user.screen_name; item.from.name = tweet.user.name; item.from.icon = tweet.user.profile_image_url; item.froms[item.from.id] = item.ref; @@ -197,11 +274,10 @@ function itemTwitter(item, tweet) { var hash = crypto.createHash('md5'); hash.update(item.text.substr(0,130)); // ignore trimming variations - item.keys['text:'+hash.digest('hex')] = item.ref; + item.keys['text:'+hash.digest('hex')] = item.ref; } // if it's tracking a reply, key it too if(tweet.in_reply_to_status_id_str) item.keys['tweet://twitter/#'+tweet.in_reply_to_status_id_str] = item.ref; - // TODO track links for link matching } // extract info from a facebook post @@ -212,7 +288,7 @@ function itemFacebook(item, post) item.last = post.updated_time * 1000; if(post.from) { - item.from.id = 'facebook:'+post.from.id; + item.from.id = 'contact://facebook/#'+post.from.id; item.from.name = post.from.name; item.from.icon = 'https://graph.facebook.com/' + post.from.id + '/picture'; item.froms[item.from.id] = item.ref; @@ -225,7 +301,31 @@ function itemFacebook(item, post) { var hash = crypto.createHash('md5'); hash.update(item.text.substr(0,130)); // ignore trimming variations - item.keys['text:'+hash.digest('hex')] = item.ref; + item.keys['text:'+hash.digest('hex')] = item.ref; + } + + // process responses! + if(post.comments && post.comments.data) + { + post.comments.data.forEach(function(comment){ + var resp = newResponse(item, "comment"); + resp.at = comment.created_time * 1000; + resp.text = comment.message; + resp.from.id = 'contact://facebook/#'+comment.from.id; + resp.from.name = comment.from.name; + resp.from.icon = 'https://graph.facebook.com/' + comment.from.id + '/picture'; + item.responses.push(resp); + }); + } + if(post.likes && post.likes.data) + { + post.likes.data.forEach(function(like){ + var resp = newResponse(item, "up"); + resp.from.id = 'contact://facebook/#'+like.id; + resp.from.name = like.name; + resp.from.icon = 'https://graph.facebook.com/' + like.id + '/picture'; + item.responses.push(resp); + }); } } @@ -237,8 +337,37 @@ function itemFoursquare(item, checkin) if(checkin.venue) item.text = "Checked in at " + checkin.venue.name; if(checkin.user) { - item.from.id = 'foursquare:'+checkin.user.id; + item.from.id = 'contact://foursquare/#'+checkin.user.id; item.from.name = checkin.user.firstName + " " + checkin.user.lastName; item.from.icon = checkin.user.photo; } + if(checkin.comments && checkin.comments.items) + { + checkin.comments.items.forEach(function(comment){ + // TODO: can't find examples of this! + }); + } + } + +// process twitter's post-tweet related data +function itemTwitterRelated(item, relateds) +{ + relateds.forEach(function(related){ + related.results.forEach(function(result){ + // ReTweet type by default is only the user object + var resp = newResponse(item, "up"); + var user = result; + if(related.resultType == "Tweet") { + resp.type = "comment"; + resp.text = result.value.text; + user = result.value.user; + resp.at = new Date(result.value.created_at).getTime() + } + resp.from.id = "contact://twitter/#"+user.screen_name; + resp.from.name = user.name; + resp.from.icon = user.profile_image_url; + item.responses.push(resp); + }); + }); +} \ No newline at end of file diff --git a/Collections/Timeline/dataStore.js b/Collections/Timeline/dataStore.js index 85f994359..8bb4cbe7a 100644 --- a/Collections/Timeline/dataStore.js +++ b/Collections/Timeline/dataStore.js @@ -8,14 +8,17 @@ */ var logger = require(__dirname + "/../../Common/node/logger").logger; var crypto = require("crypto"); +var async = require('async'); // in the future we'll probably need a visitCollection too var itemCol, respCol; exports.init = function(iCollection, rCollection) { itemCol = iCollection; -// itemCol.ensureIndex({"item":1},{unique:true},function() {}); + itemCol.ensureIndex({"id":1},{unique:true, background:true},function() {}); respCol = rCollection; + respCol.ensureIndex({"id":1},{unique:true, background:true},function() {}); + respCol.ensureIndex({"item":1},{background:true},function() {}); } exports.clear = function(callback) { @@ -42,6 +45,13 @@ exports.getItem = function(id, callback) { findWrap({id:id},{},itemCol,function(i){item=i},function(err){callback(err,item)}); } +// arg takes sort/limit/offset/find +exports.getResponses = function(arg, cbEach, cbDone) { + var f = (arg.item)?{item:arg.item}:{}; + delete arg.item; + findWrap(f,arg,respCol,cbEach,cbDone); +} + // arg takes sort/limit/offset/find exports.getItems = function(arg, cbEach, cbDone) { var f = {}; @@ -70,9 +80,38 @@ function findWrap(a,b,c,cbEach,cbDone){ // insert new (fully normalized) item, generate the id here and now exports.addItem = function(item, callback) { - var hash = crypto.createHash('md5'); - for(var i in item.keys) hash.update(i); - item.id = hash.digest('hex'); + if(!item.id) + { // first time an item comes in, make a unique id for it + var hash = crypto.createHash('md5'); + for(var i in item.keys) hash.update(i); + item.id = hash.digest('hex'); + } // logger.debug("addItem: "+JSON.stringify(item)); - itemCol.findAndModify({"id":item.id}, [['_id','asc']], {$set:item}, {safe:true, upsert:true, new: true}, callback); + var responses = item.responses; + if(responses) responses.forEach(function(r){ r.item = item.id; }); + delete item.responses; // store responses in their own table + delete item._id; // mongo is miss pissypants + itemCol.findAndModify({"id":item.id}, [['_id','asc']], {$set:item}, {safe:true, upsert:true, new: true}, function(err, doc){ + if(err) return callback(err); + if(responses) async.forEach(responses, exports.addResponse, function(err){callback(err, doc);}); // orig caller wants saved item back + }); +} + +// responses are unique by their contents +exports.addResponse = function(response, callback) { + var hash = crypto.createHash('md5'); + hash.update(JSON.stringify(response)); + response.id = hash.digest('hex'); + logger.debug("addResponse: "+JSON.stringify(response)); + delete response._id; // mongo is miss pissypants + respCol.findAndModify({"id":response.id}, [['_id','asc']], {$set:response}, {safe:true, upsert:true, new: true}, callback); +} + +// so, yeah, and that +exports.delItem = function(id, callback) { + if(!id || id.length < 10) return callback("no or invalid id to del: "+id); + itemCol.remove({id:id}, function(err){ + if(err) return callback(err); + respCol.remove({item:id}, callback); + }); } diff --git a/Collections/Timeline/fixtures/facebook.update b/Collections/Timeline/fixtures/facebook.update index fa8c0a03f..0887a2259 100644 --- a/Collections/Timeline/fixtures/facebook.update +++ b/Collections/Timeline/fixtures/facebook.update @@ -60,4 +60,3 @@ } } } -Ó \ No newline at end of file diff --git a/Collections/Timeline/fixtures/link b/Collections/Timeline/fixtures/link new file mode 100644 index 000000000..4fbec83a6 --- /dev/null +++ b/Collections/Timeline/fixtures/link @@ -0,0 +1,105 @@ +{ + "type": "link", + "via": "links", + "timestamp": 1315976215195, + "action": "new", + "obj": { + "type": "new", + "data": { + "_id": "4eb0f60f8cc09f2507e66440", + "at": 1320220061000, + "favicon": "http://t.co/favicon.ico", + "link": "http://t.co/QgLsBzHy", + "encounters": [{ + "_hash": "twitter:131638596414353400:http://t.co/QgLsBzHy", + "_id": "4eb0f60f8cc09f2507e66441", + "at": 1320220061000, + "from": "Markus Persson", + "fromID": 63485337, + "id": 131638596414353400, + "link": "https://foursquare.com/reecepacheco/checkin/4e703881d4c03fcb653e0251?s=a0huhS2FUrBtHfHd_oK2hvyvbME&ref=tw", + "network": "twitter", + "orig": "http://t.co/QgLsBzHy", + "text": "http://t.co/QgLsBzHy notch", + "via": { + "_id": "4e702f08bcb6f2f6dd06c849", + "in_reply_to_user_id_str": null, + "retweet_count": 0, + "in_reply_to_status_id": null, + "id_str": "113829905090879488", + "contributors": null, + "truncated": false, + "geo": null, + "coordinates": null, + "favorited": false, + "user": { + "default_profile": false, + "contributors_enabled": false, + "profile_use_background_image": true, + "protected": false, + "id_str": "633", + "time_zone": "Eastern Time (US & Canada)", + "profile_background_color": "1A1B1F", + "name": "danah boyd", + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme9/bg.gif", + "profile_image_url_https": "https://si0.twimg.com/profile_images/890666764/MeAtTalkWithHat-Sq_normal.jpg", + "default_profile_image": false, + "following": true, + "utc_offset": -18000, + "profile_image_url": "http://a0.twimg.com/profile_images/890666764/MeAtTalkWithHat-Sq_normal.jpg", + "description": "social media scholar, youth researcher & advocate | Microsoft Research, Harvard Berkman Center | zephoria@zephoria.org", + "show_all_inline_media": false, + "geo_enabled": false, + "friends_count": 734, + "profile_text_color": "666666", + "location": "Boston, MA", + "is_translator": false, + "profile_sidebar_fill_color": "252429", + "follow_request_sent": false, + "profile_background_tile": false, + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme9/bg.gif", + "url": "http://www.zephoria.org/thoughts/", + "statuses_count": 2471, + "followers_count": 50313, + "screen_name": "zephoria", + "notifications": false, + "profile_link_color": "2FC2EF", + "lang": "en", + "verified": false, + "favourites_count": 44, + "profile_sidebar_border_color": "181A1E", + "id": 633, + "listed_count": 5224, + "created_at": "Thu Jul 13 21:27:23 +0000 2006" + }, + "in_reply_to_user_id": null, + "in_reply_to_screen_name": null, + "possibly_sensitive": false, + "retweeted": false, + "source": "web", + "in_reply_to_status_id_str": null, + "entities": { + "user_mentions": [{ + "name": "Wil Wheaton", + "id_str": "1183041", + "indices": [108, 113], + "screen_name": "wilw", + "id": 1183041 + }], + "hashtags": [], + "urls": [{ + "indices": [116, 135], + "display_url": "amzn.to/qqvlo4", + "url": "http://t.co/zAN4G8S", + "expanded_url": "http://amzn.to/qqvlo4" + }] + }, + "id": 113829905090879490, + "place": null, + "text": "Just finished \"Ready Player One.\" YA dystopia mixed w/ 1980s geek nostalgia. Soooo much fun. (Audio read by @wilw ) http://t.co/zAN4G8S", + "created_at": "Wed Sep 14 04:22:18 +0000 2011" + } + }] + } + } +} diff --git a/Collections/Timeline/fixtures/twitter.related b/Collections/Timeline/fixtures/twitter.related index 405292557..830e0ec13 100644 --- a/Collections/Timeline/fixtures/twitter.related +++ b/Collections/Timeline/fixtures/twitter.related @@ -7,349 +7,1237 @@ "source": "twitter_related", "type": "new", "data": { - "_id": "4e703367bcb6f2f6dd06c867", - "id": "113792617996759041", + "id": "109553278831951872", "related": [{ "results": [{ - "contributors_enabled": false, - "profile_use_background_image": true, - "protected": false, - "id_str": "264083385", - "time_zone": "London", - "profile_background_color": "ffffff", - "name": "Barni", - "default_profile": false, - "profile_background_image_url": "http://a3.twimg.com/profile_background_images/307873995/Copy_of_Lion_Profile.jpg", - "profile_image_url_https": "https://si0.twimg.com/profile_images/1493346358/Copy_of_DSCN3225_normal.JPG", - "default_profile_image": false, - "following": false, - "utc_offset": 0, - "profile_image_url": "http://a1.twimg.com/profile_images/1493346358/Copy_of_DSCN3225_normal.JPG", - "description": " Whatever I did not know, I was not ashamed to inquire about, so I aquired knowledge.\r\n\r\n", - "show_all_inline_media": false, - "geo_enabled": false, - "friends_count": 80, - "profile_text_color": "7e7aff", - "location": "Global ", - "is_translator": false, - "profile_sidebar_fill_color": "000000", - "status": { - "in_reply_to_user_id_str": null, - "retweet_count": 6, - "in_reply_to_status_id": null, - "id_str": "113813170862301184", - "contributors": null, + "score": 1, + "annotations": { + "ConversationRole": "Fork" + }, + "kind": "Tweet", + "value": { + "id_str": "109570326760919041", + "in_reply_to_status_id": 109553278831951870, "truncated": false, - "retweeted_status": { - "in_reply_to_user_id_str": null, - "retweet_count": 6, - "in_reply_to_status_id": null, - "id_str": "113792617996759041", - "contributors": null, - "truncated": false, - "geo": null, - "coordinates": null, - "favorited": false, - "in_reply_to_user_id": null, - "in_reply_to_screen_name": null, - "possibly_sensitive": false, - "retweeted": false, - "source": "bitly", - "in_reply_to_status_id_str": null, - "id": 113792617996759040, - "place": null, - "text": "From March: @BarackObama explains how HuffPo would report the Emancipation Proclamation: http://t.co/kBD07wN", - "created_at": "Wed Sep 14 01:54:08 +0000 2011" + "user": { + "profile_sidebar_fill_color": "95E8EC", + "protected": false, + "id_str": "79202245", + "default_profile": false, + "notifications": null, + "profile_background_tile": false, + "screen_name": "Tiduilasa", + "name": "Tidu", + "listed_count": 1, + "location": "Brickman Road, S.Fburg. NY/USA", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": -21600, + "profile_link_color": "0099B9", + "description": "YOGI BHAGAVATA-DHARMA\r\n_______________________", + "profile_sidebar_border_color": "5ED4DC", + "url": "http://www.siddhayoga.org/gurumayi-chidvilasananda", + "time_zone": "Central Time (US & Canada)", + "default_profile_image": false, + "statuses_count": 85, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 12, + "friends_count": 5, + "profile_background_color": "0099B9", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme4/bg.gif", + "created_at": "Fri Oct 02 15:01:45 +0000 2009", + "followers_count": 3, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme4/bg.gif", + "id": 79202245, + "profile_text_color": "3C3940", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1522182333/aum-message-artwork_1__normal.jpg", + "profile_image_url": "http://a2.twimg.com/profile_images/1522182333/aum-message-artwork_1__normal.jpg" }, - "geo": null, - "coordinates": null, "favorited": false, - "in_reply_to_user_id": null, - "in_reply_to_screen_name": null, "possibly_sensitive": false, + "in_reply_to_status_id_str": "109553278831951872", + "geo": null, + "in_reply_to_screen_name": "DalaiLama", + "in_reply_to_user_id_str": "20609518", + "coordinates": null, + "in_reply_to_user_id": 20609518, + "source": "web", + "created_at": "Fri Sep 02 10:16:16 +0000 2011", + "contributors": null, "retweeted": false, + "retweet_count": 0, + "id": 109570326760919040, + "place": null, + "text": "@DalaiLama ~ Fowers to Mr, Dalai; (was called \"Delair\" - my papy)\nhttp://t.co/MXYIaaS" + } + }, + { + "score": 1, + "annotations": { + "ConversationRole": "Fork" + }, + "kind": "Tweet", + "value": { + "id_str": "109569700891082752", + "in_reply_to_status_id": 109553278831951870, + "truncated": false, + "user": { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "114698131", + "default_profile": false, + "notifications": null, + "profile_background_tile": false, + "screen_name": "jeshvik77", + "name": "rajesh", + "listed_count": 1, + "location": "KNR.HYD.GHY..", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": -36000, + "profile_link_color": "0084B4", + "description": "Hypo r Hyper n still trying for Normo, 'll bit narcissistic, as of nw Hurt-Locker, pasion n EM,belivin fnds r lyf...", + "profile_sidebar_border_color": "C0DEED", + "url": null, + "time_zone": "Hawaii", + "default_profile_image": false, + "statuses_count": 2136, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 4, + "friends_count": 37, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a2.twimg.com/profile_background_images/308608018/PIX_094-58.jpg", + "created_at": "Tue Feb 16 09:30:33 +0000 2010", + "followers_count": 49, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/308608018/PIX_094-58.jpg", + "id": 114698131, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1452584974/uu_normal.jpg", + "profile_image_url": "http://a1.twimg.com/profile_images/1452584974/uu_normal.jpg" + }, + "favorited": false, + "in_reply_to_status_id_str": "109553278831951872", + "geo": null, + "in_reply_to_screen_name": "DalaiLama", + "in_reply_to_user_id_str": "20609518", + "coordinates": null, + "in_reply_to_user_id": 20609518, "source": "web", - "in_reply_to_status_id_str": null, - "id": 113813170862301180, + "created_at": "Fri Sep 02 10:13:47 +0000 2011", + "contributors": null, + "retweeted": false, + "retweet_count": 0, + "id": 109569700891082750, "place": null, - "text": "RT @anildash: From March: @BarackObama explains how HuffPo would report the Emancipation Proclamation: http://t.co/kBD07wN", - "created_at": "Wed Sep 14 03:15:49 +0000 2011" + "text": "@DalaiLama \"@jeshvik77 The practice of altruism is the authentic way of living a human life; it is not limited only to the religious\"" + } + }, + { + "score": 1, + "annotations": { + "ConversationRole": "Fork" }, - "follow_request_sent": false, - "profile_background_tile": true, - "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/307873995/Copy_of_Lion_Profile.jpg", - "url": "http://iphotosbyab.tumblr.com/", - "statuses_count": 428, - "followers_count": 50, - "screen_name": "iJauntygrapher", - "notifications": false, - "profile_link_color": "ff802b", - "lang": "en", - "verified": false, - "favourites_count": 5, - "profile_sidebar_border_color": "ffffff", - "id": 264083385, - "listed_count": 1, - "created_at": "Fri Mar 11 09:02:42 +0000 2011" + "kind": "Tweet", + "value": { + "id_str": "109569691311288321", + "in_reply_to_status_id": 109553278831951870, + "truncated": false, + "user": { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "351589243", + "default_profile": true, + "notifications": null, + "profile_background_tile": false, + "screen_name": "nukmi", + "name": "nukmi", + "listed_count": 0, + "location": "", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": null, + "profile_link_color": "0084B4", + "description": "human being from south east london, a real tyrant :)", + "profile_sidebar_border_color": "C0DEED", + "url": null, + "time_zone": null, + "default_profile_image": true, + "statuses_count": 24, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 0, + "friends_count": 11, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Tue Aug 09 13:26:35 +0000 2011", + "followers_count": 3, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 351589243, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/sticky/default_profile_images/default_profile_5_normal.png", + "profile_image_url": "http://a0.twimg.com/sticky/default_profile_images/default_profile_5_normal.png" + }, + "favorited": false, + "in_reply_to_status_id_str": "109553278831951872", + "geo": null, + "in_reply_to_screen_name": "DalaiLama", + "in_reply_to_user_id_str": "20609518", + "coordinates": null, + "in_reply_to_user_id": 20609518, + "source": "web", + "created_at": "Fri Sep 02 10:13:44 +0000 2011", + "contributors": null, + "retweeted": false, + "retweet_count": 0, + "id": 109569691311288320, + "place": null, + "text": "@DalaiLama does HHDL have any thoughts to share on the role of cinema & film in creating a moral/altruistic society?" + } }, { - "contributors_enabled": false, - "profile_use_background_image": true, - "protected": false, - "id_str": "25013495", - "default_profile": false, - "time_zone": "Pacific Time (US & Canada)", - "profile_background_color": "030302", - "name": "Max Benavidez, PhD", - "profile_background_image_url": "http://a2.twimg.com/profile_background_images/237835820/xad508adf00907e14e42fd292e1a7ff0.jpg", - "profile_image_url_https": "https://si0.twimg.com/profile_images/996977116/Benavidez_Photo_normal.jpg", - "default_profile_image": false, - "following": false, - "utc_offset": -28800, - "profile_image_url": "http://a0.twimg.com/profile_images/996977116/Benavidez_Photo_normal.jpg", - "description": "PhD, news analyst, new media consultant, author", - "show_all_inline_media": false, - "geo_enabled": false, - "friends_count": 1886, - "profile_text_color": "786a55", - "location": "Los Angeles, California", - "is_translator": false, - "profile_sidebar_fill_color": "12291f", - "status": { - "in_reply_to_user_id_str": null, - "retweet_count": 38, - "in_reply_to_status_id": null, - "id_str": "113799556541587457", + "score": 1, + "annotations": { + "ConversationRole": "Fork" + }, + "kind": "Tweet", + "value": { + "id_str": "109568108846530560", + "in_reply_to_status_id": 109553278831951870, + "truncated": false, + "user": { + "profile_sidebar_fill_color": "efefef", + "protected": false, + "id_str": "96293863", + "default_profile": false, + "notifications": null, + "profile_background_tile": true, + "screen_name": "SofistaPrateado", + "name": "Renato(Rewah Hameth)", + "listed_count": 1, + "location": "Ilha da Rainha da Morte - RJ", + "show_all_inline_media": true, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": -10800, + "profile_link_color": "009999", + "description": "Bancário, gamer, lutador de Kung-fu, escritor de contos, tarólogo, acupuntor, leitor, futurista e pseudo-humorista. Assim sou/estou eu.", + "profile_sidebar_border_color": "eeeeee", + "url": "http://sofistaprateado.tumblr.com", + "time_zone": "Brasilia", + "default_profile_image": false, + "statuses_count": 11251, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 76, + "friends_count": 123, + "profile_background_color": "131516", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme14/bg.gif", + "created_at": "Sat Dec 12 06:54:04 +0000 2009", + "followers_count": 123, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme14/bg.gif", + "id": 96293863, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1514569915/aquiles_normal.JPG", + "profile_image_url": "http://a3.twimg.com/profile_images/1514569915/aquiles_normal.JPG" + }, + "favorited": false, + "in_reply_to_status_id_str": "109553278831951872", + "geo": null, + "in_reply_to_screen_name": "DalaiLama", + "in_reply_to_user_id_str": "20609518", + "coordinates": null, + "in_reply_to_user_id": 20609518, + "source": "web", + "created_at": "Fri Sep 02 10:07:27 +0000 2011", "contributors": null, + "retweeted": false, + "retweet_count": 0, + "id": 109568108846530560, + "place": null, + "text": "A prática de altruísmo é o modo autêntico de viver uma vida humana; não é limitada apenas aos religiosos. (via @DalaiLama) #fb" + } + }, + { + "score": 1, + "annotations": { + "ConversationRole": "Fork" + }, + "kind": "Tweet", + "value": { + "id_str": "109567960246530048", + "in_reply_to_status_id": 109553278831951870, "truncated": false, - "retweeted_status": { - "in_reply_to_user_id_str": null, - "retweet_count": 38, - "in_reply_to_status_id": null, - "id_str": "113799495749337089", - "contributors": null, - "truncated": false, - "geo": null, - "coordinates": null, - "favorited": false, - "in_reply_to_user_id": null, - "in_reply_to_screen_name": null, - "possibly_sensitive": false, - "retweeted": false, - "source": "SocialFlow", - "in_reply_to_status_id_str": null, - "id": 113799495749337090, - "place": null, - "text": "What happens to fornicating passengers who get caught? http://t.co/MZkK67k #mysteries #explained", - "created_at": "Wed Sep 14 02:21:28 +0000 2011" + "user": { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "76316278", + "default_profile": true, + "notifications": null, + "profile_background_tile": false, + "screen_name": "DrMradul", + "name": "Mradul Kaushik", + "listed_count": 0, + "location": null, + "show_all_inline_media": false, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": null, + "profile_link_color": "0084B4", + "description": null, + "profile_sidebar_border_color": "C0DEED", + "url": null, + "time_zone": null, + "default_profile_image": true, + "statuses_count": 21, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 1, + "friends_count": 26, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Tue Sep 22 12:11:22 +0000 2009", + "followers_count": 9, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 76316278, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/sticky/default_profile_images/default_profile_3_normal.png", + "profile_image_url": "http://a2.twimg.com/sticky/default_profile_images/default_profile_3_normal.png" }, + "favorited": false, + "in_reply_to_status_id_str": "109553278831951872", "geo": null, + "in_reply_to_screen_name": "DalaiLama", + "in_reply_to_user_id_str": "20609518", "coordinates": null, + "in_reply_to_user_id": 20609518, + "source": "Twitter for BlackBerry®", + "created_at": "Fri Sep 02 10:06:52 +0000 2011", + "contributors": null, + "retweeted": false, + "retweet_count": 0, + "id": 109567960246530050, + "place": null, + "text": "@DalaiLama whatever is to be followed righteously is the true religion" + } + }, + { + "score": 1, + "annotations": { + "ConversationRole": "Fork" + }, + "kind": "Tweet", + "value": { + "id_str": "109567497652543488", + "in_reply_to_status_id": 109553278831951870, + "truncated": false, + "user": { + "profile_sidebar_fill_color": "7AC3EE", + "protected": false, + "id_str": "186321654", + "default_profile": false, + "notifications": null, + "profile_background_tile": true, + "screen_name": "antdigdogdagity", + "name": "Antony Blake Munro", + "listed_count": 0, + "location": "New Zealand, Auckland", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": 43200, + "profile_link_color": "FF0000", + "description": "I like magic, witchcraft, voodoo, buddhism, any thing goes, entertain me.", + "profile_sidebar_border_color": "65B0DA", + "url": "http://www.facebook.com/antony.munro", + "time_zone": "Auckland", + "default_profile_image": false, + "statuses_count": 79, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 1, + "friends_count": 173, + "profile_background_color": "642D8B", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme10/bg.gif", + "created_at": "Fri Sep 03 03:51:49 +0000 2010", + "followers_count": 26, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme10/bg.gif", + "id": 186321654, + "profile_text_color": "3D1957", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1509387282/antdog_normal.jpg", + "profile_image_url": "http://a0.twimg.com/profile_images/1509387282/antdog_normal.jpg" + }, "favorited": false, - "in_reply_to_user_id": null, - "in_reply_to_screen_name": null, - "possibly_sensitive": false, + "in_reply_to_status_id_str": "109553278831951872", + "geo": null, + "in_reply_to_screen_name": "DalaiLama", + "in_reply_to_user_id_str": "20609518", + "coordinates": null, + "in_reply_to_user_id": 20609518, + "source": "web", + "created_at": "Fri Sep 02 10:05:01 +0000 2011", + "contributors": null, "retweeted": false, + "retweet_count": 0, + "id": 109567497652543490, + "place": null, + "text": "@DalaiLama altruism is very interesting ill think about that." + } + }, + { + "score": 1, + "annotations": { + "ConversationRole": "Fork" + }, + "kind": "Tweet", + "value": { + "id_str": "109566564193402880", + "in_reply_to_status_id": 109553278831951870, + "truncated": false, + "user": { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "261139101", + "default_profile": true, + "notifications": null, + "profile_background_tile": false, + "screen_name": "JohnGeoMatthews", + "name": "John George Matthews", + "listed_count": 0, + "location": "Margate", + "show_all_inline_media": true, + "contributors_enabled": false, + "following": null, + "geo_enabled": true, + "utc_offset": 0, + "profile_link_color": "0084B4", + "description": "Morris Minor wedding car driver, www.johngmatthews.co.uk, car restorer, guitarist a band:- http://www.mentalblock.info, guitar teacher, mobile recordist. ", + "profile_sidebar_border_color": "C0DEED", + "url": "http://www.johngmatthews.co.uk/", + "time_zone": "London", + "default_profile_image": false, + "statuses_count": 314, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 0, + "friends_count": 86, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Sat Mar 05 09:37:20 +0000 2011", + "followers_count": 20, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 261139101, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1262832286/Twitter_pic_normal.jpg", + "profile_image_url": "http://a1.twimg.com/profile_images/1262832286/Twitter_pic_normal.jpg" + }, + "favorited": false, + "in_reply_to_status_id_str": "109553278831951872", + "geo": null, + "in_reply_to_screen_name": "DalaiLama", + "in_reply_to_user_id_str": "20609518", + "coordinates": null, + "in_reply_to_user_id": 20609518, "source": "web", - "in_reply_to_status_id_str": null, - "id": 113799556541587460, + "created_at": "Fri Sep 02 10:01:19 +0000 2011", + "contributors": null, + "retweeted": false, + "retweet_count": 0, + "id": 109566564193402880, "place": null, - "text": "RT @Slate: What happens to fornicating passengers who get caught? http://t.co/MZkK67k #mysteries #explained", - "created_at": "Wed Sep 14 02:21:43 +0000 2011" + "text": "@DalaiLama- altruism- If only more people would live by this, then the world would be a much better place. altruism = all true ism" + } + }, + { + "score": 1, + "annotations": { + "ConversationRole": "Fork" }, - "follow_request_sent": false, + "kind": "Tweet", + "value": { + "id_str": "109566508916670464", + "in_reply_to_status_id": 109553278831951870, + "truncated": false, + "user": { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "362947094", + "default_profile": true, + "notifications": null, + "profile_background_tile": false, + "screen_name": "KARANHIRWAT", + "name": "KARANHIRWAT", + "listed_count": 0, + "location": "", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": null, + "profile_link_color": "0084B4", + "description": "", + "profile_sidebar_border_color": "C0DEED", + "url": null, + "time_zone": null, + "default_profile_image": false, + "statuses_count": 90, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 1, + "friends_count": 76, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Sat Aug 27 07:41:23 +0000 2011", + "followers_count": 16, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 362947094, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1515548535/karan_new_pic_normal.jpg", + "profile_image_url": "http://a0.twimg.com/profile_images/1515548535/karan_new_pic_normal.jpg" + }, + "favorited": false, + "in_reply_to_status_id_str": "109553278831951872", + "geo": null, + "in_reply_to_screen_name": "DalaiLama", + "in_reply_to_user_id_str": "20609518", + "coordinates": null, + "in_reply_to_user_id": 20609518, + "source": "web", + "created_at": "Fri Sep 02 10:01:06 +0000 2011", + "contributors": null, + "retweeted": false, + "retweet_count": 0, + "id": 109566508916670460, + "place": null, + "text": "@DalaiLama The tongue like a sharp knife... Kills without drawing blood" + } + }], + "resultType": "Tweet", + "groupName": "TweetsWithConversation", + "annotations": { + "FromUser": "DalaiLama" + }, + "score": 1 + }, + { + "results": [{ + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "348062004", + "notifications": null, "profile_background_tile": false, - "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/237835820/xad508adf00907e14e42fd292e1a7ff0.jpg", - "url": "http://theamericanshow.com/", - "statuses_count": 1749, - "followers_count": 685, - "screen_name": "MaxBenavidez", - "notifications": false, - "profile_link_color": "547b61", - "lang": "en", + "screen_name": "seshdaherbaliss", + "name": "seshi", + "listed_count": 0, + "location": "tema", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": null, + "profile_link_color": "0084B4", + "description": "last killer", + "profile_sidebar_border_color": "C0DEED", + "url": null, + "default_profile": true, + "time_zone": null, + "default_profile_image": true, + "statuses_count": 60, + "profile_use_background_image": true, "verified": false, - "favourites_count": 1, - "profile_sidebar_border_color": "3c5449", - "id": 25013495, - "listed_count": 15, - "created_at": "Wed Mar 18 02:45:11 +0000 2009" + "favourites_count": 0, + "friends_count": 13, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Wed Aug 03 20:26:46 +0000 2011", + "followers_count": 7, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 348062004, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/sticky/default_profile_images/default_profile_3_normal.png", + "profile_image_url": "http://a2.twimg.com/sticky/default_profile_images/default_profile_3_normal.png" }, { - "verified": false, - "favourites_count": 12481, - "profile_sidebar_border_color": "dfe394", + "profile_sidebar_fill_color": "99CC33", "protected": false, - "listed_count": 46, - "name": "Dinah Sanders", - "default_profile": false, + "id_str": "107930046", + "notifications": false, + "profile_background_tile": true, + "screen_name": "paolaandrea25", + "name": "Paola Andrea☆", + "listed_count": 0, + "location": "Bogota", + "show_all_inline_media": false, "contributors_enabled": false, - "profile_use_background_image": true, - "id_str": "6722", "following": false, - "time_zone": "Pacific Time (US & Canada)", - "utc_offset": -28800, - "profile_background_color": "4c8018", - "description": "Writer. Productivity and happiness coach. Drinker of classic cocktails. Pre-blogs blogger.", - "profile_background_image_url": "http://a0.twimg.com/profile_background_images/303847553/beedleumbum_edgewoodmeadow.jpg", - "profile_image_url_https": "https://si0.twimg.com/profile_images/1002780733/201006dpsbytecgirl_lookup_normal.png", - "location": "San Francisco, CA, USA", + "geo_enabled": true, + "utc_offset": -18000, + "profile_link_color": "D02B55", + "description": "Adm. de Empresas, Bogotana!!! desarrollando mi proyecto SHAROVA DAY SPA, Me gusta Soñar en cada momento... Dios siempre cumple mis sueños !) 100% Feliz. ", + "profile_sidebar_border_color": "829D5E", + "url": null, + "time_zone": "Bogota", "default_profile_image": false, - "profile_image_url": "http://a1.twimg.com/profile_images/1002780733/201006dpsbytecgirl_lookup_normal.png", + "statuses_count": 74, + "profile_use_background_image": true, + "default_profile": false, + "verified": false, + "favourites_count": 5, + "friends_count": 278, + "profile_background_color": "352726", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/profile_background_images/294286892/kjhkjk.jpg", + "created_at": "Sun Jan 24 06:39:53 +0000 2010", + "followers_count": 75, + "follow_request_sent": false, + "lang": "es", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/294286892/kjhkjk.jpg", + "id": 107930046, + "profile_text_color": "3E4415", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1431039138/IMG01029-20100603-1450_normal.jpg", + "profile_image_url": "http://a0.twimg.com/profile_images/1431039138/IMG01029-20100603-1450_normal.jpg" + }, + { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "273380258", + "notifications": false, + "profile_background_tile": false, + "screen_name": "Positivestarts", + "name": "Positive Starts", + "listed_count": 0, + "location": null, "show_all_inline_media": true, + "contributors_enabled": false, + "following": false, "geo_enabled": false, - "friends_count": 95, - "profile_text_color": "333333", - "url": "http://www.metagrrrl.com", + "utc_offset": -18000, + "profile_link_color": "0084B4", + "description": "Life is a short ride. Negativity can lead to stress. Reflect on the positive to drive away the negative.", + "profile_sidebar_border_color": "C0DEED", + "url": null, + "time_zone": "Quito", + "default_profile_image": false, + "statuses_count": 395, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 18, + "friends_count": 15, + "profile_background_color": "C0DEED", "is_translator": false, - "profile_sidebar_fill_color": "bbf08d", - "screen_name": "MetaGrrrl", + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Mon Mar 28 11:29:47 +0000 2011", + "followers_count": 28, + "default_profile": true, "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 273380258, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1290603834/image_normal.jpg", + "profile_image_url": "http://a2.twimg.com/profile_images/1290603834/image_normal.jpg" + }, + { + "profile_sidebar_fill_color": "5bda1b", + "protected": false, + "id_str": "36706744", "notifications": false, "profile_background_tile": false, - "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/303847553/beedleumbum_edgewoodmeadow.jpg", + "screen_name": "Herrardo", + "name": "Gerardo Torres", + "listed_count": 15, + "location": "Mexico City", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": false, + "utc_offset": -21600, + "profile_link_color": "1203fc", + "description": "I love playin' guitar! Or bass! Having fun! To party! Read, watch movies, learn something interesting!", + "profile_sidebar_border_color": "000000", + "url": "http://www.facebook.com/Gerrytn", + "time_zone": "Mexico City", + "default_profile_image": false, + "statuses_count": 4483, + "profile_use_background_image": true, + "default_profile": false, + "verified": false, + "favourites_count": 17, + "friends_count": 266, + "profile_background_color": "25a290", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/profile_background_images/13762946/DSC00295.JPG", + "created_at": "Thu Apr 30 16:38:47 +0000 2009", + "followers_count": 156, + "follow_request_sent": false, "lang": "en", - "statuses_count": 7093, - "followers_count": 965, - "id": 6722, - "created_at": "Sat Sep 23 18:20:44 +0000 2006", - "profile_link_color": "0084b4" + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/13762946/DSC00295.JPG", + "id": 36706744, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1303959010/yop_normal.jpg", + "profile_image_url": "http://a3.twimg.com/profile_images/1303959010/yop_normal.jpg" }, { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "199273449", + "notifications": false, + "profile_background_tile": false, + "screen_name": "sitarabeling", + "name": "sita rabeling", + "listed_count": 0, + "location": "Netherlands", + "show_all_inline_media": false, "contributors_enabled": false, + "following": false, + "geo_enabled": false, + "utc_offset": 3600, + "profile_link_color": "0084B4", + "description": "", + "profile_sidebar_border_color": "C0DEED", + "url": null, + "time_zone": "Amsterdam", + "default_profile_image": false, + "statuses_count": 120, "profile_use_background_image": true, + "default_profile": true, + "verified": false, + "favourites_count": 0, + "friends_count": 47, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Wed Oct 06 13:17:11 +0000 2010", + "followers_count": 11, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 199273449, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1379852945/IMG_0195_normal.jpg", + "profile_image_url": "http://a2.twimg.com/profile_images/1379852945/IMG_0195_normal.jpg" + }, + { + "profile_sidebar_fill_color": "E6F6F9", "protected": false, - "id_str": "14367955", - "time_zone": "Pacific Time (US & Canada)", - "profile_background_color": "333333", - "name": "Kevin Cantù", - "default_profile": false, - "profile_background_image_url": "http://a3.twimg.com/profile_background_images/232144494/3517286539_06a527be68_o.jpg", - "profile_image_url_https": "https://si0.twimg.com/profile_images/202909645/2430137243_0223b18eab_o_normal.jpg", + "id_str": "47444972", + "notifications": false, + "profile_background_tile": false, + "screen_name": "MooreofMary", + "name": "Mary Moore", + "listed_count": 8, + "location": "Oklahoma", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": false, + "utc_offset": -21600, + "profile_link_color": "CC3366", + "description": "Morning Radio Show Cohost. Sports Reporter. Emcee. Workaholic. ", + "profile_sidebar_border_color": "DBE9ED", + "url": "http://www.facebook.com/pages/Mary-MooreTULSA/160802983954993", + "time_zone": "Central Time (US & Canada)", "default_profile_image": false, + "statuses_count": 3331, + "profile_use_background_image": false, + "verified": false, + "favourites_count": 3, + "friends_count": 245, + "profile_background_color": "DBE9ED", + "is_translator": false, + "default_profile": false, + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme17/bg.gif", + "created_at": "Mon Jun 15 21:07:55 +0000 2009", + "followers_count": 455, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme17/bg.gif", + "id": 47444972, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1505124747/image_normal.jpg", + "profile_image_url": "http://a2.twimg.com/profile_images/1505124747/image_normal.jpg" + }, + { + "profile_sidebar_fill_color": "252429", + "protected": false, + "id_str": "28393220", + "notifications": false, + "profile_background_tile": false, + "screen_name": "picyadri", + "name": "Adrienn Szikszay", + "listed_count": 0, + "location": "Hungary", + "show_all_inline_media": true, + "contributors_enabled": false, "following": false, - "utc_offset": -28800, - "profile_image_url": "http://a3.twimg.com/profile_images/202909645/2430137243_0223b18eab_o_normal.jpg", - "description": "I occasionally say interesting things!", + "geo_enabled": false, + "utc_offset": 3600, + "profile_link_color": "edc345", + "description": "Optimistic. Loves a bunch of stuff. Always smiling.", + "profile_sidebar_border_color": "f1bd0e", + "url": null, + "time_zone": "Budapest", + "default_profile_image": false, + "statuses_count": 65, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 0, + "friends_count": 114, + "profile_background_color": "1A1B1F", + "is_translator": false, + "default_profile": false, + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme9/bg.gif", + "created_at": "Thu Apr 02 18:43:08 +0000 2009", + "followers_count": 53, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme9/bg.gif", + "id": 28393220, + "profile_text_color": "aba6a6", + "profile_image_url_https": "https://si0.twimg.com/profile_images/121702135/adrisapiban_normal.jpg", + "profile_image_url": "http://a1.twimg.com/profile_images/121702135/adrisapiban_normal.jpg" + }, + { + "profile_sidebar_fill_color": "bd77bd", + "protected": false, + "id_str": "97671153", + "notifications": false, + "profile_background_tile": true, + "screen_name": "purple9dragon", + "name": "purple dragon", + "listed_count": 2, + "location": "天空の城ラプタ", "show_all_inline_media": false, - "geo_enabled": true, - "friends_count": 712, - "profile_text_color": "333333", - "location": "Santa Barbara, California", + "contributors_enabled": false, + "following": false, + "geo_enabled": false, + "utc_offset": 32400, + "profile_link_color": "0055ff", + "description": "民間人 フォローミー、フォローユー 趣味は盆栽", + "profile_sidebar_border_color": "6cd966", + "url": null, + "time_zone": "Osaka", + "default_profile_image": false, + "statuses_count": 1556, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 6, + "friends_count": 112, + "profile_background_color": "000000", "is_translator": false, - "profile_sidebar_fill_color": "f6ffd1", - "status": { - "in_reply_to_user_id_str": "47951511", - "retweet_count": 0, - "in_reply_to_status_id": 113834332413624320, - "id_str": "113836727860346881", - "contributors": null, - "truncated": false, - "geo": null, - "coordinates": null, - "favorited": false, - "in_reply_to_user_id": 47951511, - "in_reply_to_screen_name": "zunguzungu", - "retweeted": false, - "source": "web", - "in_reply_to_status_id_str": "113834332413624320", - "id": 113836727860346880, - "place": { - "name": "Santa Barbara", - "attributes": {}, - "full_name": "Santa Barbara, CA", - "bounding_box": { - "type": "Polygon", - "coordinates": [[[ - 119.859784, 34.336029], [ - 119.639922, 34.336029], [ - 119.639922, 34.460896], [ - 119.859784, 34.460896]]] - }, - "place_type": "city", - "url": "http://api.twitter.com/1/geo/id/f6ebc676e5cde864.json", - "country_code": "US", - "country": "United States", - "id": "f6ebc676e5cde864" - }, - "text": "@zunguzungu Atta is the Bolshevik to Bouazizi's serfdom: they are at two corners of a larger problem", - "created_at": "Wed Sep 14 04:49:25 +0000 2011" - }, + "default_profile": false, + "profile_background_image_url": "http://a1.twimg.com/profile_background_images/63728857/the-eye-dl.jpg", + "created_at": "Fri Dec 18 13:55:27 +0000 2009", + "followers_count": 62, "follow_request_sent": false, + "lang": "ja", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/63728857/the-eye-dl.jpg", + "id": 97671153, + "profile_text_color": "000000", + "profile_image_url_https": "https://si0.twimg.com/profile_images/938357687/exit_normal.jpg", + "profile_image_url": "http://a1.twimg.com/profile_images/938357687/exit_normal.jpg" + }, + { + "profile_sidebar_fill_color": "4f468c", + "protected": false, + "id_str": "23290095", + "notifications": false, "profile_background_tile": true, - "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/232144494/3517286539_06a527be68_o.jpg", - "url": "http://kevincantu.org", - "statuses_count": 12240, - "followers_count": 316, - "screen_name": "killerswan", + "screen_name": "RachelCappucci", + "name": "Rachel Cappucci", + "listed_count": 2, + "location": "New York City", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": false, + "utc_offset": -18000, + "profile_link_color": "d911cf", + "description": " The person who risks nothing, does nothing, has nothing, is nothing. ", + "profile_sidebar_border_color": "d678d6", + "url": null, + "time_zone": "Eastern Time (US & Canada)", + "default_profile_image": false, + "statuses_count": 1810, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 5, + "friends_count": 335, + "profile_background_color": "3e416b", + "is_translator": false, + "default_profile": false, + "profile_background_image_url": "http://a1.twimg.com/profile_background_images/87659392/6040_143426809917_610109917_3324938_5333854_n.jpg", + "created_at": "Sun Mar 08 08:30:14 +0000 2009", + "followers_count": 121, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/87659392/6040_143426809917_610109917_3324938_5333854_n.jpg", + "id": 23290095, + "profile_text_color": "ed9aed", + "profile_image_url_https": "https://si0.twimg.com/profile_images/391846681/twitter_this_normal.jpg", + "profile_image_url": "http://a1.twimg.com/profile_images/391846681/twitter_this_normal.jpg" + }, + { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "209990451", "notifications": false, - "profile_link_color": "98b302", + "profile_background_tile": false, + "screen_name": "penfoldholes", + "name": "Richard Barker", + "listed_count": 0, + "location": "Shefford", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": true, + "utc_offset": null, + "profile_link_color": "0084B4", + "description": "kidney cancer survivor - former footballer and cricketer - wine , dog and music lover.", + "profile_sidebar_border_color": "C0DEED", + "url": null, + "time_zone": null, + "default_profile_image": false, + "statuses_count": 1608, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 7, + "friends_count": 66, + "profile_background_color": "C0DEED", + "is_translator": false, + "default_profile": true, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Sat Oct 30 09:40:15 +0000 2010", + "followers_count": 52, + "follow_request_sent": false, "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 209990451, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1157214189/Richard_normal.jpg", + "profile_image_url": "http://a0.twimg.com/profile_images/1157214189/Richard_normal.jpg" + }, + { + "profile_sidebar_fill_color": "F6F6F6", + "protected": false, + "id_str": "90737592", + "notifications": false, + "profile_background_tile": false, + "screen_name": "coti_rodriguez", + "name": "Constanza Rodriguez", + "listed_count": 0, + "location": "Monte Hermoso", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": true, + "utc_offset": -10800, + "profile_link_color": "038543", + "description": "", + "profile_sidebar_border_color": "EEEEEE", + "url": null, + "time_zone": "Buenos Aires", + "default_profile_image": false, + "default_profile": false, + "statuses_count": 276, + "profile_use_background_image": true, "verified": false, - "favourites_count": 7951, - "profile_sidebar_border_color": "ffffad", - "id": 14367955, - "listed_count": 24, - "created_at": "Sat Apr 12 09:25:23 +0000 2008" + "favourites_count": 2, + "friends_count": 153, + "profile_background_color": "ACDED6", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme18/bg.gif", + "created_at": "Tue Nov 17 22:36:14 +0000 2009", + "followers_count": 48, + "follow_request_sent": false, + "lang": "es", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme18/bg.gif", + "id": 90737592, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1098075067/DSCN1664_normal.JPG", + "profile_image_url": "http://a0.twimg.com/profile_images/1098075067/DSCN1664_normal.JPG" }, { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "111380497", + "notifications": false, + "profile_background_tile": false, + "screen_name": "PROFAMS", + "name": "ANTONIO MARCOS", + "listed_count": 0, + "location": "São Paulo", + "show_all_inline_media": false, "contributors_enabled": false, + "following": false, + "geo_enabled": true, + "utc_offset": -10800, + "profile_link_color": "0084B4", + "description": "Gerente Nacional de Marketing e Professor nas áreas de adminisração e marketing.", + "profile_sidebar_border_color": "C0DEED", + "url": null, + "time_zone": "Brasilia", + "default_profile_image": false, + "default_profile": true, + "statuses_count": 124, "profile_use_background_image": true, + "verified": false, + "favourites_count": 5, + "friends_count": 44, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Thu Feb 04 18:34:41 +0000 2010", + "followers_count": 45, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 111380497, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1309327114/AMS_3_normal.JPG", + "profile_image_url": "http://a2.twimg.com/profile_images/1309327114/AMS_3_normal.JPG" + }, + { + "profile_sidebar_fill_color": "333333", "protected": false, - "id_str": "6299142", - "time_zone": "New Delhi", - "profile_background_color": "8B542B", - "name": "Firas", - "profile_background_image_url": "http://a1.twimg.com/profile_background_images/88254516/palmetto_t.gif", - "profile_image_url_https": "https://si0.twimg.com/profile_images/1314697710/cropprofile1_normal.png", + "id_str": "19216042", + "notifications": false, + "profile_background_tile": true, + "screen_name": "anaverena", + "name": "Ana Verena Menezes", + "listed_count": 14, + "location": "Salvador,Bahia,Brasil", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": true, + "utc_offset": -10800, + "profile_link_color": "b36868", + "description": "all my colours turn to clouds.", + "profile_sidebar_border_color": "b6b8b6", + "url": "http://www.anaverena.tumblr.com", + "time_zone": "Brasilia", "default_profile_image": false, "default_profile": false, - "following": null, - "utc_offset": 19800, - "profile_image_url": "http://a1.twimg.com/profile_images/1314697710/cropprofile1_normal.png", - "description": "I like you already.", + "statuses_count": 8298, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 35, + "friends_count": 149, + "profile_background_color": "ffffff", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/profile_background_images/290833153/ikea_lena_2.jpg", + "created_at": "Tue Jan 20 02:45:47 +0000 2009", + "followers_count": 271, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/290833153/ikea_lena_2.jpg", + "id": 19216042, + "profile_text_color": "b8b2b2", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1445007999/IMG_7694_copie2_normal.jpg", + "profile_image_url": "http://a2.twimg.com/profile_images/1445007999/IMG_7694_copie2_normal.jpg" + }, + { + "profile_sidebar_fill_color": "fa89c9", + "protected": false, + "id_str": "192627716", + "notifications": false, + "profile_background_tile": true, + "screen_name": "soouzalaura", + "name": "Laura Souza", + "listed_count": 2, + "location": "Brasil", "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, "geo_enabled": true, - "friends_count": 322, - "profile_text_color": "333333", - "location": "Delhi, India", + "utc_offset": -10800, + "profile_link_color": "942394", + "description": "Welcome to real world. It's suck, you're gonna love it! (Friends)", + "profile_sidebar_border_color": "ed26bf", + "url": "http://laurapriscila.tumblr.com/", + "time_zone": "Brasilia", + "default_profile_image": false, + "statuses_count": 2218, + "profile_use_background_image": true, + "default_profile": false, + "verified": false, + "favourites_count": 2, + "friends_count": 119, + "profile_background_color": "ffffff", "is_translator": false, - "profile_sidebar_fill_color": "EADEAA", - "status": { - "in_reply_to_user_id_str": null, - "retweet_count": 0, - "in_reply_to_status_id": null, - "id_str": "113817061733576705", - "contributors": null, - "truncated": false, - "geo": null, - "coordinates": null, - "favorited": false, - "in_reply_to_user_id": null, - "in_reply_to_screen_name": null, - "possibly_sensitive": false, - "retweeted": false, - "source": "web", - "in_reply_to_status_id_str": null, - "id": 113817061733576700, - "place": null, - "text": "that’s what I’m talking about → RT @Leffot: Photo: Hand-sewing shell cordovan http://t.co/rcxrU20", - "created_at": "Wed Sep 14 03:31:16 +0000 2011" - }, - "follow_request_sent": null, + "profile_background_image_url": "http://a1.twimg.com/profile_background_images/237284023/bg.gif", + "created_at": "Sun Sep 19 18:12:21 +0000 2010", + "followers_count": 61, + "follow_request_sent": false, + "lang": "pt", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/237284023/bg.gif", + "id": 192627716, + "profile_text_color": "524646", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1422000163/SAM_0539_normal.JPG", + "profile_image_url": "http://a0.twimg.com/profile_images/1422000163/SAM_0539_normal.JPG" + }, + { + "profile_sidebar_fill_color": "efefef", + "protected": false, + "id_str": "58000577", + "notifications": false, "profile_background_tile": true, - "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/88254516/palmetto_t.gif", - "url": "http://firasd.org", - "statuses_count": 2821, - "followers_count": 215, - "screen_name": "firasd", + "screen_name": "colchica", + "name": "C Orne", + "listed_count": 0, + "location": "Lyon - France", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": true, + "utc_offset": 3600, + "profile_link_color": "2d66ad", + "description": "addicted to caffeine and a little bit greener... ", + "default_profile": false, + "profile_sidebar_border_color": "eeeeee", + "url": null, + "time_zone": "Paris", + "default_profile_image": false, + "statuses_count": 232, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 1, + "friends_count": 45, + "profile_background_color": "ebeef0", + "is_translator": false, + "profile_background_image_url": "http://a3.twimg.com/profile_background_images/154201528/20100912103242_2010-09-11_eboulis_05.jpg", + "created_at": "Sat Jul 18 18:44:29 +0000 2009", + "followers_count": 14, + "follow_request_sent": false, + "lang": "fr", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/154201528/20100912103242_2010-09-11_eboulis_05.jpg", + "id": 58000577, + "profile_text_color": "994399", + "profile_image_url_https": "https://si0.twimg.com/profile_images/336483751/samantha_normal.jpg", + "profile_image_url": "http://a0.twimg.com/profile_images/336483751/samantha_normal.jpg" + }, + { + "profile_sidebar_fill_color": "c9c9c9", + "protected": false, + "id_str": "67218346", "notifications": null, - "profile_link_color": "9D582E", + "profile_background_tile": true, + "screen_name": "GrandyManFTW14", + "name": "Valerie", + "listed_count": 39, + "location": "Yankee Stadium", + "show_all_inline_media": true, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": -18000, + "profile_link_color": "2138B4", + "description": "im a nerd thats infatuated with sports... mostly #yankees baseball. I should reside in a bubble. I'm 28 and speak my mind.", + "profile_sidebar_border_color": "bfbfbf", + "url": "http://www.facebook.com/yankeesangel21", + "default_profile": false, + "time_zone": "Eastern Time (US & Canada)", + "default_profile_image": false, + "statuses_count": 8744, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 11, + "friends_count": 653, + "profile_background_color": "07090b", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/profile_background_images/202039528/x1f1841c56244072a549acc188477e1e.jpg", + "created_at": "Thu Aug 20 03:43:17 +0000 2009", + "followers_count": 500, + "follow_request_sent": null, "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/202039528/x1f1841c56244072a549acc188477e1e.jpg", + "id": 67218346, + "profile_text_color": "183050", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1520181363/image_normal.jpg", + "profile_image_url": "http://a2.twimg.com/profile_images/1520181363/image_normal.jpg" + }, + { + "profile_sidebar_fill_color": "95E8EC", + "protected": false, + "id_str": "185014625", + "default_profile": false, + "notifications": false, + "profile_background_tile": false, + "screen_name": "chiquip35", + "name": "Paula Klimowitz", + "listed_count": 1, + "location": "Miami", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": false, + "utc_offset": null, + "profile_link_color": "0099B9", + "description": "I love the positive energy of the people of the world!!", + "profile_sidebar_border_color": "5ED4DC", + "url": null, + "time_zone": null, + "default_profile_image": false, + "statuses_count": 48, + "profile_use_background_image": true, "verified": false, - "favourites_count": 3, - "profile_sidebar_border_color": "D9B17E", - "id": 6299142, - "listed_count": 16, - "created_at": "Fri May 25 00:02:48 +0000 2007" + "favourites_count": 0, + "friends_count": 102, + "profile_background_color": "0099B9", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme4/bg.gif", + "created_at": "Tue Aug 31 01:15:36 +0000 2010", + "followers_count": 9, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme4/bg.gif", + "id": 185014625, + "profile_text_color": "3C3940", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1521529755/174758_normal.jpg", + "profile_image_url": "http://a3.twimg.com/profile_images/1521529755/174758_normal.jpg" }], "resultType": "ReTweet" }] } } + } + From c5efafd0caa71d3fd42436eb62a35ae8c0114638 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Wed, 2 Nov 2011 04:23:52 -0500 Subject: [PATCH 04/40] fixes to make links more usable --- Collections/Links/dataIn.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Collections/Links/dataIn.js b/Collections/Links/dataIn.js index 28496095f..f687bb0cf 100644 --- a/Collections/Links/dataIn.js +++ b/Collections/Links/dataIn.js @@ -103,9 +103,12 @@ var encounterQueue = async.queue(function(e, callback) { async.forEach(urls,function(u,cb){ linkMagic(u,function(link){ // make sure to pass in a new object, asyncutu - dataStore.addEncounter(lutil.extend(true,{orig:u,link:link},e), function(err,doc){ + dataStore.addEncounter(lutil.extend(true,{orig:u,link:link.link},e), function(err,doc){ if(err) return cb(err); dataStore.updateLinkAt(doc.link, doc.at, function() { + // event includes this encounter! + link.encounters = [doc]; + locker.event("link",link); // let happen independently search.index(doc.link, function() { cb() }); @@ -155,7 +158,6 @@ function linkMagic(origUrl, callback){ delete link.html; // don't want that stored if (!link.at) link.at = Date.now(); dataStore.addLink(link,function(err, obj){ - locker.event("link",obj); // let happen independently callback(link.link); // TODO: handle when it didn't get stored or is empty better, if even needed // background fetch oembed and save it on the link if found oembed.fetch({url:link.link, html:html}, function(e){ @@ -194,7 +196,7 @@ function getEncounterFB(post) function getEncounterTwitter(tweet) { var txt = (tweet.retweeted_status && tweet.retweeted_status.text) ? tweet.retweeted_status.text : tweet.text; - var e = {id:tweet.id + var e = {id:tweet.id_str , network:"twitter" , text: txt + " " + tweet.user.screen_name , from: (tweet.user)?tweet.user.name:"" From b33cb833d5d71b157d7208bb43bb6888bc81bc60 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Wed, 2 Nov 2011 04:46:04 -0500 Subject: [PATCH 05/40] make a bit more sane, still could use a nice refactoring --- Collections/Links/dataIn.js | 9 +++++---- Collections/Links/dataStore.js | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Collections/Links/dataIn.js b/Collections/Links/dataIn.js index f687bb0cf..7bcf6bf60 100644 --- a/Collections/Links/dataIn.js +++ b/Collections/Links/dataIn.js @@ -133,9 +133,10 @@ exports.loadQueue = function() { // given a raw url, result in a fully stored qualified link (cb's full link url) function linkMagic(origUrl, callback){ // check if the orig url is in any encounter already (that has a full link url) - dataStore.checkUrl(origUrl,function(linkUrl){ - if(linkUrl) return callback(linkUrl); // short circuit! + dataStore.checkUrl(origUrl,function(link){ + if(link) return callback(link); // short circuit! // new one, expand it to a full one + var linkUrl; util.expandUrl({url:origUrl},function(u2){linkUrl=u2},function(){ // fallback use orig if errrrr if(!linkUrl) { @@ -145,7 +146,7 @@ function linkMagic(origUrl, callback){ // does this full one already have a link stored? dataStore.getLinks({link:linkUrl,limit:1},function(l){link=l},function(err){ if(link) { - return callback(link.link); // yeah short circuit dos! + return callback(link); // yeah short circuit dos! } // new link!!! link = {link:linkUrl}; @@ -158,7 +159,7 @@ function linkMagic(origUrl, callback){ delete link.html; // don't want that stored if (!link.at) link.at = Date.now(); dataStore.addLink(link,function(err, obj){ - callback(link.link); // TODO: handle when it didn't get stored or is empty better, if even needed + callback(link); // TODO: handle when it didn't get stored or is empty better, if even needed // background fetch oembed and save it on the link if found oembed.fetch({url:link.link, html:html}, function(e){ if(!e) return; diff --git a/Collections/Links/dataStore.js b/Collections/Links/dataStore.js index 6a2c72d82..e7ec54bce 100644 --- a/Collections/Links/dataStore.js +++ b/Collections/Links/dataStore.js @@ -58,7 +58,7 @@ exports.checkUrl = function(origUrl, callback) { if(err) return callback(); cursor.nextObject(function(err, item){ if(err || !item || !item.link) return callback(); - callback(item.link); + callback(item); }); }); } @@ -70,7 +70,7 @@ exports.getLinks = function(arg, cbEach, cbDone) { findWrap(f,arg,linkCollection,cbEach,cbDone); } -exports.getFullLink = function(id, cbDone) { +exports.getLink = function(id, cbDone) { var link = null; exports.getLinks({link:id}, function(l) { link = l; }, function() { cbDone(link); }); } From f321b0ae9cd6d877b46e9ef87c6ddfd849d4db29 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Sat, 5 Nov 2011 12:14:03 -0500 Subject: [PATCH 06/40] adding instagram support and checking text-based dedup --- Collections/Timeline/dataIn.js | 58 ++++++++++++- Collections/Timeline/fixtures/facebook.tweet | 41 ++++++++++ Collections/Timeline/fixtures/ig.fb | 42 ++++++++++ Collections/Timeline/fixtures/ig.instagram | 66 +++++++++++++++ Collections/Timeline/fixtures/ig.tweet | 80 ++++++++++++++++++ Collections/Timeline/fixtures/instagram.feed | 84 +++++++++++++++++++ Collections/Timeline/fixtures/tweet.facebook | 86 ++++++++++++++++++++ 7 files changed, 453 insertions(+), 4 deletions(-) create mode 100644 Collections/Timeline/fixtures/facebook.tweet create mode 100644 Collections/Timeline/fixtures/ig.fb create mode 100644 Collections/Timeline/fixtures/ig.instagram create mode 100644 Collections/Timeline/fixtures/ig.tweet create mode 100644 Collections/Timeline/fixtures/instagram.feed create mode 100644 Collections/Timeline/fixtures/tweet.facebook diff --git a/Collections/Timeline/dataIn.js b/Collections/Timeline/dataIn.js index 4b3eb359c..957f97906 100644 --- a/Collections/Timeline/dataIn.js +++ b/Collections/Timeline/dataIn.js @@ -18,7 +18,7 @@ exports.init = function(l, dStore){ exports.update = function(locker, callback) { dataStore.clear(function(){ callback(); - locker.providers(['link/facebook', 'status/twitter', 'checkin/foursquare'], function(err, services) { + locker.providers(['link/facebook', 'status/twitter', 'checkin/foursquare', 'photo/instagram'], function(err, services) { if (!services) return; services.forEach(function(svc) { if(svc.provides.indexOf('link/facebook') >= 0) { @@ -30,6 +30,9 @@ exports.update = function(locker, callback) { } else if(svc.provides.indexOf('checkin/foursquare') >= 0) { getData("recents/foursquare", svc.id); getData("checkin/foursquare", svc.id); + } else if(svc.provides.indexOf('photo/instagram') >= 0) { + getData("photo/instagram", svc.id); + getData("feed/instagram", svc.id); } }); }); @@ -90,6 +93,11 @@ function idrHost(r, data) r.hash = data.id; r.protocol = 'checkin'; } + if(r.host === 'instagram') + { + r.hash = data.id; + r.protocol = 'photo'; + } } // take an idr and turn it into a generic network-global key @@ -119,6 +127,7 @@ exports.processEvent = function(event, callback) if(event.type == 'link') return processLink(event, callback); var idr = getIdr(event.type, event.via, event.obj.data); + if(!idr.protocol) return callback("don't understand this data"); masterMaster(idr, event.obj.data, callback); } @@ -126,6 +135,7 @@ function isItMe(idr) { if(idr.protocol == 'tweet:' && idr.pathname == '/tweets') return true; if(idr.protocol == 'checkin:' && idr.pathname == '/checkin') return true; + if(idr.protocol == 'photo:' && idr.pathname == '/photo') return true; return false; } @@ -146,12 +156,13 @@ function masterMaster(idr, data, callback) } if(idr.protocol == 'post:') itemFacebook(item, data); if(idr.protocol == 'checkin:') itemFoursquare(item, data); + if(idr.host == 'instagram') itemInstagram(item, data); var dup; // we're only looking for the first match, if there's more, that's a very odd situation but could be handled here async.forEach(Object.keys(item.keys), function(key, cb) { dataStore.getItemByKey(key,function(err, doc){ - dup = doc; - cb(true); + if(!err && doc) dup = doc; + cb(); }); }, function (err) { if(dup) item = itemMerge(dup, item); @@ -370,4 +381,43 @@ function itemTwitterRelated(item, relateds) item.responses.push(resp); }); }); -} \ No newline at end of file +} + +// extract info from an instagram pic +function itemInstagram(item, pic) +{ + item.pri = 4; // the source + item.first = item.last = pic.created_time * 1000; + if(pic.caption) item.text = pic.caption.text; + if(pic.user) + { + item.from.id = 'contact://instagram/#'+pic.user.id; + item.from.name = pic.user.full_name; + item.from.icon = pic.user.profile_picture; + } + + // process responses! + if(pic.comments && pic.comments.data) + { + pic.comments.data.forEach(function(comment){ + var resp = newResponse(item, "comment"); + resp.at = comment.created_time * 1000; + resp.text = comment.text; + resp.from.id = 'contact://instagram/#'+comment.from.id; + resp.from.name = comment.from.full_name; + resp.from.icon = comment.from.profile_picture; + item.responses.push(resp); + }); + } + if(pic.likes && pic.likes.data) + { + pic.likes.data.forEach(function(like){ + var resp = newResponse(item, "up"); + resp.from.id = 'contact://instagram/#'+like.id; + resp.from.name = like.full_name; + resp.from.icon = like.profile_picture; + item.responses.push(resp); + }); + } +} + diff --git a/Collections/Timeline/fixtures/facebook.tweet b/Collections/Timeline/fixtures/facebook.tweet new file mode 100644 index 000000000..3fd5b42ff --- /dev/null +++ b/Collections/Timeline/fixtures/facebook.tweet @@ -0,0 +1,41 @@ +{ + "type": "home/facebook", + "via": "synclet/facebook", + "timestamp": 1315976215195, + "action": "update", + "obj": + { + "timeStamp": "2011-09-20T14:09:19.051Z", + "data": { + "id": "500012126_10150282386392127", + "from": { + "name": "Alex Rainert", + "id": "500012126" + }, + "message": "Dan was the 1st to nail the Google/Motorola news and yesterday he nailed Netflix. Get on the @splatf analysis, people! http://t.co/ZMu8UkyK", + "icon": "http://photos-d.ak.fbcdn.net/photos-ak-snc1/v27562/23/2231777543/app_2_2231777543_9553.gif", + "actions": [{ + "name": "Comment", + "link": "http://www.facebook.com/500012126/posts/10150282386392127" + }, + { + "name": "Like", + "link": "http://www.facebook.com/500012126/posts/10150282386392127" + }, + { + "name": "@arainert on Twitter", + "link": "http://twitter.com/arainert?utm_source=fb&utm_medium=fb&utm_campaign=arainert&utm_content=116149996419682304" + }], + "type": "status", + "application": { + "name": "Twitter", + "id": "2231777543" + }, + "created_time": 1316527292, + "updated_time": 1316527292, + "comments": { + "count": 0 + } + } + } +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/ig.fb b/Collections/Timeline/fixtures/ig.fb new file mode 100644 index 000000000..848689d6f --- /dev/null +++ b/Collections/Timeline/fixtures/ig.fb @@ -0,0 +1,42 @@ +{ + "type": "home/facebook", + "via": "synclet/facebook", + "timestamp": 1315976215195, + "action": "update", + "obj": + { + "timeStamp": "2011-09-19T15:55:51.950Z", + "data": { + "id": "4803283_730289247379", + "from": { + "name": "Julian Missig", + "id": "4803283" + }, + "message": "Model S model", + "picture": "http://platform.ak.fbcdn.net/www/app_full_proxy.php?app=124024574287414&v=1&size=z&cksum=437fa91b29c88f863ebfe460c8aa989b&src=http%3A%2F%2Fimages.instagram.com%2Fmedia%2F2011%2F09%2F18%2F391a26f2de1f47b089f4a2a2215b6bf7_7.jpg", + "link": "http://instagr.am/p/Ne6nr/", + "name": "jmissig's photo", + "caption": "instagr.am", + "description": "jmissig's photo on Instagram", + "icon": "http://photos-e.ak.fbcdn.net/photos-ak-snc1/v27562/10/124024574287414/app_2_124024574287414_6936.gif", + "actions": [{ + "name": "Comment", + "link": "http://www.facebook.com/4803283/posts/730289247379" + }, + { + "name": "Like", + "link": "http://www.facebook.com/4803283/posts/730289247379" + }], + "type": "link", + "application": { + "name": "Instagram", + "id": "124024574287414" + }, + "created_time": 1316385395, + "updated_time": 1316385395, + "comments": { + "count": 0 + } + } + } +} diff --git a/Collections/Timeline/fixtures/ig.instagram b/Collections/Timeline/fixtures/ig.instagram new file mode 100644 index 000000000..adfbe652c --- /dev/null +++ b/Collections/Timeline/fixtures/ig.instagram @@ -0,0 +1,66 @@ +{ + "type": "feed/instagram", + "via": "synclet/instagram", + "timestamp": 1315974920913, + "action": "new", + "obj": { + "timeStamp": "2011-10-27T00:45:21.104Z", + "data": { + "tags": [], + "type": "image", + "location": { + "latitude": 37.44355, + "longitude": -122.1706 + }, + "comments": { + "count": 0, + "data": [] + }, + "filter": "Normal", + "created_time": "1316385392", + "link": "http://instagr.am/p/Ne6nr/", + "likes": { + "count": 0, + "data": [] + }, + "images": { + "low_resolution": { + "url": "http://distillery.s3.amazonaws.com/media/2011/09/18/391a26f2de1f47b089f4a2a2215b6bf7_6.jpg", + "width": 306, + "height": 306 + }, + "thumbnail": { + "url": "http://distillery.s3.amazonaws.com/media/2011/09/18/391a26f2de1f47b089f4a2a2215b6bf7_5.jpg", + "width": 150, + "height": 150 + }, + "standard_resolution": { + "url": "http://distillery.s3.amazonaws.com/media/2011/09/18/391a26f2de1f47b089f4a2a2215b6bf7_7.jpg", + "width": 612, + "height": 612 + } + }, + "caption": { + "created_time": "1316385394", + "text": "Model S model", + "from": { + "username": "jmissig", + "profile_picture": "http://images.instagram.com/profiles/profile_981179_75sq_1297666725.jpg", + "id": "981179", + "full_name": "Julian Missig" + }, + "id": "277184261" + }, + "user_has_liked": false, + "id": "226208235_981179", + "user": { + "username": "jmissig", + "website": "", + "bio": "", + "profile_picture": "http://images.instagram.com/profiles/profile_981179_75sq_1297666725.jpg", + "full_name": "Julian Missig", + "id": "981179" + } + } + } +} diff --git a/Collections/Timeline/fixtures/ig.tweet b/Collections/Timeline/fixtures/ig.tweet new file mode 100644 index 000000000..94d59c640 --- /dev/null +++ b/Collections/Timeline/fixtures/ig.tweet @@ -0,0 +1,80 @@ +{ + "type": "timeline/twitter", + "via": "synclet/twitter", + "timestamp": 1315974920913, + "action": "new", + "obj": { + "timeStamp": "2011-09-19T15:56:00.454Z", + "data": { + "in_reply_to_status_id_str": null, + "truncated": false, + "in_reply_to_user_id_str": null, + "geo": null, + "retweet_count": 0, + "contributors": null, + "coordinates": null, + "user": { + "statuses_count": 4902, + "favourites_count": 111, + "protected": false, + "profile_text_color": "333333", + "profile_image_url": "http://a1.twimg.com/profile_images/31867902/Julian_Missig_1_normal.png", + "name": "Julian Missig", + "profile_sidebar_fill_color": "A0C5C7", + "listed_count": 22, + "following": true, + "profile_background_tile": false, + "utc_offset": -28800, + "description": "interaction designer & prototyper", + "location": "Redwood City, CA", + "contributors_enabled": false, + "verified": false, + "profile_link_color": "369EAF", + "followers_count": 248, + "url": null, + "profile_sidebar_border_color": "86A4A6", + "screen_name": "jmissig", + "default_profile_image": false, + "notifications": false, + "default_profile": false, + "show_all_inline_media": false, + "geo_enabled": true, + "profile_use_background_image": true, + "friends_count": 186, + "id_str": "9147872", + "is_translator": false, + "lang": "en", + "time_zone": "Pacific Time (US & Canada)", + "created_at": "Fri Sep 28 19:19:58 +0000 2007", + "profile_background_color": "709397", + "id": 9147872, + "follow_request_sent": false, + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme6/bg.gif", + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme6/bg.gif", + "profile_image_url_https": "https://si0.twimg.com/profile_images/31867902/Julian_Missig_1_normal.png" + }, + "favorited": false, + "in_reply_to_screen_name": null, + "source": "Instagram", + "id_str": "115554837076258816", + "entities": { + "hashtags": [], + "urls": [{ + "indices": [14, 34], + "url": "http://t.co/PeedUocW", + "display_url": "instagr.am/p/Ne6nr/", + "expanded_url": "http://instagr.am/p/Ne6nr/" + }], + "user_mentions": [] + }, + "in_reply_to_status_id": null, + "id": 115554837076258820, + "created_at": "Sun Sep 18 22:36:34 +0000 2011", + "possibly_sensitive": false, + "place": null, + "retweeted": false, + "in_reply_to_user_id": null, + "text": "Model S model http://t.co/PeedUocW" + } + } +} diff --git a/Collections/Timeline/fixtures/instagram.feed b/Collections/Timeline/fixtures/instagram.feed new file mode 100644 index 000000000..27d22c3dc --- /dev/null +++ b/Collections/Timeline/fixtures/instagram.feed @@ -0,0 +1,84 @@ +{ + "type": "feed/instagram", + "via": "synclet/instagram", + "timestamp": 1315974920913, + "action": "new", + "obj": { + "timeStamp": 1319676316966, + "data": { + "tags": [], + "type": "image", + "location": null, + "comments": { + "count": 1, + "data": [{ + "created_time": "1319651890", + "text": "Great pic, love the colors", + "from": { + "username": "bradl3y", + "profile_picture": "http://images.instagram.com/profiles/profile_11076900_75sq_1319234253.jpg", + "id": "11076900", + "full_name": "Brad Barnett" + }, + "id": "367123985" + }] + }, + "filter": "Normal", + "created_time": "1319642091", + "link": "http://instagr.am/p/Rhwbj/", + "likes": { + "count": 2, + "data": [{ + "username": "seanfeezy", + "profile_picture": "http://images.instagram.com/profiles/profile_8524340_75sq_1314153519.jpg", + "id": "8524340", + "full_name": "Sean Fissel" + }, + { + "username": "bradl3y", + "profile_picture": "http://images.instagram.com/profiles/profile_11076900_75sq_1319234253.jpg", + "id": "11076900", + "full_name": "Brad Barnett" + }] + }, + "images": { + "low_resolution": { + "url": "http://images.instagram.com/media/2011/10/26/f47c9ff00aff4ae9a6f8491d24722308_6.jpg", + "width": 306, + "height": 306 + }, + "thumbnail": { + "url": "http://images.instagram.com/media/2011/10/26/f47c9ff00aff4ae9a6f8491d24722308_5.jpg", + "width": 150, + "height": 150 + }, + "standard_resolution": { + "url": "http://images.instagram.com/media/2011/10/26/f47c9ff00aff4ae9a6f8491d24722308_7.jpg", + "width": 612, + "height": 612 + } + }, + "caption": { + "created_time": "1319642158", + "text": "Morning Mission. ", + "from": { + "username": "jasoncavnar", + "profile_picture": "http://images.instagram.com/profiles/profile_2920312_75sq_1316296467.jpg", + "id": "2920312", + "full_name": "Jason Cavnar" + }, + "id": "366806429" + }, + "user_has_liked": false, + "id": "294061795_2920312", + "user": { + "username": "jasoncavnar", + "website": "http://www.singly.com", + "bio": "That guy. Using Instagr.am to try to keep up with my hipster friends and their tight jeans. ", + "profile_picture": "http://images.instagram.com/profiles/profile_2920312_75sq_1316296467.jpg", + "full_name": "Jason Cavnar", + "id": "2920312" + } + } + } +} diff --git a/Collections/Timeline/fixtures/tweet.facebook b/Collections/Timeline/fixtures/tweet.facebook new file mode 100644 index 000000000..f0c54b12f --- /dev/null +++ b/Collections/Timeline/fixtures/tweet.facebook @@ -0,0 +1,86 @@ +{ + "type": "timeline/twitter", + "via": "synclet/twitter", + "timestamp": 1315974920913, + "action": "new", + "obj": { + "timeStamp": "2011-09-20T14:09:19.331Z", + "data": { + "contributors": null, + "truncated": false, + "geo": null, + "favorited": false, + "coordinates": null, + "user": { + "profile_text_color": "393939", + "protected": false, + "listed_count": 457, + "profile_sidebar_fill_color": "e8e8e8", + "name": "Alex Rainert", + "contributors_enabled": false, + "profile_background_tile": true, + "profile_image_url_https": "https://si0.twimg.com/profile_images/1331003463/arainert_headshot2_supersm_normal.jpg", + "profile_image_url": "http://a0.twimg.com/profile_images/1331003463/arainert_headshot2_supersm_normal.jpg", + "default_profile": false, + "show_all_inline_media": true, + "following": true, + "geo_enabled": true, + "utc_offset": -18000, + "description": "Husband, father & owner of a lovably neurotic vizsla. Obsessed with information, design, emerging tech, sports & food. Head of product at foursquare.", + "profile_link_color": "ff4cb4", + "location": "Brooklyn, NY", + "default_profile_image": false, + "verified": false, + "profile_sidebar_border_color": "ff4cb4", + "statuses_count": 15165, + "friends_count": 511, + "followers_count": 4717, + "url": "http://www.alexrainert.com/", + "is_translator": false, + "profile_use_background_image": true, + "screen_name": "arainert", + "follow_request_sent": false, + "notifications": false, + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/17611084/walken_small.jpg", + "time_zone": "Eastern Time (US & Canada)", + "favourites_count": 68, + "profile_background_color": "352726", + "lang": "en", + "profile_background_image_url": "http://a1.twimg.com/profile_background_images/17611084/walken_small.jpg", + "id": 7482, + "id_str": "7482", + "created_at": "Wed Oct 04 18:33:19 +0000 2006" + }, + "in_reply_to_screen_name": null, + "possibly_sensitive": false, + "retweeted": false, + "in_reply_to_status_id_str": null, + "source": "Tweet Button", + "entities": { + "user_mentions": [{ + "name": "SplatF", + "indices": [93, 100], + "screen_name": "splatf", + "id": 174886575, + "id_str": "174886575" + }], + "hashtags": [], + "urls": [{ + "display_url": "splatf.com/2011/09/netfli…", + "indices": [119, 139], + "expanded_url": "http://www.splatf.com/2011/09/netflix-qwikster-facts/", + "url": "http://t.co/ZMu8UkyK" + }] + }, + "in_reply_to_user_id": null, + "id": 116149996419682300, + "in_reply_to_user_id_str": null, + "id_str": "116149996419682304", + "place": null, + "retweet_count": 0, + "in_reply_to_status_id": null, + "text": "Dan was the 1st to nail the Google/Motorola news and yesterday he nailed Netflix. Get on the @splatf analysis, people! http://t.co/ZMu8UkyK", + "created_at": "Tue Sep 20 14:01:31 +0000 2011" + } + } +} From 3b6c761a7eff3bb08cf87605e459cec8c79f3971 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Sat, 5 Nov 2011 12:21:51 -0500 Subject: [PATCH 07/40] instagram merging and from bugfix --- Collections/Timeline/dataIn.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Collections/Timeline/dataIn.js b/Collections/Timeline/dataIn.js index 957f97906..99bc0e294 100644 --- a/Collections/Timeline/dataIn.js +++ b/Collections/Timeline/dataIn.js @@ -284,7 +284,8 @@ function itemTwitter(item, tweet) if(item.text) { var hash = crypto.createHash('md5'); - hash.update(item.text.substr(0,130)); // ignore trimming variations + var txt = item.text.replace(/ http\:\/\/\S+$/,""); // cut off appendege links + hash.update(txt.substr(0,130)); // ignore trimming variations item.keys['text:'+hash.digest('hex')] = item.ref; } // if it's tracking a reply, key it too @@ -351,6 +352,7 @@ function itemFoursquare(item, checkin) item.from.id = 'contact://foursquare/#'+checkin.user.id; item.from.name = checkin.user.firstName + " " + checkin.user.lastName; item.from.icon = checkin.user.photo; + item.froms[item.from.id] = item.ref; } if(checkin.comments && checkin.comments.items) { @@ -394,6 +396,14 @@ function itemInstagram(item, pic) item.from.id = 'contact://instagram/#'+pic.user.id; item.from.name = pic.user.full_name; item.from.icon = pic.user.profile_picture; + item.froms[item.from.id] = item.ref; + } + + if(item.text) + { + var hash = crypto.createHash('md5'); + hash.update(item.text.substr(0,130)); // ignore trimming variations + item.keys['text:'+hash.digest('hex')] = item.ref; } // process responses! From 99bc0587d55fe6b97798837c194d4b486915f536 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Sat, 5 Nov 2011 12:29:20 -0500 Subject: [PATCH 08/40] fix a regression --- Collections/Timeline/dataIn.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Collections/Timeline/dataIn.js b/Collections/Timeline/dataIn.js index 99bc0e294..3889b1e47 100644 --- a/Collections/Timeline/dataIn.js +++ b/Collections/Timeline/dataIn.js @@ -284,8 +284,10 @@ function itemTwitter(item, tweet) if(item.text) { var hash = crypto.createHash('md5'); - var txt = item.text.replace(/ http\:\/\/\S+$/,""); // cut off appendege links - hash.update(txt.substr(0,130)); // ignore trimming variations + hash.update(item.text.substr(0,130)); // ignore trimming variations + item.keys['text:'+hash.digest('hex')] = item.ref; + var hash = crypto.createHash('md5'); + hash.update(item.text.replace(/ http\:\/\/\S+$/,"")); // cut off appendege links that some apps add (like instagram) item.keys['text:'+hash.digest('hex')] = item.ref; } // if it's tracking a reply, key it too From 8a0b798c1a5b17039e79cc984085676f39e7751c Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Sun, 6 Nov 2011 12:50:16 -0600 Subject: [PATCH 09/40] cleaning up the api endpoints --- Collections/Timeline/dataIn.js | 5 +- Collections/Timeline/dataStore.js | 23 +++++++- Collections/Timeline/timeline.js | 98 +++++++++++++++++-------------- 3 files changed, 77 insertions(+), 49 deletions(-) diff --git a/Collections/Timeline/dataIn.js b/Collections/Timeline/dataIn.js index 3889b1e47..703d6f4b8 100644 --- a/Collections/Timeline/dataIn.js +++ b/Collections/Timeline/dataIn.js @@ -15,10 +15,11 @@ exports.init = function(l, dStore){ } // manually walk and reindex all possible link sources -exports.update = function(locker, callback) { +exports.update = function(locker, type, callback) { dataStore.clear(function(){ callback(); - locker.providers(['link/facebook', 'status/twitter', 'checkin/foursquare', 'photo/instagram'], function(err, services) { + var types = (type) ? [type] : ['link/facebook', 'status/twitter', 'checkin/foursquare', 'photo/instagram']; + locker.providers(types, function(err, services) { if (!services) return; services.forEach(function(svc) { if(svc.provides.indexOf('link/facebook') >= 0) { diff --git a/Collections/Timeline/dataStore.js b/Collections/Timeline/dataStore.js index 8bb4cbe7a..358e07d11 100644 --- a/Collections/Timeline/dataStore.js +++ b/Collections/Timeline/dataStore.js @@ -9,6 +9,8 @@ var logger = require(__dirname + "/../../Common/node/logger").logger; var crypto = require("crypto"); var async = require('async'); +var lmongoutil = require("lmongoutil"); + // in the future we'll probably need a visitCollection too var itemCol, respCol; @@ -32,6 +34,14 @@ exports.getTotalResponses = function(callback) { respColl.count(callback); } +exports.getAll = function(fields, callback) { + itemCol.find({}, fields, callback); +} + +exports.getLastObjectID = function(cbDone) { + linkCollection.find({}, {fields:{_id:1}, limit:1, sort:{_id:-1}}).nextObject(cbDone); +} + exports.getItemByKey = function(key, callback) { var item; var kname = "keys."+key; @@ -49,16 +59,25 @@ exports.getItem = function(id, callback) { exports.getResponses = function(arg, cbEach, cbDone) { var f = (arg.item)?{item:arg.item}:{}; delete arg.item; + if(arg.from) f["from.id"] = arg.from; findWrap(f,arg,respCol,cbEach,cbDone); } +exports.getSince = function(arg, cbEach, cbDone) { + if(!arg || !arg.id) return cbDone("no id given"); + findWrap({"_id":{"$gt":lmongoutil.ObjectID(arg.id)}}, {sort:{_id:-1}}, linkCollection, cbEach, cbDone); +} + // arg takes sort/limit/offset/find exports.getItems = function(arg, cbEach, cbDone) { var f = {}; try { - f = JSON.parse(arg.find); // optional, can bomb out - }catch(E){} + if(arg.find) f = JSON.parse(arg.find); // optional, can bomb out + }catch(E){ + return cbDone("couldn't parse find"); + } delete arg.find; + if(arg.from) f["froms."+arg.from] = {$exists:true}; findWrap(f,arg,itemCol,cbEach,cbDone); } diff --git a/Collections/Timeline/timeline.js b/Collections/Timeline/timeline.js index 616e3100c..9da1cd95f 100644 --- a/Collections/Timeline/timeline.js +++ b/Collections/Timeline/timeline.js @@ -14,7 +14,7 @@ var fs = require('fs'), request = require('request'), locker = require('../../Common/node/locker.js'); var async = require("async"); - + var dataIn = require('./dataIn'); // for processing incoming twitter/facebook/etc data types var dataStore = require("./dataStore"); // storage/retreival of raw items and responses @@ -26,32 +26,66 @@ var app = express.createServer(connect.bodyParser()); app.set('views', __dirname); app.get('/', function(req, res) { - res.writeHead(200, { - 'Content-Type': 'text/html' - }); - dataStore.getTotalItems(function(err, countInfo) { - res.write('

Found '+ countInfo +' items

'); - res.end(); + var fields = {}; + if (req.query.fields) { + try { + fields = JSON.parse(req.query.fields); + } catch(E) {} + } + dataStore.getAll(fields, function(err, cursor) { + if(!req.query["all"]) cursor.limit(20); // default 20 unless all is set + if(req.query["limit"]) cursor.limit(parseInt(req.query["limit"])); + if(req.query["offset"]) cursor.skip(parseInt(req.query["offset"])); + var ndx = {}; + cursor.toArray(function(err, items) { + if(req.query["all"] || !req.query.full) return res.send(items); // default not include responses, forced if all + items.forEach(function(item){ ndx[item.id] = item; item.responses = []; }); // build index + var arg = {"item":{$in: Object.keys(ndx)}}; + arg.fields = fields; + dataStore.getResponses(arg, function(response) { + ndx[response.item].responses.push(response); + }, function() { + res.send(items); + }); + }); }); }); app.get('/state', function(req, res) { dataStore.getTotalItems(function(err, countInfo) { if(err) return res.send(err, 500); - var updated = new Date().getTime(); - try { - var js = JSON.parse(fs.readFileSync('state.json')); - if(js && js.updated) updated = js.updated; - } catch(E) {} - res.send({ready:1, count:countInfo, updated:updated}); + dataStore.getLastObjectID(function(err, lastObject) { + if(err) return res.send(err, 500); + var objId = "000000000000000000000000"; + if (lastObject) objId = lastObject._id.toHexString(); + var updated = new Date().getTime(); + try { + var js = JSON.parse(fs.readFileSync('state.json')); + if(js && js.updated) updated = js.updated; + } catch(E) {} + res.send({ready:1, count:countInfo, updated:updated, lastId:objId}); + }); + }); +}); + +// expose way to get the list of responses from an item id +app.get('/responses/:id', function(req, res) { + var results = []; + dataStore.getResponses({item:req.param('id')}, function(item){results.push(item);},function(err){ + if(err) return res.send(err,500); + res.send(results); }); }); +app.get('/id/:id', function(req, res, next) { + if (req.param('id').length != 24) return next(req, res, next); + dataStore.getItem(req.param('id'), function(err, doc) { res.send(doc); }); +}); app.get('/update', function(req, res) { - dataIn.update(locker, function(){ + dataIn.update(locker, req.query.type, function(){ res.writeHead(200); - res.end('Extra mince!'); + res.end('Extra mince!'); }); }); @@ -77,38 +111,12 @@ function genericApi(name,f) if(err) return res.send(err,500); res.send(results); }); - }); + }); } -// expose way to get items and responses in one -app.get('/getItemsFull', function(req, res) { - var fullResults = []; - var results = []; - var options = {sort:{"at":-1}}; - if (req.query.limit) { - options.limit = parseInt(req.query.limit); - }else{ - options.limit = 100; - } - if (req.query.offset) { - options.offset = parseInt(req.query.offset); - } - dataStore.getItems(options, function(item) { results.push(item); }, function(err) { - async.forEach(results, function(item, callback) { - item.responses = []; - dataStore.getResponses({"item":item.idr}, function(response) { - item.responses.push(response); - }, function() { - fullResults.push(item); - callback(); - }); - }, function() { - res.send(results); - }); - }); -}); genericApi('/getItems', dataStore.getItems); -genericApi('/getResponses',dataStore.getResponses); +genericApi('/getResponses', dataStore.getResponses); +genericApi('/getSince', dataStore.getSince); // Process the startup JSON object process.stdin.resume(); @@ -121,7 +129,7 @@ process.stdin.on('data', function(data) { process.exit(1); } process.chdir(lockerInfo.workingDirectory); - + locker.connectToMongo(function(mongo) { // initialize all our libs dataStore.init(mongo.collections.item,mongo.collections.response); From 9fbbc00e7355430d32a8c3956477d9c0346b5d54 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Sun, 6 Nov 2011 16:15:20 -0600 Subject: [PATCH 10/40] added twitter rt and reply support, lots of api cleanup and bugfixing --- Collections/Timeline/README.md | 16 +- Collections/Timeline/dataIn.js | 108 ++++++++----- Collections/Timeline/dataStore.js | 7 +- Collections/Timeline/fixtures/twitter.reply | 6 + .../Timeline/fixtures/twitter.reply.orig | 7 + Collections/Timeline/fixtures/twitter.rt | 146 ++++++++++++++++++ Collections/Timeline/timeline.js | 1 - 7 files changed, 249 insertions(+), 42 deletions(-) create mode 100644 Collections/Timeline/fixtures/twitter.reply create mode 100644 Collections/Timeline/fixtures/twitter.reply.orig create mode 100644 Collections/Timeline/fixtures/twitter.rt diff --git a/Collections/Timeline/README.md b/Collections/Timeline/README.md index 651072522..f482adea1 100644 --- a/Collections/Timeline/README.md +++ b/Collections/Timeline/README.md @@ -2,4 +2,18 @@ The "feeds" collection is all of the socially shared items across any service, f It consists of a primary type called an "item" which can have an array of "responses" that are comments, likes, retweets, etc from other people. -The goal is to dedup cross-posts to multiple networks from one person, particularly things like a foursquare checkin. \ No newline at end of file +The goal is to dedup cross-posts to multiple networks from one person, particularly things like a foursquare checkin. + + +Notes: + + id references and keys + cross-collection reference storage with links + link based dedup of foursquare + dedup uses keys as guids across networks and for text matching variations + /update takes type arg for individual updates + idrs as via, no original storage + prioritization scheme in merging items + generic from object, list of froms + references array of all sources + generalized responses and top-level statistics diff --git a/Collections/Timeline/dataIn.js b/Collections/Timeline/dataIn.js index 703d6f4b8..5596dc1d4 100644 --- a/Collections/Timeline/dataIn.js +++ b/Collections/Timeline/dataIn.js @@ -18,20 +18,22 @@ exports.init = function(l, dStore){ exports.update = function(locker, type, callback) { dataStore.clear(function(){ callback(); - var types = (type) ? [type] : ['link/facebook', 'status/twitter', 'checkin/foursquare', 'photo/instagram']; + var types = (type) ? [type] : ['home/facebook', 'tweets/twitter', 'recents/foursquare', 'feed/instagram']; locker.providers(types, function(err, services) { if (!services) return; services.forEach(function(svc) { - if(svc.provides.indexOf('link/facebook') >= 0) { + logger.debug("processing "+svc.id); + if(svc.provides.indexOf('home/facebook') >= 0) { getData("home/facebook", svc.id); } else if(svc.provides.indexOf('status/twitter') >= 0) { getData("tweets/twitter", svc.id); getData("timeline/twitter", svc.id); getData("mentions/twitter", svc.id); + getData("related/twitter", svc.id); } else if(svc.provides.indexOf('checkin/foursquare') >= 0) { getData("recents/foursquare", svc.id); getData("checkin/foursquare", svc.id); - } else if(svc.provides.indexOf('photo/instagram') >= 0) { + } else if(svc.provides.indexOf('feed/instagram') >= 0) { getData("photo/instagram", svc.id); getData("feed/instagram", svc.id); } @@ -110,7 +112,7 @@ function idr2key(idr, data) return url.parse(url.format(idr)); } -// useful to get key from raw data directly directly (like from a via, not from an event) +// useful to get key from raw data directly (like from a via, not from an event) function getKey(network, data) { var r = {slashes:true}; @@ -170,7 +172,8 @@ function masterMaster(idr, data, callback) dataStore.addItem(item, function(err, item){ if(err) return callback(err); // all done processing, very useful to pull out some summary stats now! - // feels inefficient to re-query and double-write here, possible optimization, but should be logically safe this way + // feels inefficient to re-query and double-write here, but should be logically safe this way + // TODO: can optimize and special case the first time! item.comments = item.ups = 0; dataStore.getResponses({item:item.id}, function(r){ if(r.type == "comment") item.comments++; @@ -182,6 +185,17 @@ function masterMaster(idr, data, callback) }); } +// save a reference +function itemRef(item, ref, callback) +{ + var refs = {}; + for(var i = 0; i < item.refs.length; i++) refs[item.refs[i]] = true; + if(refs[ref]) return callback(undefined, item); + refs[ref] = true; + item.refs = Object.keys(refs); + dataStore.addItem(item, callback); +} + // intelligently merge two items together and return function itemMerge(older, newer) { @@ -227,35 +241,32 @@ function itemMergeHard(a, b, cb) // when a processed link event comes in, check to see if it's a long url that could help us de-dup function processLink(event, callback) { - var encounter; - try { - encounter = event.obj.data.encounters[0]; - }catch(E){ - return callback(E); - } - // first, look up event via/orig key and see if we've processed it yet, if not (for some reason) ignore - var key = url.format(getKey(encounter.network, encounter.via)); - dataStore.getItemByKey(key,function(err, item){ - if(err || !item) return callback(err); - var u = url.parse(encounter.link); - // if foursquare checkin and from a tweet, generate foursquare key and look for it - if(u.host == 'foursquare.com' && u.pathname.indexOf('/checkin/') > 0) - { - var id = path.basename(u.pathname); - var k2 = 'checkin://foursquare/#'+id; - dataStore.getItemByKey(k2,function(err, item2){ - if(err || !item2) return callback(); - // found a dup! - itemMergeHard(item, item2); - }) - - } - // if from instagram, generate instagram key and look for it - }); - - // by here should have the item with the link, and the item for which the link is a target that is a duplicate - // merge them, merge any responses, delete the lesser item and it's responses - callback(); + if(!event || !event.obj || !event.obj.data || !event.obj.data.encounters) return callback("no encounter"); + // process each encounter if there's multiple + async.forEach(event.obj.data.encounters, function(encounter, cb){ + // first, look up event via/orig key and see if we've processed it yet, if not (for some reason) ignore + var key = url.format(getKey(encounter.network, encounter.via)); + dataStore.getItemByKey(key,function(err, item){ + if(err || !item) return cb(err); + // we tag this item with a ref to the link, super handy for cross-collection mashups + itemRef(item, "link://links/#"+event.obj.data._id, function(err, item){ + if(err || !item) return cb(err); + var u = url.parse(encounter.link); + // if foursquare checkin and from a tweet, generate foursquare key and look for it + if(u.host == 'foursquare.com' && u.pathname.indexOf('/checkin/') > 0) + { + var id = path.basename(u.pathname); + var k2 = 'checkin://foursquare/#'+id; + dataStore.getItemByKey(k2,function(err, item2){ + if(err || !item2) return cb(); + // found a dup! + itemMergeHard(item, item2, cb); + }); + } + // instagram uses the same caption everywhere so link-based dedup isn't needed + }); + }); + }, callback); } // give a bunch of sane defaults @@ -271,6 +282,18 @@ function newResponse(item, type) // extract info from a tweet function itemTwitter(item, tweet) { + // since RTs contain the original, add the response first then process the original! + if(tweet.retweeted_status) + { + var resp = newResponse(item, "up"); + resp.from.id = "contact://twitter/#"+tweet.user.screen_name; + resp.from.name = tweet.user.name; + resp.from.icon = tweet.user.profile_image_url; + item.responses.push(resp); + tweet = tweet.retweeted_status; + item.keys['tweet://twitter/#'+tweet.id_str] = item.ref; // tag with the original too + } + item.pri = 1; // tweets are the lowest priority? if(tweet.created_at) item.first = item.last = new Date(tweet.created_at).getTime(); if(tweet.text) item.text = tweet.text; @@ -291,8 +314,21 @@ function itemTwitter(item, tweet) hash.update(item.text.replace(/ http\:\/\/\S+$/,"")); // cut off appendege links that some apps add (like instagram) item.keys['text:'+hash.digest('hex')] = item.ref; } - // if it's tracking a reply, key it too - if(tweet.in_reply_to_status_id_str) item.keys['tweet://twitter/#'+tweet.in_reply_to_status_id_str] = item.ref; + + // if this is also a reply + if(tweet.in_reply_to_status_id_str) + { + item.pri = 0; // demoted + item.keys['tweet://twitter/#'+tweet.in_reply_to_status_id_str] = item.ref; // hopefully this merges with original if it exists + var resp = newResponse(item, "comment"); + // duplicate stuff into response too + resp.text = item.text; + resp.at = item.first; + resp.from.id = item.from.id; + resp.from.name = item.from.name; + resp.from.icon = item.from.icon; + item.responses.push(resp); + } } // extract info from a facebook post diff --git a/Collections/Timeline/dataStore.js b/Collections/Timeline/dataStore.js index 358e07d11..ba35fb1e3 100644 --- a/Collections/Timeline/dataStore.js +++ b/Collections/Timeline/dataStore.js @@ -39,7 +39,7 @@ exports.getAll = function(fields, callback) { } exports.getLastObjectID = function(cbDone) { - linkCollection.find({}, {fields:{_id:1}, limit:1, sort:{_id:-1}}).nextObject(cbDone); + itemCol.find({}, {fields:{_id:1}, limit:1, sort:{_id:-1}}).nextObject(cbDone); } exports.getItemByKey = function(key, callback) { @@ -111,8 +111,8 @@ exports.addItem = function(item, callback) { delete item.responses; // store responses in their own table delete item._id; // mongo is miss pissypants itemCol.findAndModify({"id":item.id}, [['_id','asc']], {$set:item}, {safe:true, upsert:true, new: true}, function(err, doc){ - if(err) return callback(err); - if(responses) async.forEach(responses, exports.addResponse, function(err){callback(err, doc);}); // orig caller wants saved item back + if(err || !responses) return callback(err); + async.forEach(responses, exports.addResponse, function(err){callback(err, doc);}); // orig caller wants saved item back }); } @@ -121,7 +121,6 @@ exports.addResponse = function(response, callback) { var hash = crypto.createHash('md5'); hash.update(JSON.stringify(response)); response.id = hash.digest('hex'); - logger.debug("addResponse: "+JSON.stringify(response)); delete response._id; // mongo is miss pissypants respCol.findAndModify({"id":response.id}, [['_id','asc']], {$set:response}, {safe:true, upsert:true, new: true}, callback); } diff --git a/Collections/Timeline/fixtures/twitter.reply b/Collections/Timeline/fixtures/twitter.reply new file mode 100644 index 000000000..46b74fc98 --- /dev/null +++ b/Collections/Timeline/fixtures/twitter.reply @@ -0,0 +1,6 @@ +{ + "type": "timeline/twitter", + "via": "synclet/twitter", + "timestamp": 1315974920913, + "action": "new", + "obj": {"timeStamp":"2011-11-04T16:32:06.068Z","data":{"place":{"country_code":"US","place_type":"city","name":"Cascade","bounding_box":{"type":"Polygon","coordinates":[[[-91.02358,42.289324],[-90.991574,42.289324],[-90.991574,42.308435],[-91.02358,42.308435]]]},"attributes":{},"full_name":"Cascade, IA","country":"United States","url":"http://api.twitter.com/1/geo/id/85905f44ae0575e5.json","id":"85905f44ae0575e5"},"retweet_count":0,"in_reply_to_screen_name":"FrancescoC","created_at":"Thu Nov 03 17:29:11 +0000 2011","retweeted":false,"in_reply_to_status_id_str":"132133925953875969","in_reply_to_user_id_str":"15592821","user":{"time_zone":"Central Time (US & Canada)","profile_link_color":"0000ff","protected":false,"default_profile":false,"created_at":"Wed Mar 21 02:44:51 +0000 2007","profile_image_url_https":"https://si0.twimg.com/profile_images/565991047/jer_normal.jpg","name":"Jeremie Miller","geo_enabled":true,"favourites_count":9,"profile_background_color":"ffffff","default_profile_image":false,"notifications":null,"profile_background_tile":false,"utc_offset":-21600,"description":"Currently building the Locker Project, TeleHash, and Singly with a focus on personal data + distributed protocols. Helped found Jabber/XMPP, open platforms FTW!","statuses_count":891,"following":null,"verified":false,"profile_sidebar_fill_color":"ffffff","followers_count":1375,"profile_image_url":"http://a0.twimg.com/profile_images/565991047/jer_normal.jpg","follow_request_sent":null,"profile_sidebar_border_color":"000000","location":"Cascade, IA","is_translator":false,"contributors_enabled":false,"screen_name":"jeremie","profile_use_background_image":false,"url":"http://jeremie.com/-","id_str":"1704421","show_all_inline_media":true,"profile_background_image_url_https":"https://si0.twimg.com/images/themes/theme1/bg.png","profile_text_color":"000000","id":1704421,"listed_count":141,"lang":"en","profile_background_image_url":"http://a0.twimg.com/images/themes/theme1/bg.png","friends_count":161},"contributors":null,"in_reply_to_status_id":132133925953875970,"source":"Twitter for iPhone","id_str":"132147322749599744","geo":{"type":"Point","coordinates":[42.29882394,-91.01307985]},"favorited":false,"id":132147322749599740,"entities":{"urls":[],"user_mentions":[{"name":"Francesco Cesarini","indices":[6,17],"screen_name":"FrancescoC","id_str":"15592821","id":15592821},{"name":"DizzyD","indices":[34,41],"screen_name":"dizzyd","id_str":"7206052","id":7206052}],"hashtags":[{"indices":[123,131],"text":"euc2011"}]},"coordinates":{"type":"Point","coordinates":[-91.01307985,42.29882394]},"in_reply_to_user_id":15592821,"truncated":false,"text":"Nice! @FrancescoC: Congratulating @dizzyd for winning the Erlang User of the year award for giving rebar to the community! #euc2011"}}} diff --git a/Collections/Timeline/fixtures/twitter.reply.orig b/Collections/Timeline/fixtures/twitter.reply.orig new file mode 100644 index 000000000..1ca49bfcc --- /dev/null +++ b/Collections/Timeline/fixtures/twitter.reply.orig @@ -0,0 +1,7 @@ +{ + "type": "timeline/twitter", + "via": "synclet/twitter", + "timestamp": 1315974920913, + "action": "new", + "obj":{"timeStamp":"2011-11-04T16:32:06.068Z","data":{"place":null,"retweet_count":41,"in_reply_to_screen_name":null,"created_at":"Thu Nov 03 16:54:29 +0000 2011","retweeted":false,"in_reply_to_status_id_str":null,"in_reply_to_user_id_str":null,"user":{"time_zone":null,"profile_link_color":"2FC2EF","protected":false,"created_at":"Fri Mar 13 13:15:23 +0000 2009","profile_image_url_https":"https://si0.twimg.com/profile_images/95456179/cortex_normal.png","name":"Jon Meredith","geo_enabled":false,"favourites_count":1,"profile_background_color":"1A1B1F","default_profile_image":false,"notifications":false,"profile_background_tile":false,"utc_offset":null,"description":"","statuses_count":85,"following":true,"verified":false,"profile_sidebar_fill_color":"252429","followers_count":127,"profile_image_url":"http://a0.twimg.com/profile_images/95456179/cortex_normal.png","follow_request_sent":false,"profile_sidebar_border_color":"181A1E","location":"Denver, CO, USA","is_translator":false,"contributors_enabled":false,"screen_name":"jon_meredith","default_profile":false,"profile_use_background_image":true,"url":null,"id_str":"24168124","show_all_inline_media":false,"profile_background_image_url_https":"https://si0.twimg.com/images/themes/theme9/bg.gif","profile_text_color":"666666","id":24168124,"listed_count":12,"lang":"en","profile_background_image_url":"http://a1.twimg.com/images/themes/theme9/bg.gif","friends_count":69},"contributors":null,"retweeted_status":{"place":null,"retweet_count":41,"in_reply_to_screen_name":null,"created_at":"Thu Nov 03 16:35:57 +0000 2011","retweeted":false,"in_reply_to_status_id_str":null,"in_reply_to_user_id_str":null,"user":{"time_zone":"London","profile_link_color":"D02B55","protected":false,"created_at":"Fri Jul 25 02:14:58 +0000 2008","profile_image_url_https":"https://si0.twimg.com/profile_images/337579346/francescoJulianCash_normal.jpg","name":"Francesco Cesarini","geo_enabled":false,"favourites_count":16,"profile_background_color":"352726","default_profile_image":false,"notifications":false,"profile_background_tile":false,"utc_offset":0,"description":"Founded Erlang Solutions, co-authored Erlang Programming (O'Reilly), part time uncle & godfather and full time pointy haired boss. ","statuses_count":3127,"following":false,"verified":false,"profile_sidebar_fill_color":"99CC33","followers_count":1480,"profile_image_url":"http://a1.twimg.com/profile_images/337579346/francescoJulianCash_normal.jpg","follow_request_sent":false,"profile_sidebar_border_color":"829D5E","location":"London, United Kingdom","is_translator":false,"contributors_enabled":false,"screen_name":"FrancescoC","default_profile":false,"profile_use_background_image":true,"url":"http://www.erlang-solutions.com","id_str":"15592821","show_all_inline_media":false,"profile_background_image_url_https":"https://si0.twimg.com/images/themes/theme5/bg.gif","profile_text_color":"3E4415","id":15592821,"listed_count":122,"lang":"en","profile_background_image_url":"http://a1.twimg.com/images/themes/theme5/bg.gif","friends_count":365},"contributors":null,"in_reply_to_status_id":null,"source":"Twitter for Android","id_str":"132133925953875969","geo":null,"favorited":false,"id":132133925953875970,"entities":{"urls":[],"user_mentions":[{"name":"DizzyD","indices":[15,22],"screen_name":"dizzyd","id_str":"7206052","id":7206052}],"hashtags":[{"indices":[121,129],"text":"euc2011"}]},"coordinates":null,"in_reply_to_user_id":null,"truncated":false,"text":"Congratulating @dizzyd for winning the Erlang User of the year award for giving rebar to the community! Well done Dizzy. #euc2011"},"in_reply_to_status_id":null,"source":"web","id_str":"132138590636478464","geo":null,"favorited":false,"id":132138590636478460,"entities":{"urls":[],"user_mentions":[{"name":"Francesco Cesarini","indices":[3,14],"screen_name":"FrancescoC","id_str":"15592821","id":15592821},{"name":"DizzyD","indices":[31,38],"screen_name":"dizzyd","id_str":"7206052","id":7206052}],"hashtags":[]},"coordinates":null,"in_reply_to_user_id":null,"truncated":true,"text":"RT @FrancescoC: Congratulating @dizzyd for winning the Erlang User of the year award for giving rebar to the community! Well done Dizzy. ..."}}} + diff --git a/Collections/Timeline/fixtures/twitter.rt b/Collections/Timeline/fixtures/twitter.rt new file mode 100644 index 000000000..5cf9a863f --- /dev/null +++ b/Collections/Timeline/fixtures/twitter.rt @@ -0,0 +1,146 @@ +{ + "type": "timeline/twitter", + "via": "synclet/twitter", + "timestamp": 1315974920913, + "action": "new", + "obj": { + "timeStamp": "2011-11-04T17:25:55.729Z", + "data": { + "place": null, + "retweet_count": 6, + "in_reply_to_screen_name": null, + "created_at": "Fri Nov 04 17:23:32 +0000 2011", + "retweeted": false, + "in_reply_to_status_id_str": null, + "in_reply_to_user_id_str": null, + "user": { + "time_zone": "Pacific Time (US & Canada)", + "profile_link_color": "0000ff", + "protected": false, + "created_at": "Tue Mar 27 01:14:05 +0000 2007", + "profile_image_url_https": "https://si0.twimg.com/profile_images/941827802/IMG_3811_v4_normal.jpg", + "name": "Tim O'Reilly", + "geo_enabled": true, + "favourites_count": 52, + "profile_background_color": "9ae4e8", + "default_profile_image": false, + "notifications": false, + "profile_background_tile": false, + "utc_offset": -28800, + "description": "Founder and CEO, O'Reilly Media. Watching the alpha geeks, sharing their stories, helping the future unfold.", + "statuses_count": 16827, + "following": true, + "verified": true, + "profile_sidebar_fill_color": "e0ff92", + "followers_count": 1508525, + "profile_image_url": "http://a1.twimg.com/profile_images/941827802/IMG_3811_v4_normal.jpg", + "follow_request_sent": false, + "profile_sidebar_border_color": "87bc44", + "location": "Sebastopol, CA", + "is_translator": false, + "default_profile": false, + "contributors_enabled": false, + "screen_name": "timoreilly", + "profile_use_background_image": true, + "url": "http://radar.oreilly.com", + "id_str": "2384071", + "show_all_inline_media": false, + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/3587880/notes.gif", + "profile_text_color": "000000", + "id": 2384071, + "listed_count": 20459, + "lang": "en", + "profile_background_image_url": "http://a1.twimg.com/profile_background_images/3587880/notes.gif", + "friends_count": 784 + }, + "contributors": null, + "retweeted_status": { + "place": null, + "retweet_count": 6, + "in_reply_to_screen_name": null, + "created_at": "Fri Nov 04 15:36:13 +0000 2011", + "retweeted": false, + "in_reply_to_status_id_str": null, + "in_reply_to_user_id_str": null, + "user": { + "time_zone": "Pacific Time (US & Canada)", + "profile_link_color": "0084B4", + "protected": false, + "created_at": "Tue Jan 27 02:59:48 +0000 2009", + "profile_image_url_https": "https://si0.twimg.com/profile_images/80487785/Michael_Chui_face_square_normal.jpg", + "name": "Michael Chui", + "geo_enabled": false, + "favourites_count": 0, + "profile_background_color": "C0DEED", + "default_profile_image": false, + "notifications": false, + "profile_background_tile": false, + "utc_offset": -28800, + "description": "Senior Fellow at McKinsey Global Institute (@McKinsey_MGI) leading research on the impact of technology trends. Opinions expressed are my own.", + "statuses_count": 1787, + "following": false, + "verified": false, + "profile_sidebar_fill_color": "DDEEF6", + "followers_count": 1326, + "profile_image_url": "http://a0.twimg.com/profile_images/80487785/Michael_Chui_face_square_normal.jpg", + "follow_request_sent": false, + "profile_sidebar_border_color": "C0DEED", + "location": "", + "is_translator": false, + "default_profile": true, + "contributors_enabled": false, + "screen_name": "mchui", + "profile_use_background_image": true, + "url": null, + "id_str": "19573968", + "show_all_inline_media": false, + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "profile_text_color": "333333", + "id": 19573968, + "listed_count": 85, + "lang": "en", + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "friends_count": 157 + }, + "contributors": null, + "in_reply_to_status_id": null, + "source": "Twitter for BlackBerry®", + "id_str": "132481280091754496", + "geo": null, + "favorited": false, + "id": 132481280091754500, + "entities": { + "urls": [], + "user_mentions": [], + "hashtags": [] + }, + "coordinates": null, + "in_reply_to_user_id": null, + "truncated": false, + "text": "Quoth Niall Ferguson: We no longer have the rule of law; we have the rule of lawyers and that's different." + }, + "in_reply_to_status_id": null, + "source": "Twitter for Android", + "id_str": "132508288435752960", + "geo": null, + "favorited": false, + "id": 132508288435752960, + "entities": { + "urls": [], + "user_mentions": [{ + "name": "Michael Chui", + "indices": [3, 9], + "screen_name": "mchui", + "id_str": "19573968", + "id": 19573968 + }], + "hashtags": [] + }, + "coordinates": null, + "in_reply_to_user_id": null, + "truncated": false, + "text": "RT @mchui: Quoth Niall Ferguson: We no longer have the rule of law; we have the rule of lawyers and that's different." + } + } +} + diff --git a/Collections/Timeline/timeline.js b/Collections/Timeline/timeline.js index 9da1cd95f..7c146c457 100644 --- a/Collections/Timeline/timeline.js +++ b/Collections/Timeline/timeline.js @@ -78,7 +78,6 @@ app.get('/responses/:id', function(req, res) { }); app.get('/id/:id', function(req, res, next) { - if (req.param('id').length != 24) return next(req, res, next); dataStore.getItem(req.param('id'), function(err, doc) { res.send(doc); }); }); From 09cf8ced882a9d8ce8534efb6e604531e4a08629 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Sun, 6 Nov 2011 20:13:46 -0600 Subject: [PATCH 11/40] more timeline fixes and start of a simple viewer --- Apps/HelloTimeline/index.html | 33 +++++++++++++++++++++++++++++++ Apps/HelloTimeline/manifest.app | 8 ++++++++ Apps/HelloTimeline/script.js | 28 ++++++++++++++++++++++++++ Collections/Timeline/dataIn.js | 25 ++++++++++++++--------- Collections/Timeline/dataStore.js | 6 +++++- Collections/Timeline/timeline.js | 16 +++++++++------ 6 files changed, 100 insertions(+), 16 deletions(-) create mode 100644 Apps/HelloTimeline/index.html create mode 100644 Apps/HelloTimeline/manifest.app create mode 100755 Apps/HelloTimeline/script.js diff --git a/Apps/HelloTimeline/index.html b/Apps/HelloTimeline/index.html new file mode 100644 index 000000000..73cdbcc4d --- /dev/null +++ b/Apps/HelloTimeline/index.html @@ -0,0 +1,33 @@ + + + + + + + + Timeline Viewer + + + + + + +

Forever Timeless

+
    + + + + + + + diff --git a/Apps/HelloTimeline/manifest.app b/Apps/HelloTimeline/manifest.app new file mode 100644 index 000000000..7f8f323c1 --- /dev/null +++ b/Apps/HelloTimeline/manifest.app @@ -0,0 +1,8 @@ +{ + "title":"Hello Timeline", + "desc":"A simple way to view your timeline.", + "status":"stable", + "static":"true", + "uses":["facebook", "twitter", "foursquare", "instagram"], + "handle":"hellotimeline" +} \ No newline at end of file diff --git a/Apps/HelloTimeline/script.js b/Apps/HelloTimeline/script.js new file mode 100755 index 000000000..5e89cd0a4 --- /dev/null +++ b/Apps/HelloTimeline/script.js @@ -0,0 +1,28 @@ +var baseUrl = false; + +$(document).ready(function() { + if(baseUrl === false) window.alert("Couldn't find your locker, you might need to add a config.js (see dev.singly.com)"); +}); + +var offset=0; +$(function() { + // be careful with the limit, some people have large datasets ;) + loadPhotos(); + $("#moar").click( function(){ + offset += 50; + loadPhotos(); + }); +}); + +function loadPhotos(){ + $.getJSON(baseUrl + '/Me/timeline/',{limit:50, offset:offset}, function(data) { + if(!data || !data.length) return; + var html = ''; + for(var i in data) + { + var p = data[i]; + html += "" + p.text + '
    '; + } + $("#test").append(html); + }); +} diff --git a/Collections/Timeline/dataIn.js b/Collections/Timeline/dataIn.js index 5596dc1d4..d753369ba 100644 --- a/Collections/Timeline/dataIn.js +++ b/Collections/Timeline/dataIn.js @@ -6,12 +6,20 @@ var url = require('url'); var crypto = require("crypto"); var path = require('path'); -var dataStore, locker; +var dataStore, locker, profiles = {}; // internally we need these for happy fun stuff -exports.init = function(l, dStore){ +exports.init = function(l, dStore, callback){ dataStore = dStore; locker = l; + // load our known profiles, require for foursquare checkins, and to tell "me" flag on everything + async.forEach(["foursquare", "instagram"], function(svc, cb){ + var lurl = locker.lockerBase + '/Me/' + svc + '/getCurrent/profile'; + request.get({uri:lurl, json:true}, function(err, resp, arr){ + if(arr && arr.length > 0) profiles[svc] = arr[1]; + return cb(); + }); + }, callback); } // manually walk and reindex all possible link sources @@ -359,6 +367,7 @@ function itemFacebook(item, post) if(post.comments && post.comments.data) { post.comments.data.forEach(function(comment){ + if(!comment.from) return; // anonymous comments skipped var resp = newResponse(item, "comment"); resp.at = comment.created_time * 1000; resp.text = comment.message; @@ -386,13 +395,11 @@ function itemFoursquare(item, checkin) item.pri = 3; // ideally a checkin should source here as the best item.first = item.last = checkin.createdAt * 1000; if(checkin.venue) item.text = "Checked in at " + checkin.venue.name; - if(checkin.user) - { - item.from.id = 'contact://foursquare/#'+checkin.user.id; - item.from.name = checkin.user.firstName + " " + checkin.user.lastName; - item.from.icon = checkin.user.photo; - item.froms[item.from.id] = item.ref; - } + var profile = (checkin.user) ? checkin.user : profiles["foursquare"]; + item.from.id = 'contact://foursquare/#'+profile.id; + item.from.name = profile.firstName + " " + profile.lastName; + item.from.icon = profile.photo; + item.froms[item.from.id] = item.ref; if(checkin.comments && checkin.comments.items) { checkin.comments.items.forEach(function(comment){ diff --git a/Collections/Timeline/dataStore.js b/Collections/Timeline/dataStore.js index ba35fb1e3..b453077d1 100644 --- a/Collections/Timeline/dataStore.js +++ b/Collections/Timeline/dataStore.js @@ -118,10 +118,14 @@ exports.addItem = function(item, callback) { // responses are unique by their contents exports.addResponse = function(response, callback) { + delete response.id; + delete response._id; // mongo is miss pissypants + var item = response.item; + delete response.item; var hash = crypto.createHash('md5'); hash.update(JSON.stringify(response)); + response.item = item; response.id = hash.digest('hex'); - delete response._id; // mongo is miss pissypants respCol.findAndModify({"id":response.id}, [['_id','asc']], {$set:response}, {safe:true, upsert:true, new: true}, callback); } diff --git a/Collections/Timeline/timeline.js b/Collections/Timeline/timeline.js index 7c146c457..f5a4d04aa 100644 --- a/Collections/Timeline/timeline.js +++ b/Collections/Timeline/timeline.js @@ -27,15 +27,18 @@ app.set('views', __dirname); app.get('/', function(req, res) { var fields = {}; + var sort = {"first":-1}; if (req.query.fields) { - try { - fields = JSON.parse(req.query.fields); - } catch(E) {} + try { fields = JSON.parse(req.query.fields); } catch(E) {} + } + if (req.query.sort) { + try { sort = JSON.parse(req.query.sort); } catch(E) {} } dataStore.getAll(fields, function(err, cursor) { if(!req.query["all"]) cursor.limit(20); // default 20 unless all is set if(req.query["limit"]) cursor.limit(parseInt(req.query["limit"])); if(req.query["offset"]) cursor.skip(parseInt(req.query["offset"])); + cursor.sort(sort); var ndx = {}; cursor.toArray(function(err, items) { if(req.query["all"] || !req.query.full) return res.send(items); // default not include responses, forced if all @@ -132,9 +135,10 @@ process.stdin.on('data', function(data) { locker.connectToMongo(function(mongo) { // initialize all our libs dataStore.init(mongo.collections.item,mongo.collections.response); - dataIn.init(locker, dataStore); - app.listen(lockerInfo.port, 'localhost', function() { - process.stdout.write(data); + dataIn.init(locker, dataStore, function(){ + app.listen(lockerInfo.port, 'localhost', function() { + process.stdout.write(data); + }); }); }); }); From 6e887eaa7f5e8ade4ca187366f34db77b2545c49 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Sun, 6 Nov 2011 23:45:08 -0600 Subject: [PATCH 12/40] better update processing and adding ref resolving --- Collections/Timeline/dataIn.js | 13 ++++++++----- Collections/Timeline/dataStore.js | 5 +++-- Collections/Timeline/timeline.js | 15 +++++++++++++-- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/Collections/Timeline/dataIn.js b/Collections/Timeline/dataIn.js index d753369ba..6c3614608 100644 --- a/Collections/Timeline/dataIn.js +++ b/Collections/Timeline/dataIn.js @@ -16,7 +16,7 @@ exports.init = function(l, dStore, callback){ async.forEach(["foursquare", "instagram"], function(svc, cb){ var lurl = locker.lockerBase + '/Me/' + svc + '/getCurrent/profile'; request.get({uri:lurl, json:true}, function(err, resp, arr){ - if(arr && arr.length > 0) profiles[svc] = arr[1]; + if(arr && arr.length > 0) profiles[svc] = arr[0]; return cb(); }); }, callback); @@ -24,7 +24,7 @@ exports.init = function(l, dStore, callback){ // manually walk and reindex all possible link sources exports.update = function(locker, type, callback) { - dataStore.clear(function(){ + dataStore.clear(type, function(){ callback(); var types = (type) ? [type] : ['home/facebook', 'tweets/twitter', 'recents/foursquare', 'feed/instagram']; locker.providers(types, function(err, services) { @@ -33,7 +33,7 @@ exports.update = function(locker, type, callback) { logger.debug("processing "+svc.id); if(svc.provides.indexOf('home/facebook') >= 0) { getData("home/facebook", svc.id); - } else if(svc.provides.indexOf('status/twitter') >= 0) { + } else if(svc.provides.indexOf('tweets/twitter') >= 0) { getData("tweets/twitter", svc.id); getData("timeline/twitter", svc.id); getData("mentions/twitter", svc.id); @@ -56,11 +56,12 @@ function getData(type, svcId) var subtype = type.substr(0, type.indexOf('/')); var lurl = locker.lockerBase + '/Me/' + svcId + '/getCurrent/' + subtype; request.get({uri:lurl, json:true}, function(err, resp, arr) { + if(err || !arr) return; async.forEachSeries(arr,function(a,cb){ var idr = getIdr(type, svcId, a); masterMaster(idr, a, cb); },function(err){ - logger.debug("processed "+arr.length+" items from "+lurl+" "+(err)?err:""); + logger.debug("processed "+arr.length+" items from "+lurl+" err("+err+")"); }); }); } @@ -207,7 +208,7 @@ function itemRef(item, ref, callback) // intelligently merge two items together and return function itemMerge(older, newer) { - logger.debug("MERGE\t"+JSON.stringify(older)+'\t'+JSON.stringify(newer)); +// logger.debug("MERGE\t"+JSON.stringify(older)+'\t'+JSON.stringify(newer)); if(newer.pri > older.pri) { // replace top level summary stuff if newer is deemed better older.ref = newer.ref; @@ -450,6 +451,8 @@ function itemInstagram(item, pic) var hash = crypto.createHash('md5'); hash.update(item.text.substr(0,130)); // ignore trimming variations item.keys['text:'+hash.digest('hex')] = item.ref; + }else{ + item.text = "New Picture"; } // process responses! diff --git a/Collections/Timeline/dataStore.js b/Collections/Timeline/dataStore.js index b453077d1..f196af1b8 100644 --- a/Collections/Timeline/dataStore.js +++ b/Collections/Timeline/dataStore.js @@ -23,7 +23,8 @@ exports.init = function(iCollection, rCollection) { respCol.ensureIndex({"item":1},{background:true},function() {}); } -exports.clear = function(callback) { +exports.clear = function(flag, callback) { + if(flag) return callback(); itemCol.drop(function(){respCol.drop(callback)}); } @@ -91,7 +92,7 @@ function findWrap(a,b,c,cbEach,cbDone){ if (item != null) { cbEach(item); } else { - cbDone(); + cbDone(err); } }); } diff --git a/Collections/Timeline/timeline.js b/Collections/Timeline/timeline.js index f5a4d04aa..461c4581a 100644 --- a/Collections/Timeline/timeline.js +++ b/Collections/Timeline/timeline.js @@ -14,6 +14,7 @@ var fs = require('fs'), request = require('request'), locker = require('../../Common/node/locker.js'); var async = require("async"); +var url = require("url"); var dataIn = require('./dataIn'); // for processing incoming twitter/facebook/etc data types var dataStore = require("./dataStore"); // storage/retreival of raw items and responses @@ -80,8 +81,18 @@ app.get('/responses/:id', function(req, res) { }); }); -app.get('/id/:id', function(req, res, next) { - dataStore.getItem(req.param('id'), function(err, doc) { res.send(doc); }); +app.get('/id/:id', function(req, res) { + dataStore.getItem(req.param('id'), function(err, doc) { return (err != null || !doc) ? res.send(err, 500) : res.send(doc); }); +}); + +app.get('/ref', function(req, res) { + var idr = url.parse(req.query.id); + if(!idr || !idr.hash) return res.send("missing or invalid id",500); + var lurl = locker.lockerBase + '/Me/' + idr.host + idr.pathname + '/id/' + idr.hash.substr(1); + request.get({url:lurl, json:true}, function(err, res2, body){ + if(err || !body) return res.send(err, 500); + res.send(body); + }); }); app.get('/update', function(req, res) { From d27ed1c1993464f4344ff1d88f98b208230a12dd Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Sun, 6 Nov 2011 23:45:40 -0600 Subject: [PATCH 13/40] when a bad response happens, handle the error --- Ops/webservice.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Ops/webservice.js b/Ops/webservice.js index f0b0a43d0..84051c7d4 100644 --- a/Ops/webservice.js +++ b/Ops/webservice.js @@ -309,6 +309,12 @@ function proxyRequest(method, req, res) { if (syncManager.isInstalled(id)) { var u = 'http://' + lconfig.lockerHost + ':' + lconfig.lockerPort + '/' + path.join('synclets', id, ppath); return request({method:method, uri:u, headers:req.headers, encoding:'binary'}, function(err, res2, body){ + if(err) + { + res.writeHead(500); + res.end("failed to get "+id+" "+ppath); + return; + } res.writeHead(res2.statusCode, res2.headers); res.write(body, "binary"); res.end(); From 2a13169102015484d77349ea7c0ef9a8124692ed Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Mon, 7 Nov 2011 00:23:40 -0600 Subject: [PATCH 14/40] ooh, 4sq shouts --- Collections/Timeline/dataIn.js | 1 + 1 file changed, 1 insertion(+) diff --git a/Collections/Timeline/dataIn.js b/Collections/Timeline/dataIn.js index 6c3614608..0820719a8 100644 --- a/Collections/Timeline/dataIn.js +++ b/Collections/Timeline/dataIn.js @@ -396,6 +396,7 @@ function itemFoursquare(item, checkin) item.pri = 3; // ideally a checkin should source here as the best item.first = item.last = checkin.createdAt * 1000; if(checkin.venue) item.text = "Checked in at " + checkin.venue.name; + if(checkin.shout) item.text = checkin.shout; var profile = (checkin.user) ? checkin.user : profiles["foursquare"]; item.from.id = 'contact://foursquare/#'+profile.id; item.from.name = profile.firstName + " " + profile.lastName; From d0cc40499c3411b71e9a745cd10938e888151a2d Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Mon, 7 Nov 2011 00:24:15 -0600 Subject: [PATCH 15/40] take both kinds of ids --- Common/node/synclet/datastore.js | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/Common/node/synclet/datastore.js b/Common/node/synclet/datastore.js index 43b5b8b42..bbdeb93c7 100644 --- a/Common/node/synclet/datastore.js +++ b/Common/node/synclet/datastore.js @@ -15,9 +15,12 @@ var IJOD = require('../ijod').IJOD , colls , mongoIDs = {} ; +var syncManager = require('lsyncmanager'); +var synclets; exports.init = function(callback) { if (mongo) return callback(); + synclets = syncManager.synclets(); lmongo.init('synclets', [], function(_mongo) { mongo = _mongo; colls = mongo.collections.synclets; @@ -98,8 +101,16 @@ exports.getAllCurrent = function(type, callback, options) { exports.getCurrent = function(type, id, callback) { if (!(id && (typeof id === 'string' || typeof id === 'number'))) return callback(new Error('bad id:' + id), null); var m = getMongo(type, callback); - var query = {_id: mongo.db.bson_serializer.ObjectID(id)}; - m.findOne(query, callback); + var or = [{_id: mongo.db.bson_serializer.ObjectID(id)}]; + var parts = type.split("_"); + var idname = "id"; + // this is some crazy shit, serious refactoring needed someday + if(parts.length == 2 && synclets && synclets[parts[0]] && synclets[parts[0]].mongoId && synclets[parts[0]].mongoId[parts[1]+'s']) idname = synclets[parts[0]].mongoId[parts[1]+'s']; + var id2 = {}; + id2[idname] = id; + or.push(id2); + console.error("querying "+JSON.stringify(or)); + m.findOne({$or: or}, callback); } function setCurrent(type, object, callback) { From eaf49d80069de11d5389ef5aeeb60d6d25d905c0 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Mon, 7 Nov 2011 01:35:46 -0600 Subject: [PATCH 16/40] sort default by time, and handle ids better --- Common/node/synclet/dataaccess.js | 2 +- Common/node/synclet/datastore.js | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Common/node/synclet/dataaccess.js b/Common/node/synclet/dataaccess.js index dcb9ddff7..0f1c93b8e 100644 --- a/Common/node/synclet/dataaccess.js +++ b/Common/node/synclet/dataaccess.js @@ -81,7 +81,7 @@ module.exports = function(app) { res.end(JSON.stringify(doc)); } else { res.writeHead(404); - res.end(); + res.end("not found"); } }); }); diff --git a/Common/node/synclet/datastore.js b/Common/node/synclet/datastore.js index bbdeb93c7..d314d1d1b 100644 --- a/Common/node/synclet/datastore.js +++ b/Common/node/synclet/datastore.js @@ -95,13 +95,16 @@ exports.queryCurrent = function(type, query, options, callback) { exports.getAllCurrent = function(type, callback, options) { options = options || {}; var m = getMongo(type, callback); - m.find({}, options).toArray(callback); + m.find({}, options).sort({_id:-1}).toArray(callback); } exports.getCurrent = function(type, id, callback) { if (!(id && (typeof id === 'string' || typeof id === 'number'))) return callback(new Error('bad id:' + id), null); var m = getMongo(type, callback); - var or = [{_id: mongo.db.bson_serializer.ObjectID(id)}]; + var or = []; + try { + or.push({_id: mongo.db.bson_serializer.ObjectID(id)}); + }catch(E){} var parts = type.split("_"); var idname = "id"; // this is some crazy shit, serious refactoring needed someday From 66f4a236fba4526cf77fe11e40844d740fc81e5f Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Mon, 7 Nov 2011 01:36:09 -0600 Subject: [PATCH 17/40] more bugfixes, and process links on an update --- Collections/Timeline/dataIn.js | 43 +++++++++++++++++++++---------- Collections/Timeline/dataStore.js | 1 + 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/Collections/Timeline/dataIn.js b/Collections/Timeline/dataIn.js index 0820719a8..ba1eed0a5 100644 --- a/Collections/Timeline/dataIn.js +++ b/Collections/Timeline/dataIn.js @@ -29,32 +29,38 @@ exports.update = function(locker, type, callback) { var types = (type) ? [type] : ['home/facebook', 'tweets/twitter', 'recents/foursquare', 'feed/instagram']; locker.providers(types, function(err, services) { if (!services) return; - services.forEach(function(svc) { + async.forEachSeries(services, function(svc, cb) { logger.debug("processing "+svc.id); if(svc.provides.indexOf('home/facebook') >= 0) { - getData("home/facebook", svc.id); + getData("home/facebook", svc.id, cb); } else if(svc.provides.indexOf('tweets/twitter') >= 0) { - getData("tweets/twitter", svc.id); - getData("timeline/twitter", svc.id); - getData("mentions/twitter", svc.id); - getData("related/twitter", svc.id); + async.forEachSeries(["tweets/twitter", "timeline/twitter", "mentions/twitter", "related/twitter"], function(type, cb2){ getData(type, svc.id, cb2) }, cb); } else if(svc.provides.indexOf('checkin/foursquare') >= 0) { - getData("recents/foursquare", svc.id); - getData("checkin/foursquare", svc.id); + async.forEachSeries(["recents/foursquare", "checkin/foursquare"], function(type, cb2){ getData(type, svc.id, cb2) }, cb); } else if(svc.provides.indexOf('feed/instagram') >= 0) { - getData("photo/instagram", svc.id); - getData("feed/instagram", svc.id); + async.forEachSeries(["photo/instagram", "feed/instagram"], function(type, cb2){ getData(type, svc.id, cb2) }, cb); } + }, function(err){ + // process links too + request.get({uri:locker.lockerBase+'/Me/links/?full=true&limit=100', json:true}, function(err, resp, links){ + if(err || !links) return; + async.forEachSeries(links, function(link, cb){ + var evt = {obj:{data:link}}; + processLink(evt, cb); + }, function(){ + logger.debug("done with update"); + }); + }) }); }); }); } // go fetch data from sources to bulk process -function getData(type, svcId) +function getData(type, svcId, callback) { var subtype = type.substr(0, type.indexOf('/')); - var lurl = locker.lockerBase + '/Me/' + svcId + '/getCurrent/' + subtype; + var lurl = locker.lockerBase + '/Me/' + svcId + '/getCurrent/' + subtype + "?limit=100"; request.get({uri:lurl, json:true}, function(err, resp, arr) { if(err || !arr) return; async.forEachSeries(arr,function(a,cb){ @@ -62,6 +68,7 @@ function getData(type, svcId) masterMaster(idr, a, cb); },function(err){ logger.debug("processed "+arr.length+" items from "+lurl+" err("+err+")"); + callback(err); }); }); } @@ -177,6 +184,7 @@ function masterMaster(idr, data, callback) cb(); }); }, function (err) { + if(!item.pri && !dup) return callback(); // some new items, like twitter related, are only for merging, shouldn't happen but if it does we may need to queue/stash or something if(dup) item = itemMerge(dup, item); dataStore.addItem(item, function(err, item){ if(err) return callback(err); @@ -291,6 +299,8 @@ function newResponse(item, type) // extract info from a tweet function itemTwitter(item, tweet) { + item.pri = 1; // tweets are the lowest priority? + // since RTs contain the original, add the response first then process the original! if(tweet.retweeted_status) { @@ -303,7 +313,6 @@ function itemTwitter(item, tweet) item.keys['tweet://twitter/#'+tweet.id_str] = item.ref; // tag with the original too } - item.pri = 1; // tweets are the lowest priority? if(tweet.created_at) item.first = item.last = new Date(tweet.created_at).getTime(); if(tweet.text) item.text = tweet.text; if(tweet.user) @@ -396,7 +405,13 @@ function itemFoursquare(item, checkin) item.pri = 3; // ideally a checkin should source here as the best item.first = item.last = checkin.createdAt * 1000; if(checkin.venue) item.text = "Checked in at " + checkin.venue.name; - if(checkin.shout) item.text = checkin.shout; + if(checkin.shout) + { + item.text = checkin.shout; + var hash = crypto.createHash('md5'); + hash.update(item.text.substr(0,130)); // ignore trimming variations + item.keys['text:'+hash.digest('hex')] = item.ref; + } var profile = (checkin.user) ? checkin.user : profiles["foursquare"]; item.from.id = 'contact://foursquare/#'+profile.id; item.from.name = profile.firstName + " " + profile.lastName; diff --git a/Collections/Timeline/dataStore.js b/Collections/Timeline/dataStore.js index f196af1b8..6f620a2ec 100644 --- a/Collections/Timeline/dataStore.js +++ b/Collections/Timeline/dataStore.js @@ -18,6 +18,7 @@ var itemCol, respCol; exports.init = function(iCollection, rCollection) { itemCol = iCollection; itemCol.ensureIndex({"id":1},{unique:true, background:true},function() {}); + itemCol.ensureIndex({"keys":1},{background:true},function() {}); respCol = rCollection; respCol.ensureIndex({"id":1},{unique:true, background:true},function() {}); respCol.ensureIndex({"item":1},{background:true},function() {}); From 8deab93d024a94bb922cdd0d0d97755298c4249a Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Mon, 7 Nov 2011 01:36:25 -0600 Subject: [PATCH 18/40] simple viewer updates --- Apps/HelloTimeline/script.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Apps/HelloTimeline/script.js b/Apps/HelloTimeline/script.js index 5e89cd0a4..2ef031790 100755 --- a/Apps/HelloTimeline/script.js +++ b/Apps/HelloTimeline/script.js @@ -21,7 +21,7 @@ function loadPhotos(){ for(var i in data) { var p = data[i]; - html += "" + p.text + '
    '; + html += "("+p.refs.length+":"+p.comments+":"+p.ups+") "+p.from.name+": " + p.text + ' .js
    '; } $("#test").append(html); }); From 18196af94bb89d017549a3043b85ffb92b98a293 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Mon, 14 Nov 2011 00:28:36 -0600 Subject: [PATCH 19/40] wasnt handling real link events properly, fixed --- Collections/Timeline/dataIn.js | 8 +++++--- Collections/Timeline/fixtures/link | 4 ---- Collections/Timeline/timeline.js | 9 +++++---- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/Collections/Timeline/dataIn.js b/Collections/Timeline/dataIn.js index ba1eed0a5..7d51dc403 100644 --- a/Collections/Timeline/dataIn.js +++ b/Collections/Timeline/dataIn.js @@ -258,15 +258,17 @@ function itemMergeHard(a, b, cb) // when a processed link event comes in, check to see if it's a long url that could help us de-dup function processLink(event, callback) { - if(!event || !event.obj || !event.obj.data || !event.obj.data.encounters) return callback("no encounter"); + if(!event || !event.obj || !event.obj.encounters) return callback("no encounter"); // process each encounter if there's multiple - async.forEach(event.obj.data.encounters, function(encounter, cb){ + async.forEach(event.obj.encounters, function(encounter, cb){ // first, look up event via/orig key and see if we've processed it yet, if not (for some reason) ignore var key = url.format(getKey(encounter.network, encounter.via)); dataStore.getItemByKey(key,function(err, item){ if(err || !item) return cb(err); // we tag this item with a ref to the link, super handy for cross-collection mashups - itemRef(item, "link://links/#"+event.obj.data._id, function(err, item){ + var hash = crypto.createHash('md5'); + hash.update(encounter.link); // turn link into hash as an id + itemRef(item, "link://links/#"+hash.digest('hex'), function(err, item){ if(err || !item) return cb(err); var u = url.parse(encounter.link); // if foursquare checkin and from a tweet, generate foursquare key and look for it diff --git a/Collections/Timeline/fixtures/link b/Collections/Timeline/fixtures/link index 4fbec83a6..6b96bea82 100644 --- a/Collections/Timeline/fixtures/link +++ b/Collections/Timeline/fixtures/link @@ -4,9 +4,6 @@ "timestamp": 1315976215195, "action": "new", "obj": { - "type": "new", - "data": { - "_id": "4eb0f60f8cc09f2507e66440", "at": 1320220061000, "favicon": "http://t.co/favicon.ico", "link": "http://t.co/QgLsBzHy", @@ -101,5 +98,4 @@ } }] } - } } diff --git a/Collections/Timeline/timeline.js b/Collections/Timeline/timeline.js index 461c4581a..2dce5ad1a 100644 --- a/Collections/Timeline/timeline.js +++ b/Collections/Timeline/timeline.js @@ -103,7 +103,7 @@ app.get('/update', function(req, res) { }); app.post('/events', function(req, res) { - if (!req.body.type || !req.body.obj || !req.body.obj.data){ + if (!req.body.type || !req.body.obj){ console.log('5 HUNDO bad data:',JSON.stringify(req.body)); res.writeHead(500); res.end('bad data'); @@ -111,9 +111,10 @@ app.post('/events', function(req, res) { } // handle asyncadilly - dataIn.processEvent(req.body); - res.writeHead(200); - res.end('ok'); + dataIn.processEvent(req.body, function(err){ + if(err) return res.send(err, 500); + res.send('ok'); + }); }); function genericApi(name,f) From d8098abc9137fd4fe64e5e41ba0689d1e23df1bb Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Thu, 17 Nov 2011 22:33:30 -0600 Subject: [PATCH 20/40] tiny safety check --- Collections/Timeline/dataIn.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Collections/Timeline/dataIn.js b/Collections/Timeline/dataIn.js index 7d51dc403..050a0ff16 100644 --- a/Collections/Timeline/dataIn.js +++ b/Collections/Timeline/dataIn.js @@ -39,6 +39,8 @@ exports.update = function(locker, type, callback) { async.forEachSeries(["recents/foursquare", "checkin/foursquare"], function(type, cb2){ getData(type, svc.id, cb2) }, cb); } else if(svc.provides.indexOf('feed/instagram') >= 0) { async.forEachSeries(["photo/instagram", "feed/instagram"], function(type, cb2){ getData(type, svc.id, cb2) }, cb); + } else { + cb(); } }, function(err){ // process links too @@ -330,7 +332,7 @@ function itemTwitter(item, tweet) var hash = crypto.createHash('md5'); hash.update(item.text.substr(0,130)); // ignore trimming variations item.keys['text:'+hash.digest('hex')] = item.ref; - var hash = crypto.createHash('md5'); + hash = crypto.createHash('md5'); hash.update(item.text.replace(/ http\:\/\/\S+$/,"")); // cut off appendege links that some apps add (like instagram) item.keys['text:'+hash.digest('hex')] = item.ref; } @@ -440,7 +442,7 @@ function itemTwitterRelated(item, relateds) resp.type = "comment"; resp.text = result.value.text; user = result.value.user; - resp.at = new Date(result.value.created_at).getTime() + resp.at = new Date(result.value.created_at).getTime(); } resp.from.id = "contact://twitter/#"+user.screen_name; resp.from.name = user.name; From 75f5753c8a2c724149e8e823376e6dca48a3611d Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Thu, 17 Nov 2011 23:15:19 -0600 Subject: [PATCH 21/40] trying to add some more query options to test --- Collections/Timeline/dataIn.js | 2 +- Collections/Timeline/timeline.js | 15 ++++++++++----- Common/node/synclet/dataaccess.js | 9 +++++++++ 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/Collections/Timeline/dataIn.js b/Collections/Timeline/dataIn.js index 050a0ff16..9663da3ec 100644 --- a/Collections/Timeline/dataIn.js +++ b/Collections/Timeline/dataIn.js @@ -62,7 +62,7 @@ exports.update = function(locker, type, callback) { function getData(type, svcId, callback) { var subtype = type.substr(0, type.indexOf('/')); - var lurl = locker.lockerBase + '/Me/' + svcId + '/getCurrent/' + subtype + "?limit=100"; + var lurl = locker.lockerBase + '/Me/' + svcId + '/getCurrent/' + subtype + "?limit=100&sort=_id&order=-1"; request.get({uri:lurl, json:true}, function(err, resp, arr) { if(err || !arr) return; async.forEachSeries(arr,function(a,cb){ diff --git a/Collections/Timeline/timeline.js b/Collections/Timeline/timeline.js index 2dce5ad1a..b71e496e6 100644 --- a/Collections/Timeline/timeline.js +++ b/Collections/Timeline/timeline.js @@ -28,18 +28,23 @@ app.set('views', __dirname); app.get('/', function(req, res) { var fields = {}; - var sort = {"first":-1}; if (req.query.fields) { try { fields = JSON.parse(req.query.fields); } catch(E) {} } - if (req.query.sort) { - try { sort = JSON.parse(req.query.sort); } catch(E) {} - } dataStore.getAll(fields, function(err, cursor) { if(!req.query["all"]) cursor.limit(20); // default 20 unless all is set if(req.query["limit"]) cursor.limit(parseInt(req.query["limit"])); if(req.query["offset"]) cursor.skip(parseInt(req.query["offset"])); - cursor.sort(sort); + var sorter = {"first":-1}; + if(req.query["sort"]) { + if(req.query["order"]) { + sorter[req.query["sort"]] = +req.query["order"]; + } else { + sorter[req.query["sort"]] = 1; + } + } + console.log("SORTING "+JSON.stringify(sorter)); + cursor.sort(sorter); var ndx = {}; cursor.toArray(function(err, items) { if(req.query["all"] || !req.query.full) return res.send(items); // default not include responses, forced if all diff --git a/Common/node/synclet/dataaccess.js b/Common/node/synclet/dataaccess.js index 4dc2cf9e6..c2e5bbe14 100644 --- a/Common/node/synclet/dataaccess.js +++ b/Common/node/synclet/dataaccess.js @@ -14,6 +14,15 @@ module.exports = function(app) { var options = {}; if(req.query['limit']) options.limit = parseInt(req.query['limit']); if(req.query['offset']) options.skip = parseInt(req.query['offset']); + if(req.query["sort"]) { + var sorter = {} + if(req.query["order"]) { + sorter[req.query["sort"]] = +req.query["order"]; + } else { + sorter[req.query["sort"]] = 1; + } + options.sort = sorter; + } dataStore.getAllCurrent('synclets', req.params.syncletId + "_" + req.params.type, function(err, objects) { if (err) { From 03a2638ad462531618852ae241daae4b688f1b94 Mon Sep 17 00:00:00 2001 From: Simon Murtha-Smith Date: Thu, 15 Dec 2011 22:14:08 -0800 Subject: [PATCH 22/40] updated Timeline collection to handle idrs --- Collections/Timeline/dataIn.js | 6 +++--- Collections/Timeline/timeline.js | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Collections/Timeline/dataIn.js b/Collections/Timeline/dataIn.js index 9663da3ec..a83e4f5ce 100644 --- a/Collections/Timeline/dataIn.js +++ b/Collections/Timeline/dataIn.js @@ -144,12 +144,12 @@ function getKey(network, data) exports.processEvent = function(event, callback) { if(!callback) callback = function(err){if(err) console.error(err);}; + var idr = url.parse(event.idr, true); // handle links as a special case as we're using them for post-process-deduplication - if(event.type == 'link') return processLink(event, callback); + if(idr.protocol == 'link') return processLink(event, callback); - var idr = getIdr(event.type, event.via, event.obj.data); if(!idr.protocol) return callback("don't understand this data"); - masterMaster(idr, event.obj.data, callback); + masterMaster(idr, event.data, callback); } function isItMe(idr) diff --git a/Collections/Timeline/timeline.js b/Collections/Timeline/timeline.js index b71e496e6..2cff512b0 100644 --- a/Collections/Timeline/timeline.js +++ b/Collections/Timeline/timeline.js @@ -12,7 +12,9 @@ var fs = require('fs'), url = require('url'), request = require('request'), - locker = require('../../Common/node/locker.js'); + locker = require('../../Common/node/locker.js'), + lconfig = require('../../Common/node/lconfig'); +lconfig.load('../../Config/config.json'); var async = require("async"); var url = require("url"); @@ -108,7 +110,7 @@ app.get('/update', function(req, res) { }); app.post('/events', function(req, res) { - if (!req.body.type || !req.body.obj){ + if (!req.body.idr || !req.body.data){ console.log('5 HUNDO bad data:',JSON.stringify(req.body)); res.writeHead(500); res.end('bad data'); From cc4e6f71503ead28b319281aa3faabebb5cf97da Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Wed, 21 Dec 2011 20:08:49 -0600 Subject: [PATCH 23/40] tweak --- Collections/Timeline/timeline.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Collections/Timeline/timeline.js b/Collections/Timeline/timeline.js index b71e496e6..56445e52c 100644 --- a/Collections/Timeline/timeline.js +++ b/Collections/Timeline/timeline.js @@ -37,16 +37,15 @@ app.get('/', function(req, res) { if(req.query["offset"]) cursor.skip(parseInt(req.query["offset"])); var sorter = {"first":-1}; if(req.query["sort"]) { + sorter = {}; if(req.query["order"]) { sorter[req.query["sort"]] = +req.query["order"]; } else { - sorter[req.query["sort"]] = 1; + sorter[req.query["sort"]]= 1; } } - console.log("SORTING "+JSON.stringify(sorter)); - cursor.sort(sorter); var ndx = {}; - cursor.toArray(function(err, items) { + cursor.sort(sorter).toArray(function(err, items) { if(req.query["all"] || !req.query.full) return res.send(items); // default not include responses, forced if all items.forEach(function(item){ ndx[item.id] = item; item.responses = []; }); // build index var arg = {"item":{$in: Object.keys(ndx)}}; From af6fabcf02c900552d0d6ab0959d279d9af4d8a6 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Mon, 26 Dec 2011 11:30:32 -0600 Subject: [PATCH 24/40] general cleanups --- Collections/Timeline/dataIn.js | 54 +++++++++--------------- Collections/Timeline/dataStore.js | 1 - Collections/Timeline/timeline.collection | 10 ++--- Collections/Timeline/timeline.js | 3 +- 4 files changed, 27 insertions(+), 41 deletions(-) diff --git a/Collections/Timeline/dataIn.js b/Collections/Timeline/dataIn.js index a83e4f5ce..1f768f353 100644 --- a/Collections/Timeline/dataIn.js +++ b/Collections/Timeline/dataIn.js @@ -1,6 +1,6 @@ var request = require('request'); var async = require('async'); -var logger = require(__dirname + "/../../Common/node/logger").logger; +var logger = require(__dirname + "/../../Common/node/logger"); var lutil = require('lutil'); var url = require('url'); var crypto = require("crypto"); @@ -26,7 +26,7 @@ exports.init = function(l, dStore, callback){ exports.update = function(locker, type, callback) { dataStore.clear(type, function(){ callback(); - var types = (type) ? [type] : ['home/facebook', 'tweets/twitter', 'recents/foursquare', 'feed/instagram']; + var types = (type) ? [type] : ['home/facebook', 'tweets/twitter', 'checkin/foursquare', 'feed/instagram']; locker.providers(types, function(err, services) { if (!services) return; async.forEachSeries(services, function(svc, cb) { @@ -44,15 +44,12 @@ exports.update = function(locker, type, callback) { } }, function(err){ // process links too - request.get({uri:locker.lockerBase+'/Me/links/?full=true&limit=100', json:true}, function(err, resp, links){ - if(err || !links) return; - async.forEachSeries(links, function(link, cb){ - var evt = {obj:{data:link}}; - processLink(evt, cb); - }, function(){ - logger.debug("done with update"); - }); - }) + lutil.streamFromUrl(locker.lockerBase+'/Me/links/?full=true&limit=100&stream=true', function(link, cb){ + processLink({data:link}, cb); + }, function(err){ + if(err) logger.error(err); + logger.debug("done with update"); + }); }); }); }); @@ -62,16 +59,15 @@ exports.update = function(locker, type, callback) { function getData(type, svcId, callback) { var subtype = type.substr(0, type.indexOf('/')); - var lurl = locker.lockerBase + '/Me/' + svcId + '/getCurrent/' + subtype + "?limit=100&sort=_id&order=-1"; - request.get({uri:lurl, json:true}, function(err, resp, arr) { - if(err || !arr) return; - async.forEachSeries(arr,function(a,cb){ - var idr = getIdr(type, svcId, a); - masterMaster(idr, a, cb); - },function(err){ - logger.debug("processed "+arr.length+" items from "+lurl+" err("+err+")"); - callback(err); - }); + var lurl = locker.lockerBase + '/Me/' + svcId + '/getCurrent/' + subtype + "?limit=100&sort=_id&order=-1&stream=true"; + var tot = 0; + lutil.streamFromUrl(lurl, function(a, cb){ + tot++; + var idr = getIdr(type, svcId, a); + masterMaster(idr, a, cb); + }, function(err){ + logger.debug("processed "+tot+" items from "+lurl); + callback(err); }); } @@ -143,34 +139,24 @@ function getKey(network, data) // normalize events a bit exports.processEvent = function(event, callback) { - if(!callback) callback = function(err){if(err) console.error(err);}; + if(!callback) callback = function(err){if(err) logger.error(err);}; var idr = url.parse(event.idr, true); + if(!idr || !idr.protocol) return callback("don't understand this data"); // handle links as a special case as we're using them for post-process-deduplication if(idr.protocol == 'link') return processLink(event, callback); - - if(!idr.protocol) return callback("don't understand this data"); masterMaster(idr, event.data, callback); } -function isItMe(idr) -{ - if(idr.protocol == 'tweet:' && idr.pathname == '/tweets') return true; - if(idr.protocol == 'checkin:' && idr.pathname == '/checkin') return true; - if(idr.protocol == 'photo:' && idr.pathname == '/photo') return true; - return false; -} - // figure out what to do with any data function masterMaster(idr, data, callback) { - if(typeof data != 'object') return callback(); + if(typeof data != 'object') return callback("missing or bad data"); // logger.debug("MM\t"+url.format(idr)); var ref = url.format(idr); var item = {keys:{}, refs:[], froms:{}, from:{}, responses:[], first:new Date().getTime(), last:new Date().getTime()}; item.ref = ref; item.refs.push(ref); item.keys[url.format(idr2key(idr, data))] = item.ref; - item.me = isItMe(idr); if(idr.protocol == 'tweet:'){ if(data.user && data.text) itemTwitter(item, data); if(data.related) itemTwitterRelated(item, data.related); diff --git a/Collections/Timeline/dataStore.js b/Collections/Timeline/dataStore.js index 6f620a2ec..3a05aa6e6 100644 --- a/Collections/Timeline/dataStore.js +++ b/Collections/Timeline/dataStore.js @@ -12,7 +12,6 @@ var async = require('async'); var lmongoutil = require("lmongoutil"); -// in the future we'll probably need a visitCollection too var itemCol, respCol; exports.init = function(iCollection, rCollection) { diff --git a/Collections/Timeline/timeline.collection b/Collections/Timeline/timeline.collection index fb052ce52..a4927130e 100644 --- a/Collections/Timeline/timeline.collection +++ b/Collections/Timeline/timeline.collection @@ -7,9 +7,9 @@ "provides":["item"], "mongoCollections": ["item","response"], "events":[["recents/foursquare","/events"] - ,["timeline/twitter","/events"] - ,["related/twitter","/events"] - ,["home/facebook","/events"] - ,["link","/events"] - ] + ,["tweet/twitter","/events"] + ,["related/twitter","/events"] + ,["post/facebook","/events"] + ,["link","/events"] + ] } diff --git a/Collections/Timeline/timeline.js b/Collections/Timeline/timeline.js index 9ee38237c..afdc0a9e0 100644 --- a/Collections/Timeline/timeline.js +++ b/Collections/Timeline/timeline.js @@ -15,6 +15,7 @@ var fs = require('fs'), locker = require('../../Common/node/locker.js'), lconfig = require('../../Common/node/lconfig'); lconfig.load('../../Config/config.json'); +var logger = require('logger'); var async = require("async"); var url = require("url"); @@ -110,7 +111,7 @@ app.get('/update', function(req, res) { app.post('/events', function(req, res) { if (!req.body.idr || !req.body.data){ - console.log('5 HUNDO bad data:',JSON.stringify(req.body)); + logger.error('5 HUNDO bad data:',JSON.stringify(req.body)); res.writeHead(500); res.end('bad data'); return; From 754809c59ee7123ca15eba471fcba42c3fed97df Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Tue, 27 Dec 2011 14:37:30 -0600 Subject: [PATCH 25/40] converting fixtures to new event format --- Collections/Timeline/fixtures/facebook.tweet | 46 +- Collections/Timeline/fixtures/facebook.update | 86 +- .../Timeline/fixtures/foursquare.recents | 103 +- Collections/Timeline/fixtures/ig.fb | 58 +- Collections/Timeline/fixtures/ig.instagram | 111 +- Collections/Timeline/fixtures/ig.tweet | 148 +- Collections/Timeline/fixtures/instagram.feed | 111 +- Collections/Timeline/fixtures/tweet.facebook | 157 +- Collections/Timeline/fixtures/twitter.related | 2428 ++++++++--------- Collections/Timeline/fixtures/twitter.reply | 144 +- .../Timeline/fixtures/twitter.reply.orig | 177 +- Collections/Timeline/fixtures/twitter.rt | 223 +- .../Timeline/fixtures/twitter.timeline | 160 +- 13 files changed, 2124 insertions(+), 1828 deletions(-) diff --git a/Collections/Timeline/fixtures/facebook.tweet b/Collections/Timeline/fixtures/facebook.tweet index 3fd5b42ff..20c66c565 100644 --- a/Collections/Timeline/fixtures/facebook.tweet +++ b/Collections/Timeline/fixtures/facebook.tweet @@ -1,20 +1,16 @@ { - "type": "home/facebook", - "via": "synclet/facebook", - "timestamp": 1315976215195, "action": "update", - "obj": - { - "timeStamp": "2011-09-20T14:09:19.051Z", - "data": { - "id": "500012126_10150282386392127", - "from": { - "name": "Alex Rainert", - "id": "500012126" - }, - "message": "Dan was the 1st to nail the Google/Motorola news and yesterday he nailed Netflix. Get on the @splatf analysis, people! http://t.co/ZMu8UkyK", - "icon": "http://photos-d.ak.fbcdn.net/photos-ak-snc1/v27562/23/2231777543/app_2_2231777543_9553.gif", - "actions": [{ + "idr": "post://facebook/home?id=facebook#500012126_10150282386392127", + "data": { + "id": "500012126_10150282386392127", + "from": { + "name": "Alex Rainert", + "id": "500012126" + }, + "message": "Dan was the 1st to nail the Google/Motorola news and yesterday he nailed Netflix. Get on the @splatf analysis, people! http://t.co/ZMu8UkyK", + "icon": "http://photos-d.ak.fbcdn.net/photos-ak-snc1/v27562/23/2231777543/app_2_2231777543_9553.gif", + "actions": [ + { "name": "Comment", "link": "http://www.facebook.com/500012126/posts/10150282386392127" }, @@ -25,17 +21,17 @@ { "name": "@arainert on Twitter", "link": "http://twitter.com/arainert?utm_source=fb&utm_medium=fb&utm_campaign=arainert&utm_content=116149996419682304" - }], - "type": "status", - "application": { - "name": "Twitter", - "id": "2231777543" - }, - "created_time": 1316527292, - "updated_time": 1316527292, - "comments": { - "count": 0 } + ], + "type": "status", + "application": { + "name": "Twitter", + "id": "2231777543" + }, + "created_time": 1316527292, + "updated_time": 1316527292, + "comments": { + "count": 0 } } } \ No newline at end of file diff --git a/Collections/Timeline/fixtures/facebook.update b/Collections/Timeline/fixtures/facebook.update index 0887a2259..75c226222 100644 --- a/Collections/Timeline/fixtures/facebook.update +++ b/Collections/Timeline/fixtures/facebook.update @@ -1,51 +1,53 @@ { - "type": "home/facebook", - "via": "synclet/facebook", - "timestamp": 1315976215195, "action": "update", - "obj": { - "source": "facebook_home", - "type": "update", - "data": { - "id": "1264086533_2419826536086", - "from": { - "name": "Mindframe Theaters", - "id": "1264086533" - }, - "picture": "http://vthumb.ak.fbcdn.net/hvthumb-ak-snc6/244520_2419869537161_2419826536086_33311_1352_t.jpg", - "link": "http://www.facebook.com/photo.php?v=2419826536086", - "source": "http://video.l3.fbcdn.net/cfs-l3-ash4/342015/380/2419826536086_60821.mp4?oh=115eecc26ea0e77e74bdfb39d3860a33&oe=4E72D400&l3s=20110914043344&l3e=20110916044344&lh=065388ed3f72cef427b4b", - "name": "The Tree of Life - 9/16 [HD]", - "properties": [{ + "idr": "post://facebook/home?id=facebook#1264086533_2419826536086", + "data": { + "id": "1264086533_2419826536086", + "from": { + "name": "Mindframe Theaters", + "id": "1264086533" + }, + "picture": "http://vthumb.ak.fbcdn.net/hvthumb-ak-snc6/244520_2419869537161_2419826536086_33311_1352_t.jpg", + "link": "http://www.facebook.com/photo.php?v=2419826536086", + "source": "http://video.l3.fbcdn.net/cfs-l3-ash4/342015/380/2419826536086_60821.mp4?oh=115eecc26ea0e77e74bdfb39d3860a33&oe=4E72D400&l3s=20110914043344&l3e=20110916044344&lh=065388ed3f72cef427b4b", + "name": "The Tree of Life - 9/16 [HD]", + "properties": [ + { "name": "Length", "text": "2:11" - }], - "icon": "http://static.ak.fbcdn.net/rsrc.php/v1/yD/r/DggDhA4z4tO.gif", - "actions": [{ + } + ], + "icon": "http://static.ak.fbcdn.net/rsrc.php/v1/yD/r/DggDhA4z4tO.gif", + "actions": [ + { "name": "Comment", "link": "http://www.facebook.com/1264086533/posts/2419826536086" }, { "name": "Like", "link": "http://www.facebook.com/1264086533/posts/2419826536086" - }], - "type": "video", - "object_id": "2419826536086", - "application": { - "name": "Video", - "id": "2392950137" - }, - "created_time": 1315859039, - "updated_time": 1315864735, - "likes": { - "data": [{ + } + ], + "type": "video", + "object_id": "2419826536086", + "application": { + "name": "Video", + "id": "2392950137" + }, + "created_time": 1315859039, + "updated_time": 1315864735, + "likes": { + "data": [ + { "name": "Will Kelly", "id": "760380150" - }], - "count": 9 - }, - "comments": { - "data": [{ + } + ], + "count": 9 + }, + "comments": { + "data": [ + { "id": "1264086533_2419826536086_3034673", "from": { "name": "Bryan J. Zygmont", @@ -53,10 +55,10 @@ }, "message": "Thank you. See you soon.", "created_time": 1315864735 - }], - "count": 1 - }, - "_id": "4e6e6d354f7f2ab135d2782b" - } + } + ], + "count": 1 + }, + "_id": "4e6e6d354f7f2ab135d2782b" } -} +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/foursquare.recents b/Collections/Timeline/fixtures/foursquare.recents index 9e7087fa8..020e07810 100644 --- a/Collections/Timeline/fixtures/foursquare.recents +++ b/Collections/Timeline/fixtures/foursquare.recents @@ -1,65 +1,64 @@ { - "type": "recents/foursquare", - "via": "synclet/foursquare", - "timestamp": 1315977793913, "action": "new", - "obj": { - "source": "foursquare_recents", - "data": { - "_id": "4e703a41ca17313fd2cccc6b", - "id": "4e703881d4c03fcb653e0251", - "createdAt": 1315977345, - "type": "checkin", - "timeZone": "America/Chicago", - "entities": [], - "user": { - "id": "10598838", - "firstName": "Lis", - "lastName": "Miller", - "photo": "https://playfoursquare.s3.amazonaws.com/userpix_thumbs/ZHSLPGESZ530RZEK.jpg", - "gender": "female", - "homeCity": "Cascade, IA", - "relationship": "friend" + "idr": "checkin://foursquare/recents?id=foursquare#4e703881d4c03fcb653e0251", + "data": { + "_id": "4e703a41ca17313fd2cccc6b", + "id": "4e703881d4c03fcb653e0251", + "createdAt": 1315977345, + "type": "checkin", + "timeZone": "America/Chicago", + "entities": [], + "user": { + "id": "10598838", + "firstName": "Lis", + "lastName": "Miller", + "photo": "https://playfoursquare.s3.amazonaws.com/userpix_thumbs/ZHSLPGESZ530RZEK.jpg", + "gender": "female", + "homeCity": "Cascade, IA", + "relationship": "friend" + }, + "venue": { + "id": "4e1362df18a84f0f0325cb37", + "name": "Home: 605 Garfield St SW", + "contact": {}, + "location": { + "address": "605 Garfield St. SW", + "lat": 42.292913, + "lng": -91.018424, + "city": "Cascade", + "state": "IA" }, - "venue": { - "id": "4e1362df18a84f0f0325cb37", - "name": "Home: 605 Garfield St SW", - "contact": {}, - "location": { - "address": "605 Garfield St. SW", - "lat": 42.292913, - "lng": -91.018424, - "city": "Cascade", - "state": "IA" - }, - "categories": [{ + "categories": [ + { "id": "4bf58dd8d48988d103941735", "name": "Home", "pluralName": "Homes", "shortName": "Home", "icon": "https://foursquare.com/img/categories/building/home.png", - "parents": ["Residences"], + "parents": [ + "Residences" + ], "primary": true - }], - "verified": false, - "stats": { - "checkinsCount": 246, - "usersCount": 4, - "tipCount": 0 } - }, - "source": { - "name": "foursquare for iPhone", - "url": "https://foursquare.com/download/#/iphone" - }, - "photos": { - "count": 0, - "items": [] - }, - "comments": { - "count": 0, - "items": [] + ], + "verified": false, + "stats": { + "checkinsCount": 246, + "usersCount": 4, + "tipCount": 0 } + }, + "source": { + "name": "foursquare for iPhone", + "url": "https://foursquare.com/download/#/iphone" + }, + "photos": { + "count": 0, + "items": [] + }, + "comments": { + "count": 0, + "items": [] } } -} +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/ig.fb b/Collections/Timeline/fixtures/ig.fb index 848689d6f..c4f17ff0c 100644 --- a/Collections/Timeline/fixtures/ig.fb +++ b/Collections/Timeline/fixtures/ig.fb @@ -1,42 +1,38 @@ { - "type": "home/facebook", - "via": "synclet/facebook", - "timestamp": 1315976215195, "action": "update", - "obj": - { - "timeStamp": "2011-09-19T15:55:51.950Z", - "data": { - "id": "4803283_730289247379", - "from": { - "name": "Julian Missig", - "id": "4803283" - }, - "message": "Model S model", - "picture": "http://platform.ak.fbcdn.net/www/app_full_proxy.php?app=124024574287414&v=1&size=z&cksum=437fa91b29c88f863ebfe460c8aa989b&src=http%3A%2F%2Fimages.instagram.com%2Fmedia%2F2011%2F09%2F18%2F391a26f2de1f47b089f4a2a2215b6bf7_7.jpg", - "link": "http://instagr.am/p/Ne6nr/", - "name": "jmissig's photo", - "caption": "instagr.am", - "description": "jmissig's photo on Instagram", - "icon": "http://photos-e.ak.fbcdn.net/photos-ak-snc1/v27562/10/124024574287414/app_2_124024574287414_6936.gif", - "actions": [{ + "idr": "post://facebook/home?id=facebook#4803283_730289247379", + "data": { + "id": "4803283_730289247379", + "from": { + "name": "Julian Missig", + "id": "4803283" + }, + "message": "Model S model", + "picture": "http://platform.ak.fbcdn.net/www/app_full_proxy.php?app=124024574287414&v=1&size=z&cksum=437fa91b29c88f863ebfe460c8aa989b&src=http%3A%2F%2Fimages.instagram.com%2Fmedia%2F2011%2F09%2F18%2F391a26f2de1f47b089f4a2a2215b6bf7_7.jpg", + "link": "http://instagr.am/p/Ne6nr/", + "name": "jmissig's photo", + "caption": "instagr.am", + "description": "jmissig's photo on Instagram", + "icon": "http://photos-e.ak.fbcdn.net/photos-ak-snc1/v27562/10/124024574287414/app_2_124024574287414_6936.gif", + "actions": [ + { "name": "Comment", "link": "http://www.facebook.com/4803283/posts/730289247379" }, { "name": "Like", "link": "http://www.facebook.com/4803283/posts/730289247379" - }], - "type": "link", - "application": { - "name": "Instagram", - "id": "124024574287414" - }, - "created_time": 1316385395, - "updated_time": 1316385395, - "comments": { - "count": 0 } + ], + "type": "link", + "application": { + "name": "Instagram", + "id": "124024574287414" + }, + "created_time": 1316385395, + "updated_time": 1316385395, + "comments": { + "count": 0 } } -} +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/ig.instagram b/Collections/Timeline/fixtures/ig.instagram index adfbe652c..21e9f6f94 100644 --- a/Collections/Timeline/fixtures/ig.instagram +++ b/Collections/Timeline/fixtures/ig.instagram @@ -1,66 +1,61 @@ { - "type": "feed/instagram", - "via": "synclet/instagram", - "timestamp": 1315974920913, "action": "new", - "obj": { - "timeStamp": "2011-10-27T00:45:21.104Z", - "data": { - "tags": [], - "type": "image", - "location": { - "latitude": 37.44355, - "longitude": -122.1706 + "idr": "photo://instagram/feed?id=instagram#226208235_981179", + "data": { + "tags": [], + "type": "image", + "location": { + "latitude": 37.44355, + "longitude": -122.1706 + }, + "comments": { + "count": 0, + "data": [] + }, + "filter": "Normal", + "created_time": "1316385392", + "link": "http://instagr.am/p/Ne6nr/", + "likes": { + "count": 0, + "data": [] + }, + "images": { + "low_resolution": { + "url": "http://distillery.s3.amazonaws.com/media/2011/09/18/391a26f2de1f47b089f4a2a2215b6bf7_6.jpg", + "width": 306, + "height": 306 }, - "comments": { - "count": 0, - "data": [] + "thumbnail": { + "url": "http://distillery.s3.amazonaws.com/media/2011/09/18/391a26f2de1f47b089f4a2a2215b6bf7_5.jpg", + "width": 150, + "height": 150 }, - "filter": "Normal", - "created_time": "1316385392", - "link": "http://instagr.am/p/Ne6nr/", - "likes": { - "count": 0, - "data": [] - }, - "images": { - "low_resolution": { - "url": "http://distillery.s3.amazonaws.com/media/2011/09/18/391a26f2de1f47b089f4a2a2215b6bf7_6.jpg", - "width": 306, - "height": 306 - }, - "thumbnail": { - "url": "http://distillery.s3.amazonaws.com/media/2011/09/18/391a26f2de1f47b089f4a2a2215b6bf7_5.jpg", - "width": 150, - "height": 150 - }, - "standard_resolution": { - "url": "http://distillery.s3.amazonaws.com/media/2011/09/18/391a26f2de1f47b089f4a2a2215b6bf7_7.jpg", - "width": 612, - "height": 612 - } - }, - "caption": { - "created_time": "1316385394", - "text": "Model S model", - "from": { - "username": "jmissig", - "profile_picture": "http://images.instagram.com/profiles/profile_981179_75sq_1297666725.jpg", - "id": "981179", - "full_name": "Julian Missig" - }, - "id": "277184261" - }, - "user_has_liked": false, - "id": "226208235_981179", - "user": { + "standard_resolution": { + "url": "http://distillery.s3.amazonaws.com/media/2011/09/18/391a26f2de1f47b089f4a2a2215b6bf7_7.jpg", + "width": 612, + "height": 612 + } + }, + "caption": { + "created_time": "1316385394", + "text": "Model S model", + "from": { "username": "jmissig", - "website": "", - "bio": "", "profile_picture": "http://images.instagram.com/profiles/profile_981179_75sq_1297666725.jpg", - "full_name": "Julian Missig", - "id": "981179" - } + "id": "981179", + "full_name": "Julian Missig" + }, + "id": "277184261" + }, + "user_has_liked": false, + "id": "226208235_981179", + "user": { + "username": "jmissig", + "website": "", + "bio": "", + "profile_picture": "http://images.instagram.com/profiles/profile_981179_75sq_1297666725.jpg", + "full_name": "Julian Missig", + "id": "981179" } } -} +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/ig.tweet b/Collections/Timeline/fixtures/ig.tweet index 94d59c640..5fc47aa05 100644 --- a/Collections/Timeline/fixtures/ig.tweet +++ b/Collections/Timeline/fixtures/ig.tweet @@ -1,80 +1,80 @@ { - "type": "timeline/twitter", - "via": "synclet/twitter", - "timestamp": 1315974920913, "action": "new", - "obj": { - "timeStamp": "2011-09-19T15:56:00.454Z", - "data": { - "in_reply_to_status_id_str": null, - "truncated": false, - "in_reply_to_user_id_str": null, - "geo": null, - "retweet_count": 0, - "contributors": null, - "coordinates": null, - "user": { - "statuses_count": 4902, - "favourites_count": 111, - "protected": false, - "profile_text_color": "333333", - "profile_image_url": "http://a1.twimg.com/profile_images/31867902/Julian_Missig_1_normal.png", - "name": "Julian Missig", - "profile_sidebar_fill_color": "A0C5C7", - "listed_count": 22, - "following": true, - "profile_background_tile": false, - "utc_offset": -28800, - "description": "interaction designer & prototyper", - "location": "Redwood City, CA", - "contributors_enabled": false, - "verified": false, - "profile_link_color": "369EAF", - "followers_count": 248, - "url": null, - "profile_sidebar_border_color": "86A4A6", - "screen_name": "jmissig", - "default_profile_image": false, - "notifications": false, - "default_profile": false, - "show_all_inline_media": false, - "geo_enabled": true, - "profile_use_background_image": true, - "friends_count": 186, - "id_str": "9147872", - "is_translator": false, - "lang": "en", - "time_zone": "Pacific Time (US & Canada)", - "created_at": "Fri Sep 28 19:19:58 +0000 2007", - "profile_background_color": "709397", - "id": 9147872, - "follow_request_sent": false, - "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme6/bg.gif", - "profile_background_image_url": "http://a1.twimg.com/images/themes/theme6/bg.gif", - "profile_image_url_https": "https://si0.twimg.com/profile_images/31867902/Julian_Missig_1_normal.png" - }, - "favorited": false, - "in_reply_to_screen_name": null, - "source": "Instagram", - "id_str": "115554837076258816", - "entities": { - "hashtags": [], - "urls": [{ - "indices": [14, 34], + "idr": "tweet://twitter/timeline?id=twitter#115554837076258816", + "data": { + "in_reply_to_status_id_str": null, + "truncated": false, + "in_reply_to_user_id_str": null, + "geo": null, + "retweet_count": 0, + "contributors": null, + "coordinates": null, + "user": { + "statuses_count": 4902, + "favourites_count": 111, + "protected": false, + "profile_text_color": "333333", + "profile_image_url": "http://a1.twimg.com/profile_images/31867902/Julian_Missig_1_normal.png", + "name": "Julian Missig", + "profile_sidebar_fill_color": "A0C5C7", + "listed_count": 22, + "following": true, + "profile_background_tile": false, + "utc_offset": -28800, + "description": "interaction designer & prototyper", + "location": "Redwood City, CA", + "contributors_enabled": false, + "verified": false, + "profile_link_color": "369EAF", + "followers_count": 248, + "url": null, + "profile_sidebar_border_color": "86A4A6", + "screen_name": "jmissig", + "default_profile_image": false, + "notifications": false, + "default_profile": false, + "show_all_inline_media": false, + "geo_enabled": true, + "profile_use_background_image": true, + "friends_count": 186, + "id_str": "9147872", + "is_translator": false, + "lang": "en", + "time_zone": "Pacific Time (US & Canada)", + "created_at": "Fri Sep 28 19:19:58 +0000 2007", + "profile_background_color": "709397", + "id": 9147872, + "follow_request_sent": false, + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme6/bg.gif", + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme6/bg.gif", + "profile_image_url_https": "https://si0.twimg.com/profile_images/31867902/Julian_Missig_1_normal.png" + }, + "favorited": false, + "in_reply_to_screen_name": null, + "source": "Instagram", + "id_str": "115554837076258816", + "entities": { + "hashtags": [], + "urls": [ + { + "indices": [ + 14, + 34 + ], "url": "http://t.co/PeedUocW", "display_url": "instagr.am/p/Ne6nr/", "expanded_url": "http://instagr.am/p/Ne6nr/" - }], - "user_mentions": [] - }, - "in_reply_to_status_id": null, - "id": 115554837076258820, - "created_at": "Sun Sep 18 22:36:34 +0000 2011", - "possibly_sensitive": false, - "place": null, - "retweeted": false, - "in_reply_to_user_id": null, - "text": "Model S model http://t.co/PeedUocW" - } + } + ], + "user_mentions": [] + }, + "in_reply_to_status_id": null, + "id": 115554837076258820, + "created_at": "Sun Sep 18 22:36:34 +0000 2011", + "possibly_sensitive": false, + "place": null, + "retweeted": false, + "in_reply_to_user_id": null, + "text": "Model S model http://t.co/PeedUocW" } -} +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/instagram.feed b/Collections/Timeline/fixtures/instagram.feed index 27d22c3dc..779fd3723 100644 --- a/Collections/Timeline/fixtures/instagram.feed +++ b/Collections/Timeline/fixtures/instagram.feed @@ -1,17 +1,14 @@ { - "type": "feed/instagram", - "via": "synclet/instagram", - "timestamp": 1315974920913, "action": "new", - "obj": { - "timeStamp": 1319676316966, - "data": { - "tags": [], - "type": "image", - "location": null, - "comments": { - "count": 1, - "data": [{ + "idr": "photo://instagram/feed?id=instagram#294061795_2920312", + "data": { + "tags": [], + "type": "image", + "location": null, + "comments": { + "count": 1, + "data": [ + { "created_time": "1319651890", "text": "Great pic, love the colors", "from": { @@ -21,14 +18,16 @@ "full_name": "Brad Barnett" }, "id": "367123985" - }] - }, - "filter": "Normal", - "created_time": "1319642091", - "link": "http://instagr.am/p/Rhwbj/", - "likes": { - "count": 2, - "data": [{ + } + ] + }, + "filter": "Normal", + "created_time": "1319642091", + "link": "http://instagr.am/p/Rhwbj/", + "likes": { + "count": 2, + "data": [ + { "username": "seanfeezy", "profile_picture": "http://images.instagram.com/profiles/profile_8524340_75sq_1314153519.jpg", "id": "8524340", @@ -39,46 +38,46 @@ "profile_picture": "http://images.instagram.com/profiles/profile_11076900_75sq_1319234253.jpg", "id": "11076900", "full_name": "Brad Barnett" - }] - }, - "images": { - "low_resolution": { - "url": "http://images.instagram.com/media/2011/10/26/f47c9ff00aff4ae9a6f8491d24722308_6.jpg", - "width": 306, - "height": 306 - }, - "thumbnail": { - "url": "http://images.instagram.com/media/2011/10/26/f47c9ff00aff4ae9a6f8491d24722308_5.jpg", - "width": 150, - "height": 150 - }, - "standard_resolution": { - "url": "http://images.instagram.com/media/2011/10/26/f47c9ff00aff4ae9a6f8491d24722308_7.jpg", - "width": 612, - "height": 612 } + ] + }, + "images": { + "low_resolution": { + "url": "http://images.instagram.com/media/2011/10/26/f47c9ff00aff4ae9a6f8491d24722308_6.jpg", + "width": 306, + "height": 306 }, - "caption": { - "created_time": "1319642158", - "text": "Morning Mission. ", - "from": { - "username": "jasoncavnar", - "profile_picture": "http://images.instagram.com/profiles/profile_2920312_75sq_1316296467.jpg", - "id": "2920312", - "full_name": "Jason Cavnar" - }, - "id": "366806429" + "thumbnail": { + "url": "http://images.instagram.com/media/2011/10/26/f47c9ff00aff4ae9a6f8491d24722308_5.jpg", + "width": 150, + "height": 150 }, - "user_has_liked": false, - "id": "294061795_2920312", - "user": { + "standard_resolution": { + "url": "http://images.instagram.com/media/2011/10/26/f47c9ff00aff4ae9a6f8491d24722308_7.jpg", + "width": 612, + "height": 612 + } + }, + "caption": { + "created_time": "1319642158", + "text": "Morning Mission. ", + "from": { "username": "jasoncavnar", - "website": "http://www.singly.com", - "bio": "That guy. Using Instagr.am to try to keep up with my hipster friends and their tight jeans. ", "profile_picture": "http://images.instagram.com/profiles/profile_2920312_75sq_1316296467.jpg", - "full_name": "Jason Cavnar", - "id": "2920312" - } + "id": "2920312", + "full_name": "Jason Cavnar" + }, + "id": "366806429" + }, + "user_has_liked": false, + "id": "294061795_2920312", + "user": { + "username": "jasoncavnar", + "website": "http://www.singly.com", + "bio": "That guy. Using Instagr.am to try to keep up with my hipster friends and their tight jeans. ", + "profile_picture": "http://images.instagram.com/profiles/profile_2920312_75sq_1316296467.jpg", + "full_name": "Jason Cavnar", + "id": "2920312" } } -} +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/tweet.facebook b/Collections/Timeline/fixtures/tweet.facebook index f0c54b12f..2d128348b 100644 --- a/Collections/Timeline/fixtures/tweet.facebook +++ b/Collections/Timeline/fixtures/tweet.facebook @@ -1,86 +1,91 @@ { - "type": "timeline/twitter", - "via": "synclet/twitter", - "timestamp": 1315974920913, "action": "new", - "obj": { - "timeStamp": "2011-09-20T14:09:19.331Z", - "data": { - "contributors": null, - "truncated": false, - "geo": null, - "favorited": false, - "coordinates": null, - "user": { - "profile_text_color": "393939", - "protected": false, - "listed_count": 457, - "profile_sidebar_fill_color": "e8e8e8", - "name": "Alex Rainert", - "contributors_enabled": false, - "profile_background_tile": true, - "profile_image_url_https": "https://si0.twimg.com/profile_images/1331003463/arainert_headshot2_supersm_normal.jpg", - "profile_image_url": "http://a0.twimg.com/profile_images/1331003463/arainert_headshot2_supersm_normal.jpg", - "default_profile": false, - "show_all_inline_media": true, - "following": true, - "geo_enabled": true, - "utc_offset": -18000, - "description": "Husband, father & owner of a lovably neurotic vizsla. Obsessed with information, design, emerging tech, sports & food. Head of product at foursquare.", - "profile_link_color": "ff4cb4", - "location": "Brooklyn, NY", - "default_profile_image": false, - "verified": false, - "profile_sidebar_border_color": "ff4cb4", - "statuses_count": 15165, - "friends_count": 511, - "followers_count": 4717, - "url": "http://www.alexrainert.com/", - "is_translator": false, - "profile_use_background_image": true, - "screen_name": "arainert", - "follow_request_sent": false, - "notifications": false, - "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/17611084/walken_small.jpg", - "time_zone": "Eastern Time (US & Canada)", - "favourites_count": 68, - "profile_background_color": "352726", - "lang": "en", - "profile_background_image_url": "http://a1.twimg.com/profile_background_images/17611084/walken_small.jpg", - "id": 7482, - "id_str": "7482", - "created_at": "Wed Oct 04 18:33:19 +0000 2006" - }, - "in_reply_to_screen_name": null, - "possibly_sensitive": false, - "retweeted": false, - "in_reply_to_status_id_str": null, - "source": "Tweet Button", - "entities": { - "user_mentions": [{ + "idr": "tweet://twitter/timeline?id=twitter#116149996419682304", + "data": { + "contributors": null, + "truncated": false, + "geo": null, + "favorited": false, + "coordinates": null, + "user": { + "profile_text_color": "393939", + "protected": false, + "listed_count": 457, + "profile_sidebar_fill_color": "e8e8e8", + "name": "Alex Rainert", + "contributors_enabled": false, + "profile_background_tile": true, + "profile_image_url_https": "https://si0.twimg.com/profile_images/1331003463/arainert_headshot2_supersm_normal.jpg", + "profile_image_url": "http://a0.twimg.com/profile_images/1331003463/arainert_headshot2_supersm_normal.jpg", + "default_profile": false, + "show_all_inline_media": true, + "following": true, + "geo_enabled": true, + "utc_offset": -18000, + "description": "Husband, father & owner of a lovably neurotic vizsla. Obsessed with information, design, emerging tech, sports & food. Head of product at foursquare.", + "profile_link_color": "ff4cb4", + "location": "Brooklyn, NY", + "default_profile_image": false, + "verified": false, + "profile_sidebar_border_color": "ff4cb4", + "statuses_count": 15165, + "friends_count": 511, + "followers_count": 4717, + "url": "http://www.alexrainert.com/", + "is_translator": false, + "profile_use_background_image": true, + "screen_name": "arainert", + "follow_request_sent": false, + "notifications": false, + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/17611084/walken_small.jpg", + "time_zone": "Eastern Time (US & Canada)", + "favourites_count": 68, + "profile_background_color": "352726", + "lang": "en", + "profile_background_image_url": "http://a1.twimg.com/profile_background_images/17611084/walken_small.jpg", + "id": 7482, + "id_str": "7482", + "created_at": "Wed Oct 04 18:33:19 +0000 2006" + }, + "in_reply_to_screen_name": null, + "possibly_sensitive": false, + "retweeted": false, + "in_reply_to_status_id_str": null, + "source": "Tweet Button", + "entities": { + "user_mentions": [ + { "name": "SplatF", - "indices": [93, 100], + "indices": [ + 93, + 100 + ], "screen_name": "splatf", "id": 174886575, "id_str": "174886575" - }], - "hashtags": [], - "urls": [{ + } + ], + "hashtags": [], + "urls": [ + { "display_url": "splatf.com/2011/09/netfli…", - "indices": [119, 139], + "indices": [ + 119, + 139 + ], "expanded_url": "http://www.splatf.com/2011/09/netflix-qwikster-facts/", "url": "http://t.co/ZMu8UkyK" - }] - }, - "in_reply_to_user_id": null, - "id": 116149996419682300, - "in_reply_to_user_id_str": null, - "id_str": "116149996419682304", - "place": null, - "retweet_count": 0, - "in_reply_to_status_id": null, - "text": "Dan was the 1st to nail the Google/Motorola news and yesterday he nailed Netflix. Get on the @splatf analysis, people! http://t.co/ZMu8UkyK", - "created_at": "Tue Sep 20 14:01:31 +0000 2011" - } + } + ] + }, + "in_reply_to_user_id": null, + "id": 116149996419682300, + "in_reply_to_user_id_str": null, + "id_str": "116149996419682304", + "place": null, + "retweet_count": 0, + "in_reply_to_status_id": null, + "text": "Dan was the 1st to nail the Google/Motorola news and yesterday he nailed Netflix. Get on the @splatf analysis, people! http://t.co/ZMu8UkyK", + "created_at": "Tue Sep 20 14:01:31 +0000 2011" } -} +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/twitter.related b/Collections/Timeline/fixtures/twitter.related index 830e0ec13..cf220dff5 100644 --- a/Collections/Timeline/fixtures/twitter.related +++ b/Collections/Timeline/fixtures/twitter.related @@ -1,551 +1,549 @@ { - "type": "related/twitter", - "via": "synclet/twitter", - "timestamp": 1315976039292, "action": "new", - "obj": { - "source": "twitter_related", - "type": "new", - "data": { - "id": "109553278831951872", - "related": [{ - "results": [{ - "score": 1, - "annotations": { - "ConversationRole": "Fork" - }, - "kind": "Tweet", - "value": { - "id_str": "109570326760919041", - "in_reply_to_status_id": 109553278831951870, - "truncated": false, - "user": { - "profile_sidebar_fill_color": "95E8EC", - "protected": false, - "id_str": "79202245", - "default_profile": false, - "notifications": null, - "profile_background_tile": false, - "screen_name": "Tiduilasa", - "name": "Tidu", - "listed_count": 1, - "location": "Brickman Road, S.Fburg. NY/USA", - "show_all_inline_media": false, - "contributors_enabled": false, - "following": null, - "geo_enabled": false, - "utc_offset": -21600, - "profile_link_color": "0099B9", - "description": "YOGI BHAGAVATA-DHARMA\r\n_______________________", - "profile_sidebar_border_color": "5ED4DC", - "url": "http://www.siddhayoga.org/gurumayi-chidvilasananda", - "time_zone": "Central Time (US & Canada)", - "default_profile_image": false, - "statuses_count": 85, - "profile_use_background_image": true, - "verified": false, - "favourites_count": 12, - "friends_count": 5, - "profile_background_color": "0099B9", - "is_translator": false, - "profile_background_image_url": "http://a1.twimg.com/images/themes/theme4/bg.gif", - "created_at": "Fri Oct 02 15:01:45 +0000 2009", - "followers_count": 3, - "follow_request_sent": null, - "lang": "en", - "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme4/bg.gif", - "id": 79202245, - "profile_text_color": "3C3940", - "profile_image_url_https": "https://si0.twimg.com/profile_images/1522182333/aum-message-artwork_1__normal.jpg", - "profile_image_url": "http://a2.twimg.com/profile_images/1522182333/aum-message-artwork_1__normal.jpg" + "idr": "tweet://twitter/related?id=twitter#109553278831951872", + "data": { + "id": "109553278831951872", + "related": [ + { + "results": [ + { + "score": 1, + "annotations": { + "ConversationRole": "Fork" }, - "favorited": false, - "possibly_sensitive": false, - "in_reply_to_status_id_str": "109553278831951872", - "geo": null, - "in_reply_to_screen_name": "DalaiLama", - "in_reply_to_user_id_str": "20609518", - "coordinates": null, - "in_reply_to_user_id": 20609518, - "source": "web", - "created_at": "Fri Sep 02 10:16:16 +0000 2011", - "contributors": null, - "retweeted": false, - "retweet_count": 0, - "id": 109570326760919040, - "place": null, - "text": "@DalaiLama ~ Fowers to Mr, Dalai; (was called \"Delair\" - my papy)\nhttp://t.co/MXYIaaS" - } - }, - { - "score": 1, - "annotations": { - "ConversationRole": "Fork" + "kind": "Tweet", + "value": { + "id_str": "109570326760919041", + "in_reply_to_status_id": 109553278831951870, + "truncated": false, + "user": { + "profile_sidebar_fill_color": "95E8EC", + "protected": false, + "id_str": "79202245", + "default_profile": false, + "notifications": null, + "profile_background_tile": false, + "screen_name": "Tiduilasa", + "name": "Tidu", + "listed_count": 1, + "location": "Brickman Road, S.Fburg. NY/USA", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": -21600, + "profile_link_color": "0099B9", + "description": "YOGI BHAGAVATA-DHARMA\r\n_______________________", + "profile_sidebar_border_color": "5ED4DC", + "url": "http://www.siddhayoga.org/gurumayi-chidvilasananda", + "time_zone": "Central Time (US & Canada)", + "default_profile_image": false, + "statuses_count": 85, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 12, + "friends_count": 5, + "profile_background_color": "0099B9", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme4/bg.gif", + "created_at": "Fri Oct 02 15:01:45 +0000 2009", + "followers_count": 3, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme4/bg.gif", + "id": 79202245, + "profile_text_color": "3C3940", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1522182333/aum-message-artwork_1__normal.jpg", + "profile_image_url": "http://a2.twimg.com/profile_images/1522182333/aum-message-artwork_1__normal.jpg" + }, + "favorited": false, + "possibly_sensitive": false, + "in_reply_to_status_id_str": "109553278831951872", + "geo": null, + "in_reply_to_screen_name": "DalaiLama", + "in_reply_to_user_id_str": "20609518", + "coordinates": null, + "in_reply_to_user_id": 20609518, + "source": "web", + "created_at": "Fri Sep 02 10:16:16 +0000 2011", + "contributors": null, + "retweeted": false, + "retweet_count": 0, + "id": 109570326760919040, + "place": null, + "text": "@DalaiLama ~ Fowers to Mr, Dalai; (was called \"Delair\" - my papy)\nhttp://t.co/MXYIaaS" + } }, - "kind": "Tweet", - "value": { - "id_str": "109569700891082752", - "in_reply_to_status_id": 109553278831951870, - "truncated": false, - "user": { - "profile_sidebar_fill_color": "DDEEF6", - "protected": false, - "id_str": "114698131", - "default_profile": false, - "notifications": null, - "profile_background_tile": false, - "screen_name": "jeshvik77", - "name": "rajesh", - "listed_count": 1, - "location": "KNR.HYD.GHY..", - "show_all_inline_media": false, - "contributors_enabled": false, - "following": null, - "geo_enabled": false, - "utc_offset": -36000, - "profile_link_color": "0084B4", - "description": "Hypo r Hyper n still trying for Normo, 'll bit narcissistic, as of nw Hurt-Locker, pasion n EM,belivin fnds r lyf...", - "profile_sidebar_border_color": "C0DEED", - "url": null, - "time_zone": "Hawaii", - "default_profile_image": false, - "statuses_count": 2136, - "profile_use_background_image": true, - "verified": false, - "favourites_count": 4, - "friends_count": 37, - "profile_background_color": "C0DEED", - "is_translator": false, - "profile_background_image_url": "http://a2.twimg.com/profile_background_images/308608018/PIX_094-58.jpg", - "created_at": "Tue Feb 16 09:30:33 +0000 2010", - "followers_count": 49, - "follow_request_sent": null, - "lang": "en", - "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/308608018/PIX_094-58.jpg", - "id": 114698131, - "profile_text_color": "333333", - "profile_image_url_https": "https://si0.twimg.com/profile_images/1452584974/uu_normal.jpg", - "profile_image_url": "http://a1.twimg.com/profile_images/1452584974/uu_normal.jpg" + { + "score": 1, + "annotations": { + "ConversationRole": "Fork" }, - "favorited": false, - "in_reply_to_status_id_str": "109553278831951872", - "geo": null, - "in_reply_to_screen_name": "DalaiLama", - "in_reply_to_user_id_str": "20609518", - "coordinates": null, - "in_reply_to_user_id": 20609518, - "source": "web", - "created_at": "Fri Sep 02 10:13:47 +0000 2011", - "contributors": null, - "retweeted": false, - "retweet_count": 0, - "id": 109569700891082750, - "place": null, - "text": "@DalaiLama \"@jeshvik77 The practice of altruism is the authentic way of living a human life; it is not limited only to the religious\"" - } - }, - { - "score": 1, - "annotations": { - "ConversationRole": "Fork" + "kind": "Tweet", + "value": { + "id_str": "109569700891082752", + "in_reply_to_status_id": 109553278831951870, + "truncated": false, + "user": { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "114698131", + "default_profile": false, + "notifications": null, + "profile_background_tile": false, + "screen_name": "jeshvik77", + "name": "rajesh", + "listed_count": 1, + "location": "KNR.HYD.GHY..", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": -36000, + "profile_link_color": "0084B4", + "description": "Hypo r Hyper n still trying for Normo, 'll bit narcissistic, as of nw Hurt-Locker, pasion n EM,belivin fnds r lyf...", + "profile_sidebar_border_color": "C0DEED", + "url": null, + "time_zone": "Hawaii", + "default_profile_image": false, + "statuses_count": 2136, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 4, + "friends_count": 37, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a2.twimg.com/profile_background_images/308608018/PIX_094-58.jpg", + "created_at": "Tue Feb 16 09:30:33 +0000 2010", + "followers_count": 49, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/308608018/PIX_094-58.jpg", + "id": 114698131, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1452584974/uu_normal.jpg", + "profile_image_url": "http://a1.twimg.com/profile_images/1452584974/uu_normal.jpg" + }, + "favorited": false, + "in_reply_to_status_id_str": "109553278831951872", + "geo": null, + "in_reply_to_screen_name": "DalaiLama", + "in_reply_to_user_id_str": "20609518", + "coordinates": null, + "in_reply_to_user_id": 20609518, + "source": "web", + "created_at": "Fri Sep 02 10:13:47 +0000 2011", + "contributors": null, + "retweeted": false, + "retweet_count": 0, + "id": 109569700891082750, + "place": null, + "text": "@DalaiLama \"@jeshvik77 The practice of altruism is the authentic way of living a human life; it is not limited only to the religious\"" + } }, - "kind": "Tweet", - "value": { - "id_str": "109569691311288321", - "in_reply_to_status_id": 109553278831951870, - "truncated": false, - "user": { - "profile_sidebar_fill_color": "DDEEF6", - "protected": false, - "id_str": "351589243", - "default_profile": true, - "notifications": null, - "profile_background_tile": false, - "screen_name": "nukmi", - "name": "nukmi", - "listed_count": 0, - "location": "", - "show_all_inline_media": false, - "contributors_enabled": false, - "following": null, - "geo_enabled": false, - "utc_offset": null, - "profile_link_color": "0084B4", - "description": "human being from south east london, a real tyrant :)", - "profile_sidebar_border_color": "C0DEED", - "url": null, - "time_zone": null, - "default_profile_image": true, - "statuses_count": 24, - "profile_use_background_image": true, - "verified": false, - "favourites_count": 0, - "friends_count": 11, - "profile_background_color": "C0DEED", - "is_translator": false, - "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", - "created_at": "Tue Aug 09 13:26:35 +0000 2011", - "followers_count": 3, - "follow_request_sent": null, - "lang": "en", - "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", - "id": 351589243, - "profile_text_color": "333333", - "profile_image_url_https": "https://si0.twimg.com/sticky/default_profile_images/default_profile_5_normal.png", - "profile_image_url": "http://a0.twimg.com/sticky/default_profile_images/default_profile_5_normal.png" + { + "score": 1, + "annotations": { + "ConversationRole": "Fork" }, - "favorited": false, - "in_reply_to_status_id_str": "109553278831951872", - "geo": null, - "in_reply_to_screen_name": "DalaiLama", - "in_reply_to_user_id_str": "20609518", - "coordinates": null, - "in_reply_to_user_id": 20609518, - "source": "web", - "created_at": "Fri Sep 02 10:13:44 +0000 2011", - "contributors": null, - "retweeted": false, - "retweet_count": 0, - "id": 109569691311288320, - "place": null, - "text": "@DalaiLama does HHDL have any thoughts to share on the role of cinema & film in creating a moral/altruistic society?" - } - }, - { - "score": 1, - "annotations": { - "ConversationRole": "Fork" + "kind": "Tweet", + "value": { + "id_str": "109569691311288321", + "in_reply_to_status_id": 109553278831951870, + "truncated": false, + "user": { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "351589243", + "default_profile": true, + "notifications": null, + "profile_background_tile": false, + "screen_name": "nukmi", + "name": "nukmi", + "listed_count": 0, + "location": "", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": null, + "profile_link_color": "0084B4", + "description": "human being from south east london, a real tyrant :)", + "profile_sidebar_border_color": "C0DEED", + "url": null, + "time_zone": null, + "default_profile_image": true, + "statuses_count": 24, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 0, + "friends_count": 11, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Tue Aug 09 13:26:35 +0000 2011", + "followers_count": 3, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 351589243, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/sticky/default_profile_images/default_profile_5_normal.png", + "profile_image_url": "http://a0.twimg.com/sticky/default_profile_images/default_profile_5_normal.png" + }, + "favorited": false, + "in_reply_to_status_id_str": "109553278831951872", + "geo": null, + "in_reply_to_screen_name": "DalaiLama", + "in_reply_to_user_id_str": "20609518", + "coordinates": null, + "in_reply_to_user_id": 20609518, + "source": "web", + "created_at": "Fri Sep 02 10:13:44 +0000 2011", + "contributors": null, + "retweeted": false, + "retweet_count": 0, + "id": 109569691311288320, + "place": null, + "text": "@DalaiLama does HHDL have any thoughts to share on the role of cinema & film in creating a moral/altruistic society?" + } }, - "kind": "Tweet", - "value": { - "id_str": "109568108846530560", - "in_reply_to_status_id": 109553278831951870, - "truncated": false, - "user": { - "profile_sidebar_fill_color": "efefef", - "protected": false, - "id_str": "96293863", - "default_profile": false, - "notifications": null, - "profile_background_tile": true, - "screen_name": "SofistaPrateado", - "name": "Renato(Rewah Hameth)", - "listed_count": 1, - "location": "Ilha da Rainha da Morte - RJ", - "show_all_inline_media": true, - "contributors_enabled": false, - "following": null, - "geo_enabled": false, - "utc_offset": -10800, - "profile_link_color": "009999", - "description": "Bancário, gamer, lutador de Kung-fu, escritor de contos, tarólogo, acupuntor, leitor, futurista e pseudo-humorista. Assim sou/estou eu.", - "profile_sidebar_border_color": "eeeeee", - "url": "http://sofistaprateado.tumblr.com", - "time_zone": "Brasilia", - "default_profile_image": false, - "statuses_count": 11251, - "profile_use_background_image": true, - "verified": false, - "favourites_count": 76, - "friends_count": 123, - "profile_background_color": "131516", - "is_translator": false, - "profile_background_image_url": "http://a1.twimg.com/images/themes/theme14/bg.gif", - "created_at": "Sat Dec 12 06:54:04 +0000 2009", - "followers_count": 123, - "follow_request_sent": null, - "lang": "en", - "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme14/bg.gif", - "id": 96293863, - "profile_text_color": "333333", - "profile_image_url_https": "https://si0.twimg.com/profile_images/1514569915/aquiles_normal.JPG", - "profile_image_url": "http://a3.twimg.com/profile_images/1514569915/aquiles_normal.JPG" + { + "score": 1, + "annotations": { + "ConversationRole": "Fork" }, - "favorited": false, - "in_reply_to_status_id_str": "109553278831951872", - "geo": null, - "in_reply_to_screen_name": "DalaiLama", - "in_reply_to_user_id_str": "20609518", - "coordinates": null, - "in_reply_to_user_id": 20609518, - "source": "web", - "created_at": "Fri Sep 02 10:07:27 +0000 2011", - "contributors": null, - "retweeted": false, - "retweet_count": 0, - "id": 109568108846530560, - "place": null, - "text": "A prática de altruísmo é o modo autêntico de viver uma vida humana; não é limitada apenas aos religiosos. (via @DalaiLama) #fb" - } - }, - { - "score": 1, - "annotations": { - "ConversationRole": "Fork" + "kind": "Tweet", + "value": { + "id_str": "109568108846530560", + "in_reply_to_status_id": 109553278831951870, + "truncated": false, + "user": { + "profile_sidebar_fill_color": "efefef", + "protected": false, + "id_str": "96293863", + "default_profile": false, + "notifications": null, + "profile_background_tile": true, + "screen_name": "SofistaPrateado", + "name": "Renato(Rewah Hameth)", + "listed_count": 1, + "location": "Ilha da Rainha da Morte - RJ", + "show_all_inline_media": true, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": -10800, + "profile_link_color": "009999", + "description": "Bancário, gamer, lutador de Kung-fu, escritor de contos, tarólogo, acupuntor, leitor, futurista e pseudo-humorista. Assim sou/estou eu.", + "profile_sidebar_border_color": "eeeeee", + "url": "http://sofistaprateado.tumblr.com", + "time_zone": "Brasilia", + "default_profile_image": false, + "statuses_count": 11251, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 76, + "friends_count": 123, + "profile_background_color": "131516", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme14/bg.gif", + "created_at": "Sat Dec 12 06:54:04 +0000 2009", + "followers_count": 123, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme14/bg.gif", + "id": 96293863, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1514569915/aquiles_normal.JPG", + "profile_image_url": "http://a3.twimg.com/profile_images/1514569915/aquiles_normal.JPG" + }, + "favorited": false, + "in_reply_to_status_id_str": "109553278831951872", + "geo": null, + "in_reply_to_screen_name": "DalaiLama", + "in_reply_to_user_id_str": "20609518", + "coordinates": null, + "in_reply_to_user_id": 20609518, + "source": "web", + "created_at": "Fri Sep 02 10:07:27 +0000 2011", + "contributors": null, + "retweeted": false, + "retweet_count": 0, + "id": 109568108846530560, + "place": null, + "text": "A prática de altruísmo é o modo autêntico de viver uma vida humana; não é limitada apenas aos religiosos. (via @DalaiLama) #fb" + } }, - "kind": "Tweet", - "value": { - "id_str": "109567960246530048", - "in_reply_to_status_id": 109553278831951870, - "truncated": false, - "user": { - "profile_sidebar_fill_color": "DDEEF6", - "protected": false, - "id_str": "76316278", - "default_profile": true, - "notifications": null, - "profile_background_tile": false, - "screen_name": "DrMradul", - "name": "Mradul Kaushik", - "listed_count": 0, - "location": null, - "show_all_inline_media": false, - "contributors_enabled": false, - "following": null, - "geo_enabled": false, - "utc_offset": null, - "profile_link_color": "0084B4", - "description": null, - "profile_sidebar_border_color": "C0DEED", - "url": null, - "time_zone": null, - "default_profile_image": true, - "statuses_count": 21, - "profile_use_background_image": true, - "verified": false, - "favourites_count": 1, - "friends_count": 26, - "profile_background_color": "C0DEED", - "is_translator": false, - "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", - "created_at": "Tue Sep 22 12:11:22 +0000 2009", - "followers_count": 9, - "follow_request_sent": null, - "lang": "en", - "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", - "id": 76316278, - "profile_text_color": "333333", - "profile_image_url_https": "https://si0.twimg.com/sticky/default_profile_images/default_profile_3_normal.png", - "profile_image_url": "http://a2.twimg.com/sticky/default_profile_images/default_profile_3_normal.png" + { + "score": 1, + "annotations": { + "ConversationRole": "Fork" }, - "favorited": false, - "in_reply_to_status_id_str": "109553278831951872", - "geo": null, - "in_reply_to_screen_name": "DalaiLama", - "in_reply_to_user_id_str": "20609518", - "coordinates": null, - "in_reply_to_user_id": 20609518, - "source": "Twitter for BlackBerry®", - "created_at": "Fri Sep 02 10:06:52 +0000 2011", - "contributors": null, - "retweeted": false, - "retweet_count": 0, - "id": 109567960246530050, - "place": null, - "text": "@DalaiLama whatever is to be followed righteously is the true religion" - } - }, - { - "score": 1, - "annotations": { - "ConversationRole": "Fork" + "kind": "Tweet", + "value": { + "id_str": "109567960246530048", + "in_reply_to_status_id": 109553278831951870, + "truncated": false, + "user": { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "76316278", + "default_profile": true, + "notifications": null, + "profile_background_tile": false, + "screen_name": "DrMradul", + "name": "Mradul Kaushik", + "listed_count": 0, + "location": null, + "show_all_inline_media": false, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": null, + "profile_link_color": "0084B4", + "description": null, + "profile_sidebar_border_color": "C0DEED", + "url": null, + "time_zone": null, + "default_profile_image": true, + "statuses_count": 21, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 1, + "friends_count": 26, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Tue Sep 22 12:11:22 +0000 2009", + "followers_count": 9, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 76316278, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/sticky/default_profile_images/default_profile_3_normal.png", + "profile_image_url": "http://a2.twimg.com/sticky/default_profile_images/default_profile_3_normal.png" + }, + "favorited": false, + "in_reply_to_status_id_str": "109553278831951872", + "geo": null, + "in_reply_to_screen_name": "DalaiLama", + "in_reply_to_user_id_str": "20609518", + "coordinates": null, + "in_reply_to_user_id": 20609518, + "source": "Twitter for BlackBerry®", + "created_at": "Fri Sep 02 10:06:52 +0000 2011", + "contributors": null, + "retweeted": false, + "retweet_count": 0, + "id": 109567960246530050, + "place": null, + "text": "@DalaiLama whatever is to be followed righteously is the true religion" + } }, - "kind": "Tweet", - "value": { - "id_str": "109567497652543488", - "in_reply_to_status_id": 109553278831951870, - "truncated": false, - "user": { - "profile_sidebar_fill_color": "7AC3EE", - "protected": false, - "id_str": "186321654", - "default_profile": false, - "notifications": null, - "profile_background_tile": true, - "screen_name": "antdigdogdagity", - "name": "Antony Blake Munro", - "listed_count": 0, - "location": "New Zealand, Auckland", - "show_all_inline_media": false, - "contributors_enabled": false, - "following": null, - "geo_enabled": false, - "utc_offset": 43200, - "profile_link_color": "FF0000", - "description": "I like magic, witchcraft, voodoo, buddhism, any thing goes, entertain me.", - "profile_sidebar_border_color": "65B0DA", - "url": "http://www.facebook.com/antony.munro", - "time_zone": "Auckland", - "default_profile_image": false, - "statuses_count": 79, - "profile_use_background_image": true, - "verified": false, - "favourites_count": 1, - "friends_count": 173, - "profile_background_color": "642D8B", - "is_translator": false, - "profile_background_image_url": "http://a1.twimg.com/images/themes/theme10/bg.gif", - "created_at": "Fri Sep 03 03:51:49 +0000 2010", - "followers_count": 26, - "follow_request_sent": null, - "lang": "en", - "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme10/bg.gif", - "id": 186321654, - "profile_text_color": "3D1957", - "profile_image_url_https": "https://si0.twimg.com/profile_images/1509387282/antdog_normal.jpg", - "profile_image_url": "http://a0.twimg.com/profile_images/1509387282/antdog_normal.jpg" + { + "score": 1, + "annotations": { + "ConversationRole": "Fork" }, - "favorited": false, - "in_reply_to_status_id_str": "109553278831951872", - "geo": null, - "in_reply_to_screen_name": "DalaiLama", - "in_reply_to_user_id_str": "20609518", - "coordinates": null, - "in_reply_to_user_id": 20609518, - "source": "web", - "created_at": "Fri Sep 02 10:05:01 +0000 2011", - "contributors": null, - "retweeted": false, - "retweet_count": 0, - "id": 109567497652543490, - "place": null, - "text": "@DalaiLama altruism is very interesting ill think about that." - } - }, - { - "score": 1, - "annotations": { - "ConversationRole": "Fork" + "kind": "Tweet", + "value": { + "id_str": "109567497652543488", + "in_reply_to_status_id": 109553278831951870, + "truncated": false, + "user": { + "profile_sidebar_fill_color": "7AC3EE", + "protected": false, + "id_str": "186321654", + "default_profile": false, + "notifications": null, + "profile_background_tile": true, + "screen_name": "antdigdogdagity", + "name": "Antony Blake Munro", + "listed_count": 0, + "location": "New Zealand, Auckland", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": 43200, + "profile_link_color": "FF0000", + "description": "I like magic, witchcraft, voodoo, buddhism, any thing goes, entertain me.", + "profile_sidebar_border_color": "65B0DA", + "url": "http://www.facebook.com/antony.munro", + "time_zone": "Auckland", + "default_profile_image": false, + "statuses_count": 79, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 1, + "friends_count": 173, + "profile_background_color": "642D8B", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme10/bg.gif", + "created_at": "Fri Sep 03 03:51:49 +0000 2010", + "followers_count": 26, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme10/bg.gif", + "id": 186321654, + "profile_text_color": "3D1957", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1509387282/antdog_normal.jpg", + "profile_image_url": "http://a0.twimg.com/profile_images/1509387282/antdog_normal.jpg" + }, + "favorited": false, + "in_reply_to_status_id_str": "109553278831951872", + "geo": null, + "in_reply_to_screen_name": "DalaiLama", + "in_reply_to_user_id_str": "20609518", + "coordinates": null, + "in_reply_to_user_id": 20609518, + "source": "web", + "created_at": "Fri Sep 02 10:05:01 +0000 2011", + "contributors": null, + "retweeted": false, + "retweet_count": 0, + "id": 109567497652543490, + "place": null, + "text": "@DalaiLama altruism is very interesting ill think about that." + } }, - "kind": "Tweet", - "value": { - "id_str": "109566564193402880", - "in_reply_to_status_id": 109553278831951870, - "truncated": false, - "user": { - "profile_sidebar_fill_color": "DDEEF6", - "protected": false, - "id_str": "261139101", - "default_profile": true, - "notifications": null, - "profile_background_tile": false, - "screen_name": "JohnGeoMatthews", - "name": "John George Matthews", - "listed_count": 0, - "location": "Margate", - "show_all_inline_media": true, - "contributors_enabled": false, - "following": null, - "geo_enabled": true, - "utc_offset": 0, - "profile_link_color": "0084B4", - "description": "Morris Minor wedding car driver, www.johngmatthews.co.uk, car restorer, guitarist a band:- http://www.mentalblock.info, guitar teacher, mobile recordist. ", - "profile_sidebar_border_color": "C0DEED", - "url": "http://www.johngmatthews.co.uk/", - "time_zone": "London", - "default_profile_image": false, - "statuses_count": 314, - "profile_use_background_image": true, - "verified": false, - "favourites_count": 0, - "friends_count": 86, - "profile_background_color": "C0DEED", - "is_translator": false, - "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", - "created_at": "Sat Mar 05 09:37:20 +0000 2011", - "followers_count": 20, - "follow_request_sent": null, - "lang": "en", - "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", - "id": 261139101, - "profile_text_color": "333333", - "profile_image_url_https": "https://si0.twimg.com/profile_images/1262832286/Twitter_pic_normal.jpg", - "profile_image_url": "http://a1.twimg.com/profile_images/1262832286/Twitter_pic_normal.jpg" + { + "score": 1, + "annotations": { + "ConversationRole": "Fork" }, - "favorited": false, - "in_reply_to_status_id_str": "109553278831951872", - "geo": null, - "in_reply_to_screen_name": "DalaiLama", - "in_reply_to_user_id_str": "20609518", - "coordinates": null, - "in_reply_to_user_id": 20609518, - "source": "web", - "created_at": "Fri Sep 02 10:01:19 +0000 2011", - "contributors": null, - "retweeted": false, - "retweet_count": 0, - "id": 109566564193402880, - "place": null, - "text": "@DalaiLama- altruism- If only more people would live by this, then the world would be a much better place. altruism = all true ism" - } - }, - { - "score": 1, - "annotations": { - "ConversationRole": "Fork" + "kind": "Tweet", + "value": { + "id_str": "109566564193402880", + "in_reply_to_status_id": 109553278831951870, + "truncated": false, + "user": { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "261139101", + "default_profile": true, + "notifications": null, + "profile_background_tile": false, + "screen_name": "JohnGeoMatthews", + "name": "John George Matthews", + "listed_count": 0, + "location": "Margate", + "show_all_inline_media": true, + "contributors_enabled": false, + "following": null, + "geo_enabled": true, + "utc_offset": 0, + "profile_link_color": "0084B4", + "description": "Morris Minor wedding car driver, www.johngmatthews.co.uk, car restorer, guitarist a band:- http://www.mentalblock.info, guitar teacher, mobile recordist. ", + "profile_sidebar_border_color": "C0DEED", + "url": "http://www.johngmatthews.co.uk/", + "time_zone": "London", + "default_profile_image": false, + "statuses_count": 314, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 0, + "friends_count": 86, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Sat Mar 05 09:37:20 +0000 2011", + "followers_count": 20, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 261139101, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1262832286/Twitter_pic_normal.jpg", + "profile_image_url": "http://a1.twimg.com/profile_images/1262832286/Twitter_pic_normal.jpg" + }, + "favorited": false, + "in_reply_to_status_id_str": "109553278831951872", + "geo": null, + "in_reply_to_screen_name": "DalaiLama", + "in_reply_to_user_id_str": "20609518", + "coordinates": null, + "in_reply_to_user_id": 20609518, + "source": "web", + "created_at": "Fri Sep 02 10:01:19 +0000 2011", + "contributors": null, + "retweeted": false, + "retweet_count": 0, + "id": 109566564193402880, + "place": null, + "text": "@DalaiLama- altruism- If only more people would live by this, then the world would be a much better place. altruism = all true ism" + } }, - "kind": "Tweet", - "value": { - "id_str": "109566508916670464", - "in_reply_to_status_id": 109553278831951870, - "truncated": false, - "user": { - "profile_sidebar_fill_color": "DDEEF6", - "protected": false, - "id_str": "362947094", - "default_profile": true, - "notifications": null, - "profile_background_tile": false, - "screen_name": "KARANHIRWAT", - "name": "KARANHIRWAT", - "listed_count": 0, - "location": "", - "show_all_inline_media": false, - "contributors_enabled": false, - "following": null, - "geo_enabled": false, - "utc_offset": null, - "profile_link_color": "0084B4", - "description": "", - "profile_sidebar_border_color": "C0DEED", - "url": null, - "time_zone": null, - "default_profile_image": false, - "statuses_count": 90, - "profile_use_background_image": true, - "verified": false, - "favourites_count": 1, - "friends_count": 76, - "profile_background_color": "C0DEED", - "is_translator": false, - "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", - "created_at": "Sat Aug 27 07:41:23 +0000 2011", - "followers_count": 16, - "follow_request_sent": null, - "lang": "en", - "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", - "id": 362947094, - "profile_text_color": "333333", - "profile_image_url_https": "https://si0.twimg.com/profile_images/1515548535/karan_new_pic_normal.jpg", - "profile_image_url": "http://a0.twimg.com/profile_images/1515548535/karan_new_pic_normal.jpg" + { + "score": 1, + "annotations": { + "ConversationRole": "Fork" }, - "favorited": false, - "in_reply_to_status_id_str": "109553278831951872", - "geo": null, - "in_reply_to_screen_name": "DalaiLama", - "in_reply_to_user_id_str": "20609518", - "coordinates": null, - "in_reply_to_user_id": 20609518, - "source": "web", - "created_at": "Fri Sep 02 10:01:06 +0000 2011", - "contributors": null, - "retweeted": false, - "retweet_count": 0, - "id": 109566508916670460, - "place": null, - "text": "@DalaiLama The tongue like a sharp knife... Kills without drawing blood" + "kind": "Tweet", + "value": { + "id_str": "109566508916670464", + "in_reply_to_status_id": 109553278831951870, + "truncated": false, + "user": { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "362947094", + "default_profile": true, + "notifications": null, + "profile_background_tile": false, + "screen_name": "KARANHIRWAT", + "name": "KARANHIRWAT", + "listed_count": 0, + "location": "", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": null, + "profile_link_color": "0084B4", + "description": "", + "profile_sidebar_border_color": "C0DEED", + "url": null, + "time_zone": null, + "default_profile_image": false, + "statuses_count": 90, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 1, + "friends_count": 76, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Sat Aug 27 07:41:23 +0000 2011", + "followers_count": 16, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 362947094, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1515548535/karan_new_pic_normal.jpg", + "profile_image_url": "http://a0.twimg.com/profile_images/1515548535/karan_new_pic_normal.jpg" + }, + "favorited": false, + "in_reply_to_status_id_str": "109553278831951872", + "geo": null, + "in_reply_to_screen_name": "DalaiLama", + "in_reply_to_user_id_str": "20609518", + "coordinates": null, + "in_reply_to_user_id": 20609518, + "source": "web", + "created_at": "Fri Sep 02 10:01:06 +0000 2011", + "contributors": null, + "retweeted": false, + "retweet_count": 0, + "id": 109566508916670460, + "place": null, + "text": "@DalaiLama The tongue like a sharp knife... Kills without drawing blood" + } } - }], + ], "resultType": "Tweet", "groupName": "TweetsWithConversation", "annotations": { @@ -554,690 +552,690 @@ "score": 1 }, { - "results": [{ - "profile_sidebar_fill_color": "DDEEF6", - "protected": false, - "id_str": "348062004", - "notifications": null, - "profile_background_tile": false, - "screen_name": "seshdaherbaliss", - "name": "seshi", - "listed_count": 0, - "location": "tema", - "show_all_inline_media": false, - "contributors_enabled": false, - "following": null, - "geo_enabled": false, - "utc_offset": null, - "profile_link_color": "0084B4", - "description": "last killer", - "profile_sidebar_border_color": "C0DEED", - "url": null, - "default_profile": true, - "time_zone": null, - "default_profile_image": true, - "statuses_count": 60, - "profile_use_background_image": true, - "verified": false, - "favourites_count": 0, - "friends_count": 13, - "profile_background_color": "C0DEED", - "is_translator": false, - "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", - "created_at": "Wed Aug 03 20:26:46 +0000 2011", - "followers_count": 7, - "follow_request_sent": null, - "lang": "en", - "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", - "id": 348062004, - "profile_text_color": "333333", - "profile_image_url_https": "https://si0.twimg.com/sticky/default_profile_images/default_profile_3_normal.png", - "profile_image_url": "http://a2.twimg.com/sticky/default_profile_images/default_profile_3_normal.png" - }, - { - "profile_sidebar_fill_color": "99CC33", - "protected": false, - "id_str": "107930046", - "notifications": false, - "profile_background_tile": true, - "screen_name": "paolaandrea25", - "name": "Paola Andrea☆", - "listed_count": 0, - "location": "Bogota", - "show_all_inline_media": false, - "contributors_enabled": false, - "following": false, - "geo_enabled": true, - "utc_offset": -18000, - "profile_link_color": "D02B55", - "description": "Adm. de Empresas, Bogotana!!! desarrollando mi proyecto SHAROVA DAY SPA, Me gusta Soñar en cada momento... Dios siempre cumple mis sueños !) 100% Feliz. ", - "profile_sidebar_border_color": "829D5E", - "url": null, - "time_zone": "Bogota", - "default_profile_image": false, - "statuses_count": 74, - "profile_use_background_image": true, - "default_profile": false, - "verified": false, - "favourites_count": 5, - "friends_count": 278, - "profile_background_color": "352726", - "is_translator": false, - "profile_background_image_url": "http://a0.twimg.com/profile_background_images/294286892/kjhkjk.jpg", - "created_at": "Sun Jan 24 06:39:53 +0000 2010", - "followers_count": 75, - "follow_request_sent": false, - "lang": "es", - "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/294286892/kjhkjk.jpg", - "id": 107930046, - "profile_text_color": "3E4415", - "profile_image_url_https": "https://si0.twimg.com/profile_images/1431039138/IMG01029-20100603-1450_normal.jpg", - "profile_image_url": "http://a0.twimg.com/profile_images/1431039138/IMG01029-20100603-1450_normal.jpg" - }, - { - "profile_sidebar_fill_color": "DDEEF6", - "protected": false, - "id_str": "273380258", - "notifications": false, - "profile_background_tile": false, - "screen_name": "Positivestarts", - "name": "Positive Starts", - "listed_count": 0, - "location": null, - "show_all_inline_media": true, - "contributors_enabled": false, - "following": false, - "geo_enabled": false, - "utc_offset": -18000, - "profile_link_color": "0084B4", - "description": "Life is a short ride. Negativity can lead to stress. Reflect on the positive to drive away the negative.", - "profile_sidebar_border_color": "C0DEED", - "url": null, - "time_zone": "Quito", - "default_profile_image": false, - "statuses_count": 395, - "profile_use_background_image": true, - "verified": false, - "favourites_count": 18, - "friends_count": 15, - "profile_background_color": "C0DEED", - "is_translator": false, - "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", - "created_at": "Mon Mar 28 11:29:47 +0000 2011", - "followers_count": 28, - "default_profile": true, - "follow_request_sent": false, - "lang": "en", - "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", - "id": 273380258, - "profile_text_color": "333333", - "profile_image_url_https": "https://si0.twimg.com/profile_images/1290603834/image_normal.jpg", - "profile_image_url": "http://a2.twimg.com/profile_images/1290603834/image_normal.jpg" - }, - { - "profile_sidebar_fill_color": "5bda1b", - "protected": false, - "id_str": "36706744", - "notifications": false, - "profile_background_tile": false, - "screen_name": "Herrardo", - "name": "Gerardo Torres", - "listed_count": 15, - "location": "Mexico City", - "show_all_inline_media": false, - "contributors_enabled": false, - "following": false, - "geo_enabled": false, - "utc_offset": -21600, - "profile_link_color": "1203fc", - "description": "I love playin' guitar! Or bass! Having fun! To party! Read, watch movies, learn something interesting!", - "profile_sidebar_border_color": "000000", - "url": "http://www.facebook.com/Gerrytn", - "time_zone": "Mexico City", - "default_profile_image": false, - "statuses_count": 4483, - "profile_use_background_image": true, - "default_profile": false, - "verified": false, - "favourites_count": 17, - "friends_count": 266, - "profile_background_color": "25a290", - "is_translator": false, - "profile_background_image_url": "http://a1.twimg.com/profile_background_images/13762946/DSC00295.JPG", - "created_at": "Thu Apr 30 16:38:47 +0000 2009", - "followers_count": 156, - "follow_request_sent": false, - "lang": "en", - "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/13762946/DSC00295.JPG", - "id": 36706744, - "profile_text_color": "333333", - "profile_image_url_https": "https://si0.twimg.com/profile_images/1303959010/yop_normal.jpg", - "profile_image_url": "http://a3.twimg.com/profile_images/1303959010/yop_normal.jpg" - }, - { - "profile_sidebar_fill_color": "DDEEF6", - "protected": false, - "id_str": "199273449", - "notifications": false, - "profile_background_tile": false, - "screen_name": "sitarabeling", - "name": "sita rabeling", - "listed_count": 0, - "location": "Netherlands", - "show_all_inline_media": false, - "contributors_enabled": false, - "following": false, - "geo_enabled": false, - "utc_offset": 3600, - "profile_link_color": "0084B4", - "description": "", - "profile_sidebar_border_color": "C0DEED", - "url": null, - "time_zone": "Amsterdam", - "default_profile_image": false, - "statuses_count": 120, - "profile_use_background_image": true, - "default_profile": true, - "verified": false, - "favourites_count": 0, - "friends_count": 47, - "profile_background_color": "C0DEED", - "is_translator": false, - "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", - "created_at": "Wed Oct 06 13:17:11 +0000 2010", - "followers_count": 11, - "follow_request_sent": false, - "lang": "en", - "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", - "id": 199273449, - "profile_text_color": "333333", - "profile_image_url_https": "https://si0.twimg.com/profile_images/1379852945/IMG_0195_normal.jpg", - "profile_image_url": "http://a2.twimg.com/profile_images/1379852945/IMG_0195_normal.jpg" - }, - { - "profile_sidebar_fill_color": "E6F6F9", - "protected": false, - "id_str": "47444972", - "notifications": false, - "profile_background_tile": false, - "screen_name": "MooreofMary", - "name": "Mary Moore", - "listed_count": 8, - "location": "Oklahoma", - "show_all_inline_media": false, - "contributors_enabled": false, - "following": false, - "geo_enabled": false, - "utc_offset": -21600, - "profile_link_color": "CC3366", - "description": "Morning Radio Show Cohost. Sports Reporter. Emcee. Workaholic. ", - "profile_sidebar_border_color": "DBE9ED", - "url": "http://www.facebook.com/pages/Mary-MooreTULSA/160802983954993", - "time_zone": "Central Time (US & Canada)", - "default_profile_image": false, - "statuses_count": 3331, - "profile_use_background_image": false, - "verified": false, - "favourites_count": 3, - "friends_count": 245, - "profile_background_color": "DBE9ED", - "is_translator": false, - "default_profile": false, - "profile_background_image_url": "http://a1.twimg.com/images/themes/theme17/bg.gif", - "created_at": "Mon Jun 15 21:07:55 +0000 2009", - "followers_count": 455, - "follow_request_sent": false, - "lang": "en", - "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme17/bg.gif", - "id": 47444972, - "profile_text_color": "333333", - "profile_image_url_https": "https://si0.twimg.com/profile_images/1505124747/image_normal.jpg", - "profile_image_url": "http://a2.twimg.com/profile_images/1505124747/image_normal.jpg" - }, - { - "profile_sidebar_fill_color": "252429", - "protected": false, - "id_str": "28393220", - "notifications": false, - "profile_background_tile": false, - "screen_name": "picyadri", - "name": "Adrienn Szikszay", - "listed_count": 0, - "location": "Hungary", - "show_all_inline_media": true, - "contributors_enabled": false, - "following": false, - "geo_enabled": false, - "utc_offset": 3600, - "profile_link_color": "edc345", - "description": "Optimistic. Loves a bunch of stuff. Always smiling.", - "profile_sidebar_border_color": "f1bd0e", - "url": null, - "time_zone": "Budapest", - "default_profile_image": false, - "statuses_count": 65, - "profile_use_background_image": true, - "verified": false, - "favourites_count": 0, - "friends_count": 114, - "profile_background_color": "1A1B1F", - "is_translator": false, - "default_profile": false, - "profile_background_image_url": "http://a1.twimg.com/images/themes/theme9/bg.gif", - "created_at": "Thu Apr 02 18:43:08 +0000 2009", - "followers_count": 53, - "follow_request_sent": false, - "lang": "en", - "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme9/bg.gif", - "id": 28393220, - "profile_text_color": "aba6a6", - "profile_image_url_https": "https://si0.twimg.com/profile_images/121702135/adrisapiban_normal.jpg", - "profile_image_url": "http://a1.twimg.com/profile_images/121702135/adrisapiban_normal.jpg" - }, - { - "profile_sidebar_fill_color": "bd77bd", - "protected": false, - "id_str": "97671153", - "notifications": false, - "profile_background_tile": true, - "screen_name": "purple9dragon", - "name": "purple dragon", - "listed_count": 2, - "location": "天空の城ラプタ", - "show_all_inline_media": false, - "contributors_enabled": false, - "following": false, - "geo_enabled": false, - "utc_offset": 32400, - "profile_link_color": "0055ff", - "description": "民間人 フォローミー、フォローユー 趣味は盆栽", - "profile_sidebar_border_color": "6cd966", - "url": null, - "time_zone": "Osaka", - "default_profile_image": false, - "statuses_count": 1556, - "profile_use_background_image": true, - "verified": false, - "favourites_count": 6, - "friends_count": 112, - "profile_background_color": "000000", - "is_translator": false, - "default_profile": false, - "profile_background_image_url": "http://a1.twimg.com/profile_background_images/63728857/the-eye-dl.jpg", - "created_at": "Fri Dec 18 13:55:27 +0000 2009", - "followers_count": 62, - "follow_request_sent": false, - "lang": "ja", - "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/63728857/the-eye-dl.jpg", - "id": 97671153, - "profile_text_color": "000000", - "profile_image_url_https": "https://si0.twimg.com/profile_images/938357687/exit_normal.jpg", - "profile_image_url": "http://a1.twimg.com/profile_images/938357687/exit_normal.jpg" - }, - { - "profile_sidebar_fill_color": "4f468c", - "protected": false, - "id_str": "23290095", - "notifications": false, - "profile_background_tile": true, - "screen_name": "RachelCappucci", - "name": "Rachel Cappucci", - "listed_count": 2, - "location": "New York City", - "show_all_inline_media": false, - "contributors_enabled": false, - "following": false, - "geo_enabled": false, - "utc_offset": -18000, - "profile_link_color": "d911cf", - "description": " The person who risks nothing, does nothing, has nothing, is nothing. ", - "profile_sidebar_border_color": "d678d6", - "url": null, - "time_zone": "Eastern Time (US & Canada)", - "default_profile_image": false, - "statuses_count": 1810, - "profile_use_background_image": true, - "verified": false, - "favourites_count": 5, - "friends_count": 335, - "profile_background_color": "3e416b", - "is_translator": false, - "default_profile": false, - "profile_background_image_url": "http://a1.twimg.com/profile_background_images/87659392/6040_143426809917_610109917_3324938_5333854_n.jpg", - "created_at": "Sun Mar 08 08:30:14 +0000 2009", - "followers_count": 121, - "follow_request_sent": false, - "lang": "en", - "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/87659392/6040_143426809917_610109917_3324938_5333854_n.jpg", - "id": 23290095, - "profile_text_color": "ed9aed", - "profile_image_url_https": "https://si0.twimg.com/profile_images/391846681/twitter_this_normal.jpg", - "profile_image_url": "http://a1.twimg.com/profile_images/391846681/twitter_this_normal.jpg" - }, - { - "profile_sidebar_fill_color": "DDEEF6", - "protected": false, - "id_str": "209990451", - "notifications": false, - "profile_background_tile": false, - "screen_name": "penfoldholes", - "name": "Richard Barker", - "listed_count": 0, - "location": "Shefford", - "show_all_inline_media": false, - "contributors_enabled": false, - "following": false, - "geo_enabled": true, - "utc_offset": null, - "profile_link_color": "0084B4", - "description": "kidney cancer survivor - former footballer and cricketer - wine , dog and music lover.", - "profile_sidebar_border_color": "C0DEED", - "url": null, - "time_zone": null, - "default_profile_image": false, - "statuses_count": 1608, - "profile_use_background_image": true, - "verified": false, - "favourites_count": 7, - "friends_count": 66, - "profile_background_color": "C0DEED", - "is_translator": false, - "default_profile": true, - "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", - "created_at": "Sat Oct 30 09:40:15 +0000 2010", - "followers_count": 52, - "follow_request_sent": false, - "lang": "en", - "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", - "id": 209990451, - "profile_text_color": "333333", - "profile_image_url_https": "https://si0.twimg.com/profile_images/1157214189/Richard_normal.jpg", - "profile_image_url": "http://a0.twimg.com/profile_images/1157214189/Richard_normal.jpg" - }, - { - "profile_sidebar_fill_color": "F6F6F6", - "protected": false, - "id_str": "90737592", - "notifications": false, - "profile_background_tile": false, - "screen_name": "coti_rodriguez", - "name": "Constanza Rodriguez", - "listed_count": 0, - "location": "Monte Hermoso", - "show_all_inline_media": false, - "contributors_enabled": false, - "following": false, - "geo_enabled": true, - "utc_offset": -10800, - "profile_link_color": "038543", - "description": "", - "profile_sidebar_border_color": "EEEEEE", - "url": null, - "time_zone": "Buenos Aires", - "default_profile_image": false, - "default_profile": false, - "statuses_count": 276, - "profile_use_background_image": true, - "verified": false, - "favourites_count": 2, - "friends_count": 153, - "profile_background_color": "ACDED6", - "is_translator": false, - "profile_background_image_url": "http://a1.twimg.com/images/themes/theme18/bg.gif", - "created_at": "Tue Nov 17 22:36:14 +0000 2009", - "followers_count": 48, - "follow_request_sent": false, - "lang": "es", - "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme18/bg.gif", - "id": 90737592, - "profile_text_color": "333333", - "profile_image_url_https": "https://si0.twimg.com/profile_images/1098075067/DSCN1664_normal.JPG", - "profile_image_url": "http://a0.twimg.com/profile_images/1098075067/DSCN1664_normal.JPG" - }, - { - "profile_sidebar_fill_color": "DDEEF6", - "protected": false, - "id_str": "111380497", - "notifications": false, - "profile_background_tile": false, - "screen_name": "PROFAMS", - "name": "ANTONIO MARCOS", - "listed_count": 0, - "location": "São Paulo", - "show_all_inline_media": false, - "contributors_enabled": false, - "following": false, - "geo_enabled": true, - "utc_offset": -10800, - "profile_link_color": "0084B4", - "description": "Gerente Nacional de Marketing e Professor nas áreas de adminisração e marketing.", - "profile_sidebar_border_color": "C0DEED", - "url": null, - "time_zone": "Brasilia", - "default_profile_image": false, - "default_profile": true, - "statuses_count": 124, - "profile_use_background_image": true, - "verified": false, - "favourites_count": 5, - "friends_count": 44, - "profile_background_color": "C0DEED", - "is_translator": false, - "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", - "created_at": "Thu Feb 04 18:34:41 +0000 2010", - "followers_count": 45, - "follow_request_sent": false, - "lang": "en", - "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", - "id": 111380497, - "profile_text_color": "333333", - "profile_image_url_https": "https://si0.twimg.com/profile_images/1309327114/AMS_3_normal.JPG", - "profile_image_url": "http://a2.twimg.com/profile_images/1309327114/AMS_3_normal.JPG" - }, - { - "profile_sidebar_fill_color": "333333", - "protected": false, - "id_str": "19216042", - "notifications": false, - "profile_background_tile": true, - "screen_name": "anaverena", - "name": "Ana Verena Menezes", - "listed_count": 14, - "location": "Salvador,Bahia,Brasil", - "show_all_inline_media": false, - "contributors_enabled": false, - "following": false, - "geo_enabled": true, - "utc_offset": -10800, - "profile_link_color": "b36868", - "description": "all my colours turn to clouds.", - "profile_sidebar_border_color": "b6b8b6", - "url": "http://www.anaverena.tumblr.com", - "time_zone": "Brasilia", - "default_profile_image": false, - "default_profile": false, - "statuses_count": 8298, - "profile_use_background_image": true, - "verified": false, - "favourites_count": 35, - "friends_count": 149, - "profile_background_color": "ffffff", - "is_translator": false, - "profile_background_image_url": "http://a1.twimg.com/profile_background_images/290833153/ikea_lena_2.jpg", - "created_at": "Tue Jan 20 02:45:47 +0000 2009", - "followers_count": 271, - "follow_request_sent": false, - "lang": "en", - "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/290833153/ikea_lena_2.jpg", - "id": 19216042, - "profile_text_color": "b8b2b2", - "profile_image_url_https": "https://si0.twimg.com/profile_images/1445007999/IMG_7694_copie2_normal.jpg", - "profile_image_url": "http://a2.twimg.com/profile_images/1445007999/IMG_7694_copie2_normal.jpg" - }, - { - "profile_sidebar_fill_color": "fa89c9", - "protected": false, - "id_str": "192627716", - "notifications": false, - "profile_background_tile": true, - "screen_name": "soouzalaura", - "name": "Laura Souza", - "listed_count": 2, - "location": "Brasil", - "show_all_inline_media": false, - "contributors_enabled": false, - "following": false, - "geo_enabled": true, - "utc_offset": -10800, - "profile_link_color": "942394", - "description": "Welcome to real world. It's suck, you're gonna love it! (Friends)", - "profile_sidebar_border_color": "ed26bf", - "url": "http://laurapriscila.tumblr.com/", - "time_zone": "Brasilia", - "default_profile_image": false, - "statuses_count": 2218, - "profile_use_background_image": true, - "default_profile": false, - "verified": false, - "favourites_count": 2, - "friends_count": 119, - "profile_background_color": "ffffff", - "is_translator": false, - "profile_background_image_url": "http://a1.twimg.com/profile_background_images/237284023/bg.gif", - "created_at": "Sun Sep 19 18:12:21 +0000 2010", - "followers_count": 61, - "follow_request_sent": false, - "lang": "pt", - "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/237284023/bg.gif", - "id": 192627716, - "profile_text_color": "524646", - "profile_image_url_https": "https://si0.twimg.com/profile_images/1422000163/SAM_0539_normal.JPG", - "profile_image_url": "http://a0.twimg.com/profile_images/1422000163/SAM_0539_normal.JPG" - }, - { - "profile_sidebar_fill_color": "efefef", - "protected": false, - "id_str": "58000577", - "notifications": false, - "profile_background_tile": true, - "screen_name": "colchica", - "name": "C Orne", - "listed_count": 0, - "location": "Lyon - France", - "show_all_inline_media": false, - "contributors_enabled": false, - "following": false, - "geo_enabled": true, - "utc_offset": 3600, - "profile_link_color": "2d66ad", - "description": "addicted to caffeine and a little bit greener... ", - "default_profile": false, - "profile_sidebar_border_color": "eeeeee", - "url": null, - "time_zone": "Paris", - "default_profile_image": false, - "statuses_count": 232, - "profile_use_background_image": true, - "verified": false, - "favourites_count": 1, - "friends_count": 45, - "profile_background_color": "ebeef0", - "is_translator": false, - "profile_background_image_url": "http://a3.twimg.com/profile_background_images/154201528/20100912103242_2010-09-11_eboulis_05.jpg", - "created_at": "Sat Jul 18 18:44:29 +0000 2009", - "followers_count": 14, - "follow_request_sent": false, - "lang": "fr", - "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/154201528/20100912103242_2010-09-11_eboulis_05.jpg", - "id": 58000577, - "profile_text_color": "994399", - "profile_image_url_https": "https://si0.twimg.com/profile_images/336483751/samantha_normal.jpg", - "profile_image_url": "http://a0.twimg.com/profile_images/336483751/samantha_normal.jpg" - }, - { - "profile_sidebar_fill_color": "c9c9c9", - "protected": false, - "id_str": "67218346", - "notifications": null, - "profile_background_tile": true, - "screen_name": "GrandyManFTW14", - "name": "Valerie", - "listed_count": 39, - "location": "Yankee Stadium", - "show_all_inline_media": true, - "contributors_enabled": false, - "following": null, - "geo_enabled": false, - "utc_offset": -18000, - "profile_link_color": "2138B4", - "description": "im a nerd thats infatuated with sports... mostly #yankees baseball. I should reside in a bubble. I'm 28 and speak my mind.", - "profile_sidebar_border_color": "bfbfbf", - "url": "http://www.facebook.com/yankeesangel21", - "default_profile": false, - "time_zone": "Eastern Time (US & Canada)", - "default_profile_image": false, - "statuses_count": 8744, - "profile_use_background_image": true, - "verified": false, - "favourites_count": 11, - "friends_count": 653, - "profile_background_color": "07090b", - "is_translator": false, - "profile_background_image_url": "http://a1.twimg.com/profile_background_images/202039528/x1f1841c56244072a549acc188477e1e.jpg", - "created_at": "Thu Aug 20 03:43:17 +0000 2009", - "followers_count": 500, - "follow_request_sent": null, - "lang": "en", - "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/202039528/x1f1841c56244072a549acc188477e1e.jpg", - "id": 67218346, - "profile_text_color": "183050", - "profile_image_url_https": "https://si0.twimg.com/profile_images/1520181363/image_normal.jpg", - "profile_image_url": "http://a2.twimg.com/profile_images/1520181363/image_normal.jpg" - }, - { - "profile_sidebar_fill_color": "95E8EC", - "protected": false, - "id_str": "185014625", - "default_profile": false, - "notifications": false, - "profile_background_tile": false, - "screen_name": "chiquip35", - "name": "Paula Klimowitz", - "listed_count": 1, - "location": "Miami", - "show_all_inline_media": false, - "contributors_enabled": false, - "following": false, - "geo_enabled": false, - "utc_offset": null, - "profile_link_color": "0099B9", - "description": "I love the positive energy of the people of the world!!", - "profile_sidebar_border_color": "5ED4DC", - "url": null, - "time_zone": null, - "default_profile_image": false, - "statuses_count": 48, - "profile_use_background_image": true, - "verified": false, - "favourites_count": 0, - "friends_count": 102, - "profile_background_color": "0099B9", - "is_translator": false, - "profile_background_image_url": "http://a1.twimg.com/images/themes/theme4/bg.gif", - "created_at": "Tue Aug 31 01:15:36 +0000 2010", - "followers_count": 9, - "follow_request_sent": false, - "lang": "en", - "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme4/bg.gif", - "id": 185014625, - "profile_text_color": "3C3940", - "profile_image_url_https": "https://si0.twimg.com/profile_images/1521529755/174758_normal.jpg", - "profile_image_url": "http://a3.twimg.com/profile_images/1521529755/174758_normal.jpg" - }], + "results": [ + { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "348062004", + "notifications": null, + "profile_background_tile": false, + "screen_name": "seshdaherbaliss", + "name": "seshi", + "listed_count": 0, + "location": "tema", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": null, + "profile_link_color": "0084B4", + "description": "last killer", + "profile_sidebar_border_color": "C0DEED", + "url": null, + "default_profile": true, + "time_zone": null, + "default_profile_image": true, + "statuses_count": 60, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 0, + "friends_count": 13, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Wed Aug 03 20:26:46 +0000 2011", + "followers_count": 7, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 348062004, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/sticky/default_profile_images/default_profile_3_normal.png", + "profile_image_url": "http://a2.twimg.com/sticky/default_profile_images/default_profile_3_normal.png" + }, + { + "profile_sidebar_fill_color": "99CC33", + "protected": false, + "id_str": "107930046", + "notifications": false, + "profile_background_tile": true, + "screen_name": "paolaandrea25", + "name": "Paola Andrea☆", + "listed_count": 0, + "location": "Bogota", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": true, + "utc_offset": -18000, + "profile_link_color": "D02B55", + "description": "Adm. de Empresas, Bogotana!!! desarrollando mi proyecto SHAROVA DAY SPA, Me gusta Soñar en cada momento... Dios siempre cumple mis sueños !) 100% Feliz. ", + "profile_sidebar_border_color": "829D5E", + "url": null, + "time_zone": "Bogota", + "default_profile_image": false, + "statuses_count": 74, + "profile_use_background_image": true, + "default_profile": false, + "verified": false, + "favourites_count": 5, + "friends_count": 278, + "profile_background_color": "352726", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/profile_background_images/294286892/kjhkjk.jpg", + "created_at": "Sun Jan 24 06:39:53 +0000 2010", + "followers_count": 75, + "follow_request_sent": false, + "lang": "es", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/294286892/kjhkjk.jpg", + "id": 107930046, + "profile_text_color": "3E4415", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1431039138/IMG01029-20100603-1450_normal.jpg", + "profile_image_url": "http://a0.twimg.com/profile_images/1431039138/IMG01029-20100603-1450_normal.jpg" + }, + { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "273380258", + "notifications": false, + "profile_background_tile": false, + "screen_name": "Positivestarts", + "name": "Positive Starts", + "listed_count": 0, + "location": null, + "show_all_inline_media": true, + "contributors_enabled": false, + "following": false, + "geo_enabled": false, + "utc_offset": -18000, + "profile_link_color": "0084B4", + "description": "Life is a short ride. Negativity can lead to stress. Reflect on the positive to drive away the negative.", + "profile_sidebar_border_color": "C0DEED", + "url": null, + "time_zone": "Quito", + "default_profile_image": false, + "statuses_count": 395, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 18, + "friends_count": 15, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Mon Mar 28 11:29:47 +0000 2011", + "followers_count": 28, + "default_profile": true, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 273380258, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1290603834/image_normal.jpg", + "profile_image_url": "http://a2.twimg.com/profile_images/1290603834/image_normal.jpg" + }, + { + "profile_sidebar_fill_color": "5bda1b", + "protected": false, + "id_str": "36706744", + "notifications": false, + "profile_background_tile": false, + "screen_name": "Herrardo", + "name": "Gerardo Torres", + "listed_count": 15, + "location": "Mexico City", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": false, + "utc_offset": -21600, + "profile_link_color": "1203fc", + "description": "I love playin' guitar! Or bass! Having fun! To party! Read, watch movies, learn something interesting!", + "profile_sidebar_border_color": "000000", + "url": "http://www.facebook.com/Gerrytn", + "time_zone": "Mexico City", + "default_profile_image": false, + "statuses_count": 4483, + "profile_use_background_image": true, + "default_profile": false, + "verified": false, + "favourites_count": 17, + "friends_count": 266, + "profile_background_color": "25a290", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/profile_background_images/13762946/DSC00295.JPG", + "created_at": "Thu Apr 30 16:38:47 +0000 2009", + "followers_count": 156, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/13762946/DSC00295.JPG", + "id": 36706744, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1303959010/yop_normal.jpg", + "profile_image_url": "http://a3.twimg.com/profile_images/1303959010/yop_normal.jpg" + }, + { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "199273449", + "notifications": false, + "profile_background_tile": false, + "screen_name": "sitarabeling", + "name": "sita rabeling", + "listed_count": 0, + "location": "Netherlands", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": false, + "utc_offset": 3600, + "profile_link_color": "0084B4", + "description": "", + "profile_sidebar_border_color": "C0DEED", + "url": null, + "time_zone": "Amsterdam", + "default_profile_image": false, + "statuses_count": 120, + "profile_use_background_image": true, + "default_profile": true, + "verified": false, + "favourites_count": 0, + "friends_count": 47, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Wed Oct 06 13:17:11 +0000 2010", + "followers_count": 11, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 199273449, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1379852945/IMG_0195_normal.jpg", + "profile_image_url": "http://a2.twimg.com/profile_images/1379852945/IMG_0195_normal.jpg" + }, + { + "profile_sidebar_fill_color": "E6F6F9", + "protected": false, + "id_str": "47444972", + "notifications": false, + "profile_background_tile": false, + "screen_name": "MooreofMary", + "name": "Mary Moore", + "listed_count": 8, + "location": "Oklahoma", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": false, + "utc_offset": -21600, + "profile_link_color": "CC3366", + "description": "Morning Radio Show Cohost. Sports Reporter. Emcee. Workaholic. ", + "profile_sidebar_border_color": "DBE9ED", + "url": "http://www.facebook.com/pages/Mary-MooreTULSA/160802983954993", + "time_zone": "Central Time (US & Canada)", + "default_profile_image": false, + "statuses_count": 3331, + "profile_use_background_image": false, + "verified": false, + "favourites_count": 3, + "friends_count": 245, + "profile_background_color": "DBE9ED", + "is_translator": false, + "default_profile": false, + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme17/bg.gif", + "created_at": "Mon Jun 15 21:07:55 +0000 2009", + "followers_count": 455, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme17/bg.gif", + "id": 47444972, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1505124747/image_normal.jpg", + "profile_image_url": "http://a2.twimg.com/profile_images/1505124747/image_normal.jpg" + }, + { + "profile_sidebar_fill_color": "252429", + "protected": false, + "id_str": "28393220", + "notifications": false, + "profile_background_tile": false, + "screen_name": "picyadri", + "name": "Adrienn Szikszay", + "listed_count": 0, + "location": "Hungary", + "show_all_inline_media": true, + "contributors_enabled": false, + "following": false, + "geo_enabled": false, + "utc_offset": 3600, + "profile_link_color": "edc345", + "description": "Optimistic. Loves a bunch of stuff. Always smiling.", + "profile_sidebar_border_color": "f1bd0e", + "url": null, + "time_zone": "Budapest", + "default_profile_image": false, + "statuses_count": 65, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 0, + "friends_count": 114, + "profile_background_color": "1A1B1F", + "is_translator": false, + "default_profile": false, + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme9/bg.gif", + "created_at": "Thu Apr 02 18:43:08 +0000 2009", + "followers_count": 53, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme9/bg.gif", + "id": 28393220, + "profile_text_color": "aba6a6", + "profile_image_url_https": "https://si0.twimg.com/profile_images/121702135/adrisapiban_normal.jpg", + "profile_image_url": "http://a1.twimg.com/profile_images/121702135/adrisapiban_normal.jpg" + }, + { + "profile_sidebar_fill_color": "bd77bd", + "protected": false, + "id_str": "97671153", + "notifications": false, + "profile_background_tile": true, + "screen_name": "purple9dragon", + "name": "purple dragon", + "listed_count": 2, + "location": "天空の城ラプタ", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": false, + "utc_offset": 32400, + "profile_link_color": "0055ff", + "description": "民間人 フォローミー、フォローユー 趣味は盆栽", + "profile_sidebar_border_color": "6cd966", + "url": null, + "time_zone": "Osaka", + "default_profile_image": false, + "statuses_count": 1556, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 6, + "friends_count": 112, + "profile_background_color": "000000", + "is_translator": false, + "default_profile": false, + "profile_background_image_url": "http://a1.twimg.com/profile_background_images/63728857/the-eye-dl.jpg", + "created_at": "Fri Dec 18 13:55:27 +0000 2009", + "followers_count": 62, + "follow_request_sent": false, + "lang": "ja", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/63728857/the-eye-dl.jpg", + "id": 97671153, + "profile_text_color": "000000", + "profile_image_url_https": "https://si0.twimg.com/profile_images/938357687/exit_normal.jpg", + "profile_image_url": "http://a1.twimg.com/profile_images/938357687/exit_normal.jpg" + }, + { + "profile_sidebar_fill_color": "4f468c", + "protected": false, + "id_str": "23290095", + "notifications": false, + "profile_background_tile": true, + "screen_name": "RachelCappucci", + "name": "Rachel Cappucci", + "listed_count": 2, + "location": "New York City", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": false, + "utc_offset": -18000, + "profile_link_color": "d911cf", + "description": " The person who risks nothing, does nothing, has nothing, is nothing. ", + "profile_sidebar_border_color": "d678d6", + "url": null, + "time_zone": "Eastern Time (US & Canada)", + "default_profile_image": false, + "statuses_count": 1810, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 5, + "friends_count": 335, + "profile_background_color": "3e416b", + "is_translator": false, + "default_profile": false, + "profile_background_image_url": "http://a1.twimg.com/profile_background_images/87659392/6040_143426809917_610109917_3324938_5333854_n.jpg", + "created_at": "Sun Mar 08 08:30:14 +0000 2009", + "followers_count": 121, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/87659392/6040_143426809917_610109917_3324938_5333854_n.jpg", + "id": 23290095, + "profile_text_color": "ed9aed", + "profile_image_url_https": "https://si0.twimg.com/profile_images/391846681/twitter_this_normal.jpg", + "profile_image_url": "http://a1.twimg.com/profile_images/391846681/twitter_this_normal.jpg" + }, + { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "209990451", + "notifications": false, + "profile_background_tile": false, + "screen_name": "penfoldholes", + "name": "Richard Barker", + "listed_count": 0, + "location": "Shefford", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": true, + "utc_offset": null, + "profile_link_color": "0084B4", + "description": "kidney cancer survivor - former footballer and cricketer - wine , dog and music lover.", + "profile_sidebar_border_color": "C0DEED", + "url": null, + "time_zone": null, + "default_profile_image": false, + "statuses_count": 1608, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 7, + "friends_count": 66, + "profile_background_color": "C0DEED", + "is_translator": false, + "default_profile": true, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Sat Oct 30 09:40:15 +0000 2010", + "followers_count": 52, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 209990451, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1157214189/Richard_normal.jpg", + "profile_image_url": "http://a0.twimg.com/profile_images/1157214189/Richard_normal.jpg" + }, + { + "profile_sidebar_fill_color": "F6F6F6", + "protected": false, + "id_str": "90737592", + "notifications": false, + "profile_background_tile": false, + "screen_name": "coti_rodriguez", + "name": "Constanza Rodriguez", + "listed_count": 0, + "location": "Monte Hermoso", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": true, + "utc_offset": -10800, + "profile_link_color": "038543", + "description": "", + "profile_sidebar_border_color": "EEEEEE", + "url": null, + "time_zone": "Buenos Aires", + "default_profile_image": false, + "default_profile": false, + "statuses_count": 276, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 2, + "friends_count": 153, + "profile_background_color": "ACDED6", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme18/bg.gif", + "created_at": "Tue Nov 17 22:36:14 +0000 2009", + "followers_count": 48, + "follow_request_sent": false, + "lang": "es", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme18/bg.gif", + "id": 90737592, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1098075067/DSCN1664_normal.JPG", + "profile_image_url": "http://a0.twimg.com/profile_images/1098075067/DSCN1664_normal.JPG" + }, + { + "profile_sidebar_fill_color": "DDEEF6", + "protected": false, + "id_str": "111380497", + "notifications": false, + "profile_background_tile": false, + "screen_name": "PROFAMS", + "name": "ANTONIO MARCOS", + "listed_count": 0, + "location": "São Paulo", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": true, + "utc_offset": -10800, + "profile_link_color": "0084B4", + "description": "Gerente Nacional de Marketing e Professor nas áreas de adminisração e marketing.", + "profile_sidebar_border_color": "C0DEED", + "url": null, + "time_zone": "Brasilia", + "default_profile_image": false, + "default_profile": true, + "statuses_count": 124, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 5, + "friends_count": 44, + "profile_background_color": "C0DEED", + "is_translator": false, + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "created_at": "Thu Feb 04 18:34:41 +0000 2010", + "followers_count": 45, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "id": 111380497, + "profile_text_color": "333333", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1309327114/AMS_3_normal.JPG", + "profile_image_url": "http://a2.twimg.com/profile_images/1309327114/AMS_3_normal.JPG" + }, + { + "profile_sidebar_fill_color": "333333", + "protected": false, + "id_str": "19216042", + "notifications": false, + "profile_background_tile": true, + "screen_name": "anaverena", + "name": "Ana Verena Menezes", + "listed_count": 14, + "location": "Salvador,Bahia,Brasil", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": true, + "utc_offset": -10800, + "profile_link_color": "b36868", + "description": "all my colours turn to clouds.", + "profile_sidebar_border_color": "b6b8b6", + "url": "http://www.anaverena.tumblr.com", + "time_zone": "Brasilia", + "default_profile_image": false, + "default_profile": false, + "statuses_count": 8298, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 35, + "friends_count": 149, + "profile_background_color": "ffffff", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/profile_background_images/290833153/ikea_lena_2.jpg", + "created_at": "Tue Jan 20 02:45:47 +0000 2009", + "followers_count": 271, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/290833153/ikea_lena_2.jpg", + "id": 19216042, + "profile_text_color": "b8b2b2", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1445007999/IMG_7694_copie2_normal.jpg", + "profile_image_url": "http://a2.twimg.com/profile_images/1445007999/IMG_7694_copie2_normal.jpg" + }, + { + "profile_sidebar_fill_color": "fa89c9", + "protected": false, + "id_str": "192627716", + "notifications": false, + "profile_background_tile": true, + "screen_name": "soouzalaura", + "name": "Laura Souza", + "listed_count": 2, + "location": "Brasil", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": true, + "utc_offset": -10800, + "profile_link_color": "942394", + "description": "Welcome to real world. It's suck, you're gonna love it! (Friends)", + "profile_sidebar_border_color": "ed26bf", + "url": "http://laurapriscila.tumblr.com/", + "time_zone": "Brasilia", + "default_profile_image": false, + "statuses_count": 2218, + "profile_use_background_image": true, + "default_profile": false, + "verified": false, + "favourites_count": 2, + "friends_count": 119, + "profile_background_color": "ffffff", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/profile_background_images/237284023/bg.gif", + "created_at": "Sun Sep 19 18:12:21 +0000 2010", + "followers_count": 61, + "follow_request_sent": false, + "lang": "pt", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/237284023/bg.gif", + "id": 192627716, + "profile_text_color": "524646", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1422000163/SAM_0539_normal.JPG", + "profile_image_url": "http://a0.twimg.com/profile_images/1422000163/SAM_0539_normal.JPG" + }, + { + "profile_sidebar_fill_color": "efefef", + "protected": false, + "id_str": "58000577", + "notifications": false, + "profile_background_tile": true, + "screen_name": "colchica", + "name": "C Orne", + "listed_count": 0, + "location": "Lyon - France", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": true, + "utc_offset": 3600, + "profile_link_color": "2d66ad", + "description": "addicted to caffeine and a little bit greener... ", + "default_profile": false, + "profile_sidebar_border_color": "eeeeee", + "url": null, + "time_zone": "Paris", + "default_profile_image": false, + "statuses_count": 232, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 1, + "friends_count": 45, + "profile_background_color": "ebeef0", + "is_translator": false, + "profile_background_image_url": "http://a3.twimg.com/profile_background_images/154201528/20100912103242_2010-09-11_eboulis_05.jpg", + "created_at": "Sat Jul 18 18:44:29 +0000 2009", + "followers_count": 14, + "follow_request_sent": false, + "lang": "fr", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/154201528/20100912103242_2010-09-11_eboulis_05.jpg", + "id": 58000577, + "profile_text_color": "994399", + "profile_image_url_https": "https://si0.twimg.com/profile_images/336483751/samantha_normal.jpg", + "profile_image_url": "http://a0.twimg.com/profile_images/336483751/samantha_normal.jpg" + }, + { + "profile_sidebar_fill_color": "c9c9c9", + "protected": false, + "id_str": "67218346", + "notifications": null, + "profile_background_tile": true, + "screen_name": "GrandyManFTW14", + "name": "Valerie", + "listed_count": 39, + "location": "Yankee Stadium", + "show_all_inline_media": true, + "contributors_enabled": false, + "following": null, + "geo_enabled": false, + "utc_offset": -18000, + "profile_link_color": "2138B4", + "description": "im a nerd thats infatuated with sports... mostly #yankees baseball. I should reside in a bubble. I'm 28 and speak my mind.", + "profile_sidebar_border_color": "bfbfbf", + "url": "http://www.facebook.com/yankeesangel21", + "default_profile": false, + "time_zone": "Eastern Time (US & Canada)", + "default_profile_image": false, + "statuses_count": 8744, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 11, + "friends_count": 653, + "profile_background_color": "07090b", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/profile_background_images/202039528/x1f1841c56244072a549acc188477e1e.jpg", + "created_at": "Thu Aug 20 03:43:17 +0000 2009", + "followers_count": 500, + "follow_request_sent": null, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/202039528/x1f1841c56244072a549acc188477e1e.jpg", + "id": 67218346, + "profile_text_color": "183050", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1520181363/image_normal.jpg", + "profile_image_url": "http://a2.twimg.com/profile_images/1520181363/image_normal.jpg" + }, + { + "profile_sidebar_fill_color": "95E8EC", + "protected": false, + "id_str": "185014625", + "default_profile": false, + "notifications": false, + "profile_background_tile": false, + "screen_name": "chiquip35", + "name": "Paula Klimowitz", + "listed_count": 1, + "location": "Miami", + "show_all_inline_media": false, + "contributors_enabled": false, + "following": false, + "geo_enabled": false, + "utc_offset": null, + "profile_link_color": "0099B9", + "description": "I love the positive energy of the people of the world!!", + "profile_sidebar_border_color": "5ED4DC", + "url": null, + "time_zone": null, + "default_profile_image": false, + "statuses_count": 48, + "profile_use_background_image": true, + "verified": false, + "favourites_count": 0, + "friends_count": 102, + "profile_background_color": "0099B9", + "is_translator": false, + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme4/bg.gif", + "created_at": "Tue Aug 31 01:15:36 +0000 2010", + "followers_count": 9, + "follow_request_sent": false, + "lang": "en", + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme4/bg.gif", + "id": 185014625, + "profile_text_color": "3C3940", + "profile_image_url_https": "https://si0.twimg.com/profile_images/1521529755/174758_normal.jpg", + "profile_image_url": "http://a3.twimg.com/profile_images/1521529755/174758_normal.jpg" + } + ], "resultType": "ReTweet" - }] - } + } + ] } - -} - +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/twitter.reply b/Collections/Timeline/fixtures/twitter.reply index 46b74fc98..f8044b273 100644 --- a/Collections/Timeline/fixtures/twitter.reply +++ b/Collections/Timeline/fixtures/twitter.reply @@ -1,6 +1,142 @@ { - "type": "timeline/twitter", - "via": "synclet/twitter", - "timestamp": 1315974920913, "action": "new", - "obj": {"timeStamp":"2011-11-04T16:32:06.068Z","data":{"place":{"country_code":"US","place_type":"city","name":"Cascade","bounding_box":{"type":"Polygon","coordinates":[[[-91.02358,42.289324],[-90.991574,42.289324],[-90.991574,42.308435],[-91.02358,42.308435]]]},"attributes":{},"full_name":"Cascade, IA","country":"United States","url":"http://api.twitter.com/1/geo/id/85905f44ae0575e5.json","id":"85905f44ae0575e5"},"retweet_count":0,"in_reply_to_screen_name":"FrancescoC","created_at":"Thu Nov 03 17:29:11 +0000 2011","retweeted":false,"in_reply_to_status_id_str":"132133925953875969","in_reply_to_user_id_str":"15592821","user":{"time_zone":"Central Time (US & Canada)","profile_link_color":"0000ff","protected":false,"default_profile":false,"created_at":"Wed Mar 21 02:44:51 +0000 2007","profile_image_url_https":"https://si0.twimg.com/profile_images/565991047/jer_normal.jpg","name":"Jeremie Miller","geo_enabled":true,"favourites_count":9,"profile_background_color":"ffffff","default_profile_image":false,"notifications":null,"profile_background_tile":false,"utc_offset":-21600,"description":"Currently building the Locker Project, TeleHash, and Singly with a focus on personal data + distributed protocols. Helped found Jabber/XMPP, open platforms FTW!","statuses_count":891,"following":null,"verified":false,"profile_sidebar_fill_color":"ffffff","followers_count":1375,"profile_image_url":"http://a0.twimg.com/profile_images/565991047/jer_normal.jpg","follow_request_sent":null,"profile_sidebar_border_color":"000000","location":"Cascade, IA","is_translator":false,"contributors_enabled":false,"screen_name":"jeremie","profile_use_background_image":false,"url":"http://jeremie.com/-","id_str":"1704421","show_all_inline_media":true,"profile_background_image_url_https":"https://si0.twimg.com/images/themes/theme1/bg.png","profile_text_color":"000000","id":1704421,"listed_count":141,"lang":"en","profile_background_image_url":"http://a0.twimg.com/images/themes/theme1/bg.png","friends_count":161},"contributors":null,"in_reply_to_status_id":132133925953875970,"source":"Twitter for iPhone","id_str":"132147322749599744","geo":{"type":"Point","coordinates":[42.29882394,-91.01307985]},"favorited":false,"id":132147322749599740,"entities":{"urls":[],"user_mentions":[{"name":"Francesco Cesarini","indices":[6,17],"screen_name":"FrancescoC","id_str":"15592821","id":15592821},{"name":"DizzyD","indices":[34,41],"screen_name":"dizzyd","id_str":"7206052","id":7206052}],"hashtags":[{"indices":[123,131],"text":"euc2011"}]},"coordinates":{"type":"Point","coordinates":[-91.01307985,42.29882394]},"in_reply_to_user_id":15592821,"truncated":false,"text":"Nice! @FrancescoC: Congratulating @dizzyd for winning the Erlang User of the year award for giving rebar to the community! #euc2011"}}} + "idr": "tweet://twitter/timeline?id=twitter#132147322749599744", + "data": { + "place": { + "country_code": "US", + "place_type": "city", + "name": "Cascade", + "bounding_box": { + "type": "Polygon", + "coordinates": [ + [ + [ + -91.02358, + 42.289324 + ], + [ + -90.991574, + 42.289324 + ], + [ + -90.991574, + 42.308435 + ], + [ + -91.02358, + 42.308435 + ] + ] + ] + }, + "attributes": {}, + "full_name": "Cascade, IA", + "country": "United States", + "url": "http://api.twitter.com/1/geo/id/85905f44ae0575e5.json", + "id": "85905f44ae0575e5" + }, + "retweet_count": 0, + "in_reply_to_screen_name": "FrancescoC", + "created_at": "Thu Nov 03 17:29:11 +0000 2011", + "retweeted": false, + "in_reply_to_status_id_str": "132133925953875969", + "in_reply_to_user_id_str": "15592821", + "user": { + "time_zone": "Central Time (US & Canada)", + "profile_link_color": "0000ff", + "protected": false, + "default_profile": false, + "created_at": "Wed Mar 21 02:44:51 +0000 2007", + "profile_image_url_https": "https://si0.twimg.com/profile_images/565991047/jer_normal.jpg", + "name": "Jeremie Miller", + "geo_enabled": true, + "favourites_count": 9, + "profile_background_color": "ffffff", + "default_profile_image": false, + "notifications": null, + "profile_background_tile": false, + "utc_offset": -21600, + "description": "Currently building the Locker Project, TeleHash, and Singly with a focus on personal data + distributed protocols. Helped found Jabber/XMPP, open platforms FTW!", + "statuses_count": 891, + "following": null, + "verified": false, + "profile_sidebar_fill_color": "ffffff", + "followers_count": 1375, + "profile_image_url": "http://a0.twimg.com/profile_images/565991047/jer_normal.jpg", + "follow_request_sent": null, + "profile_sidebar_border_color": "000000", + "location": "Cascade, IA", + "is_translator": false, + "contributors_enabled": false, + "screen_name": "jeremie", + "profile_use_background_image": false, + "url": "http://jeremie.com/-", + "id_str": "1704421", + "show_all_inline_media": true, + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "profile_text_color": "000000", + "id": 1704421, + "listed_count": 141, + "lang": "en", + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "friends_count": 161 + }, + "contributors": null, + "in_reply_to_status_id": 132133925953875970, + "source": "Twitter for iPhone", + "id_str": "132147322749599744", + "geo": { + "type": "Point", + "coordinates": [ + 42.29882394, + -91.01307985 + ] + }, + "favorited": false, + "id": 132147322749599740, + "entities": { + "urls": [], + "user_mentions": [ + { + "name": "Francesco Cesarini", + "indices": [ + 6, + 17 + ], + "screen_name": "FrancescoC", + "id_str": "15592821", + "id": 15592821 + }, + { + "name": "DizzyD", + "indices": [ + 34, + 41 + ], + "screen_name": "dizzyd", + "id_str": "7206052", + "id": 7206052 + } + ], + "hashtags": [ + { + "indices": [ + 123, + 131 + ], + "text": "euc2011" + } + ] + }, + "coordinates": { + "type": "Point", + "coordinates": [ + -91.01307985, + 42.29882394 + ] + }, + "in_reply_to_user_id": 15592821, + "truncated": false, + "text": "Nice! @FrancescoC: Congratulating @dizzyd for winning the Erlang User of the year award for giving rebar to the community! #euc2011" + } +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/twitter.reply.orig b/Collections/Timeline/fixtures/twitter.reply.orig index 1ca49bfcc..3e2c1f3d2 100644 --- a/Collections/Timeline/fixtures/twitter.reply.orig +++ b/Collections/Timeline/fixtures/twitter.reply.orig @@ -1,7 +1,174 @@ { - "type": "timeline/twitter", - "via": "synclet/twitter", - "timestamp": 1315974920913, "action": "new", - "obj":{"timeStamp":"2011-11-04T16:32:06.068Z","data":{"place":null,"retweet_count":41,"in_reply_to_screen_name":null,"created_at":"Thu Nov 03 16:54:29 +0000 2011","retweeted":false,"in_reply_to_status_id_str":null,"in_reply_to_user_id_str":null,"user":{"time_zone":null,"profile_link_color":"2FC2EF","protected":false,"created_at":"Fri Mar 13 13:15:23 +0000 2009","profile_image_url_https":"https://si0.twimg.com/profile_images/95456179/cortex_normal.png","name":"Jon Meredith","geo_enabled":false,"favourites_count":1,"profile_background_color":"1A1B1F","default_profile_image":false,"notifications":false,"profile_background_tile":false,"utc_offset":null,"description":"","statuses_count":85,"following":true,"verified":false,"profile_sidebar_fill_color":"252429","followers_count":127,"profile_image_url":"http://a0.twimg.com/profile_images/95456179/cortex_normal.png","follow_request_sent":false,"profile_sidebar_border_color":"181A1E","location":"Denver, CO, USA","is_translator":false,"contributors_enabled":false,"screen_name":"jon_meredith","default_profile":false,"profile_use_background_image":true,"url":null,"id_str":"24168124","show_all_inline_media":false,"profile_background_image_url_https":"https://si0.twimg.com/images/themes/theme9/bg.gif","profile_text_color":"666666","id":24168124,"listed_count":12,"lang":"en","profile_background_image_url":"http://a1.twimg.com/images/themes/theme9/bg.gif","friends_count":69},"contributors":null,"retweeted_status":{"place":null,"retweet_count":41,"in_reply_to_screen_name":null,"created_at":"Thu Nov 03 16:35:57 +0000 2011","retweeted":false,"in_reply_to_status_id_str":null,"in_reply_to_user_id_str":null,"user":{"time_zone":"London","profile_link_color":"D02B55","protected":false,"created_at":"Fri Jul 25 02:14:58 +0000 2008","profile_image_url_https":"https://si0.twimg.com/profile_images/337579346/francescoJulianCash_normal.jpg","name":"Francesco Cesarini","geo_enabled":false,"favourites_count":16,"profile_background_color":"352726","default_profile_image":false,"notifications":false,"profile_background_tile":false,"utc_offset":0,"description":"Founded Erlang Solutions, co-authored Erlang Programming (O'Reilly), part time uncle & godfather and full time pointy haired boss. ","statuses_count":3127,"following":false,"verified":false,"profile_sidebar_fill_color":"99CC33","followers_count":1480,"profile_image_url":"http://a1.twimg.com/profile_images/337579346/francescoJulianCash_normal.jpg","follow_request_sent":false,"profile_sidebar_border_color":"829D5E","location":"London, United Kingdom","is_translator":false,"contributors_enabled":false,"screen_name":"FrancescoC","default_profile":false,"profile_use_background_image":true,"url":"http://www.erlang-solutions.com","id_str":"15592821","show_all_inline_media":false,"profile_background_image_url_https":"https://si0.twimg.com/images/themes/theme5/bg.gif","profile_text_color":"3E4415","id":15592821,"listed_count":122,"lang":"en","profile_background_image_url":"http://a1.twimg.com/images/themes/theme5/bg.gif","friends_count":365},"contributors":null,"in_reply_to_status_id":null,"source":"Twitter for Android","id_str":"132133925953875969","geo":null,"favorited":false,"id":132133925953875970,"entities":{"urls":[],"user_mentions":[{"name":"DizzyD","indices":[15,22],"screen_name":"dizzyd","id_str":"7206052","id":7206052}],"hashtags":[{"indices":[121,129],"text":"euc2011"}]},"coordinates":null,"in_reply_to_user_id":null,"truncated":false,"text":"Congratulating @dizzyd for winning the Erlang User of the year award for giving rebar to the community! Well done Dizzy. #euc2011"},"in_reply_to_status_id":null,"source":"web","id_str":"132138590636478464","geo":null,"favorited":false,"id":132138590636478460,"entities":{"urls":[],"user_mentions":[{"name":"Francesco Cesarini","indices":[3,14],"screen_name":"FrancescoC","id_str":"15592821","id":15592821},{"name":"DizzyD","indices":[31,38],"screen_name":"dizzyd","id_str":"7206052","id":7206052}],"hashtags":[]},"coordinates":null,"in_reply_to_user_id":null,"truncated":true,"text":"RT @FrancescoC: Congratulating @dizzyd for winning the Erlang User of the year award for giving rebar to the community! Well done Dizzy. ..."}}} - + "idr": "tweet://twitter/timeline?id=twitter#132138590636478464", + "data": { + "place": null, + "retweet_count": 41, + "in_reply_to_screen_name": null, + "created_at": "Thu Nov 03 16:54:29 +0000 2011", + "retweeted": false, + "in_reply_to_status_id_str": null, + "in_reply_to_user_id_str": null, + "user": { + "time_zone": null, + "profile_link_color": "2FC2EF", + "protected": false, + "created_at": "Fri Mar 13 13:15:23 +0000 2009", + "profile_image_url_https": "https://si0.twimg.com/profile_images/95456179/cortex_normal.png", + "name": "Jon Meredith", + "geo_enabled": false, + "favourites_count": 1, + "profile_background_color": "1A1B1F", + "default_profile_image": false, + "notifications": false, + "profile_background_tile": false, + "utc_offset": null, + "description": "", + "statuses_count": 85, + "following": true, + "verified": false, + "profile_sidebar_fill_color": "252429", + "followers_count": 127, + "profile_image_url": "http://a0.twimg.com/profile_images/95456179/cortex_normal.png", + "follow_request_sent": false, + "profile_sidebar_border_color": "181A1E", + "location": "Denver, CO, USA", + "is_translator": false, + "contributors_enabled": false, + "screen_name": "jon_meredith", + "default_profile": false, + "profile_use_background_image": true, + "url": null, + "id_str": "24168124", + "show_all_inline_media": false, + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme9/bg.gif", + "profile_text_color": "666666", + "id": 24168124, + "listed_count": 12, + "lang": "en", + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme9/bg.gif", + "friends_count": 69 + }, + "contributors": null, + "retweeted_status": { + "place": null, + "retweet_count": 41, + "in_reply_to_screen_name": null, + "created_at": "Thu Nov 03 16:35:57 +0000 2011", + "retweeted": false, + "in_reply_to_status_id_str": null, + "in_reply_to_user_id_str": null, + "user": { + "time_zone": "London", + "profile_link_color": "D02B55", + "protected": false, + "created_at": "Fri Jul 25 02:14:58 +0000 2008", + "profile_image_url_https": "https://si0.twimg.com/profile_images/337579346/francescoJulianCash_normal.jpg", + "name": "Francesco Cesarini", + "geo_enabled": false, + "favourites_count": 16, + "profile_background_color": "352726", + "default_profile_image": false, + "notifications": false, + "profile_background_tile": false, + "utc_offset": 0, + "description": "Founded Erlang Solutions, co-authored Erlang Programming (O'Reilly), part time uncle & godfather and full time pointy haired boss. ", + "statuses_count": 3127, + "following": false, + "verified": false, + "profile_sidebar_fill_color": "99CC33", + "followers_count": 1480, + "profile_image_url": "http://a1.twimg.com/profile_images/337579346/francescoJulianCash_normal.jpg", + "follow_request_sent": false, + "profile_sidebar_border_color": "829D5E", + "location": "London, United Kingdom", + "is_translator": false, + "contributors_enabled": false, + "screen_name": "FrancescoC", + "default_profile": false, + "profile_use_background_image": true, + "url": "http://www.erlang-solutions.com", + "id_str": "15592821", + "show_all_inline_media": false, + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme5/bg.gif", + "profile_text_color": "3E4415", + "id": 15592821, + "listed_count": 122, + "lang": "en", + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme5/bg.gif", + "friends_count": 365 + }, + "contributors": null, + "in_reply_to_status_id": null, + "source": "Twitter for Android", + "id_str": "132133925953875969", + "geo": null, + "favorited": false, + "id": 132133925953875970, + "entities": { + "urls": [], + "user_mentions": [ + { + "name": "DizzyD", + "indices": [ + 15, + 22 + ], + "screen_name": "dizzyd", + "id_str": "7206052", + "id": 7206052 + } + ], + "hashtags": [ + { + "indices": [ + 121, + 129 + ], + "text": "euc2011" + } + ] + }, + "coordinates": null, + "in_reply_to_user_id": null, + "truncated": false, + "text": "Congratulating @dizzyd for winning the Erlang User of the year award for giving rebar to the community! Well done Dizzy. #euc2011" + }, + "in_reply_to_status_id": null, + "source": "web", + "id_str": "132138590636478464", + "geo": null, + "favorited": false, + "id": 132138590636478460, + "entities": { + "urls": [], + "user_mentions": [ + { + "name": "Francesco Cesarini", + "indices": [ + 3, + 14 + ], + "screen_name": "FrancescoC", + "id_str": "15592821", + "id": 15592821 + }, + { + "name": "DizzyD", + "indices": [ + 31, + 38 + ], + "screen_name": "dizzyd", + "id_str": "7206052", + "id": 7206052 + } + ], + "hashtags": [] + }, + "coordinates": null, + "in_reply_to_user_id": null, + "truncated": true, + "text": "RT @FrancescoC: Congratulating @dizzyd for winning the Erlang User of the year award for giving rebar to the community! Well done Dizzy. ..." + } +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/twitter.rt b/Collections/Timeline/fixtures/twitter.rt index 5cf9a863f..8ea409f8d 100644 --- a/Collections/Timeline/fixtures/twitter.rt +++ b/Collections/Timeline/fixtures/twitter.rt @@ -1,146 +1,145 @@ { - "type": "timeline/twitter", - "via": "synclet/twitter", - "timestamp": 1315974920913, "action": "new", - "obj": { - "timeStamp": "2011-11-04T17:25:55.729Z", - "data": { + "idr": "tweet://twitter/timeline?id=twitter#132508288435752960", + "data": { + "place": null, + "retweet_count": 6, + "in_reply_to_screen_name": null, + "created_at": "Fri Nov 04 17:23:32 +0000 2011", + "retweeted": false, + "in_reply_to_status_id_str": null, + "in_reply_to_user_id_str": null, + "user": { + "time_zone": "Pacific Time (US & Canada)", + "profile_link_color": "0000ff", + "protected": false, + "created_at": "Tue Mar 27 01:14:05 +0000 2007", + "profile_image_url_https": "https://si0.twimg.com/profile_images/941827802/IMG_3811_v4_normal.jpg", + "name": "Tim O'Reilly", + "geo_enabled": true, + "favourites_count": 52, + "profile_background_color": "9ae4e8", + "default_profile_image": false, + "notifications": false, + "profile_background_tile": false, + "utc_offset": -28800, + "description": "Founder and CEO, O'Reilly Media. Watching the alpha geeks, sharing their stories, helping the future unfold.", + "statuses_count": 16827, + "following": true, + "verified": true, + "profile_sidebar_fill_color": "e0ff92", + "followers_count": 1508525, + "profile_image_url": "http://a1.twimg.com/profile_images/941827802/IMG_3811_v4_normal.jpg", + "follow_request_sent": false, + "profile_sidebar_border_color": "87bc44", + "location": "Sebastopol, CA", + "is_translator": false, + "default_profile": false, + "contributors_enabled": false, + "screen_name": "timoreilly", + "profile_use_background_image": true, + "url": "http://radar.oreilly.com", + "id_str": "2384071", + "show_all_inline_media": false, + "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/3587880/notes.gif", + "profile_text_color": "000000", + "id": 2384071, + "listed_count": 20459, + "lang": "en", + "profile_background_image_url": "http://a1.twimg.com/profile_background_images/3587880/notes.gif", + "friends_count": 784 + }, + "contributors": null, + "retweeted_status": { "place": null, "retweet_count": 6, "in_reply_to_screen_name": null, - "created_at": "Fri Nov 04 17:23:32 +0000 2011", + "created_at": "Fri Nov 04 15:36:13 +0000 2011", "retweeted": false, "in_reply_to_status_id_str": null, "in_reply_to_user_id_str": null, "user": { "time_zone": "Pacific Time (US & Canada)", - "profile_link_color": "0000ff", + "profile_link_color": "0084B4", "protected": false, - "created_at": "Tue Mar 27 01:14:05 +0000 2007", - "profile_image_url_https": "https://si0.twimg.com/profile_images/941827802/IMG_3811_v4_normal.jpg", - "name": "Tim O'Reilly", - "geo_enabled": true, - "favourites_count": 52, - "profile_background_color": "9ae4e8", + "created_at": "Tue Jan 27 02:59:48 +0000 2009", + "profile_image_url_https": "https://si0.twimg.com/profile_images/80487785/Michael_Chui_face_square_normal.jpg", + "name": "Michael Chui", + "geo_enabled": false, + "favourites_count": 0, + "profile_background_color": "C0DEED", "default_profile_image": false, "notifications": false, "profile_background_tile": false, "utc_offset": -28800, - "description": "Founder and CEO, O'Reilly Media. Watching the alpha geeks, sharing their stories, helping the future unfold.", - "statuses_count": 16827, - "following": true, - "verified": true, - "profile_sidebar_fill_color": "e0ff92", - "followers_count": 1508525, - "profile_image_url": "http://a1.twimg.com/profile_images/941827802/IMG_3811_v4_normal.jpg", + "description": "Senior Fellow at McKinsey Global Institute (@McKinsey_MGI) leading research on the impact of technology trends. Opinions expressed are my own.", + "statuses_count": 1787, + "following": false, + "verified": false, + "profile_sidebar_fill_color": "DDEEF6", + "followers_count": 1326, + "profile_image_url": "http://a0.twimg.com/profile_images/80487785/Michael_Chui_face_square_normal.jpg", "follow_request_sent": false, - "profile_sidebar_border_color": "87bc44", - "location": "Sebastopol, CA", + "profile_sidebar_border_color": "C0DEED", + "location": "", "is_translator": false, - "default_profile": false, + "default_profile": true, "contributors_enabled": false, - "screen_name": "timoreilly", + "screen_name": "mchui", "profile_use_background_image": true, - "url": "http://radar.oreilly.com", - "id_str": "2384071", + "url": null, + "id_str": "19573968", "show_all_inline_media": false, - "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/3587880/notes.gif", - "profile_text_color": "000000", - "id": 2384071, - "listed_count": 20459, + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "profile_text_color": "333333", + "id": 19573968, + "listed_count": 85, "lang": "en", - "profile_background_image_url": "http://a1.twimg.com/profile_background_images/3587880/notes.gif", - "friends_count": 784 + "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", + "friends_count": 157 }, "contributors": null, - "retweeted_status": { - "place": null, - "retweet_count": 6, - "in_reply_to_screen_name": null, - "created_at": "Fri Nov 04 15:36:13 +0000 2011", - "retweeted": false, - "in_reply_to_status_id_str": null, - "in_reply_to_user_id_str": null, - "user": { - "time_zone": "Pacific Time (US & Canada)", - "profile_link_color": "0084B4", - "protected": false, - "created_at": "Tue Jan 27 02:59:48 +0000 2009", - "profile_image_url_https": "https://si0.twimg.com/profile_images/80487785/Michael_Chui_face_square_normal.jpg", - "name": "Michael Chui", - "geo_enabled": false, - "favourites_count": 0, - "profile_background_color": "C0DEED", - "default_profile_image": false, - "notifications": false, - "profile_background_tile": false, - "utc_offset": -28800, - "description": "Senior Fellow at McKinsey Global Institute (@McKinsey_MGI) leading research on the impact of technology trends. Opinions expressed are my own.", - "statuses_count": 1787, - "following": false, - "verified": false, - "profile_sidebar_fill_color": "DDEEF6", - "followers_count": 1326, - "profile_image_url": "http://a0.twimg.com/profile_images/80487785/Michael_Chui_face_square_normal.jpg", - "follow_request_sent": false, - "profile_sidebar_border_color": "C0DEED", - "location": "", - "is_translator": false, - "default_profile": true, - "contributors_enabled": false, - "screen_name": "mchui", - "profile_use_background_image": true, - "url": null, - "id_str": "19573968", - "show_all_inline_media": false, - "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", - "profile_text_color": "333333", - "id": 19573968, - "listed_count": 85, - "lang": "en", - "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", - "friends_count": 157 - }, - "contributors": null, - "in_reply_to_status_id": null, - "source": "Twitter for BlackBerry®", - "id_str": "132481280091754496", - "geo": null, - "favorited": false, - "id": 132481280091754500, - "entities": { - "urls": [], - "user_mentions": [], - "hashtags": [] - }, - "coordinates": null, - "in_reply_to_user_id": null, - "truncated": false, - "text": "Quoth Niall Ferguson: We no longer have the rule of law; we have the rule of lawyers and that's different." - }, "in_reply_to_status_id": null, - "source": "Twitter for Android", - "id_str": "132508288435752960", + "source": "Twitter for BlackBerry®", + "id_str": "132481280091754496", "geo": null, "favorited": false, - "id": 132508288435752960, + "id": 132481280091754500, "entities": { "urls": [], - "user_mentions": [{ - "name": "Michael Chui", - "indices": [3, 9], - "screen_name": "mchui", - "id_str": "19573968", - "id": 19573968 - }], + "user_mentions": [], "hashtags": [] }, "coordinates": null, "in_reply_to_user_id": null, "truncated": false, - "text": "RT @mchui: Quoth Niall Ferguson: We no longer have the rule of law; we have the rule of lawyers and that's different." - } + "text": "Quoth Niall Ferguson: We no longer have the rule of law; we have the rule of lawyers and that's different." + }, + "in_reply_to_status_id": null, + "source": "Twitter for Android", + "id_str": "132508288435752960", + "geo": null, + "favorited": false, + "id": 132508288435752960, + "entities": { + "urls": [], + "user_mentions": [ + { + "name": "Michael Chui", + "indices": [ + 3, + 9 + ], + "screen_name": "mchui", + "id_str": "19573968", + "id": 19573968 + } + ], + "hashtags": [] + }, + "coordinates": null, + "in_reply_to_user_id": null, + "truncated": false, + "text": "RT @mchui: Quoth Niall Ferguson: We no longer have the rule of law; we have the rule of lawyers and that's different." } -} - +} \ No newline at end of file diff --git a/Collections/Timeline/fixtures/twitter.timeline b/Collections/Timeline/fixtures/twitter.timeline index 89bd37d1c..f840662df 100644 --- a/Collections/Timeline/fixtures/twitter.timeline +++ b/Collections/Timeline/fixtures/twitter.timeline @@ -1,88 +1,92 @@ { - "type": "timeline/twitter", - "via": "synclet/twitter", - "timestamp": 1315974920913, "action": "new", - "obj": { - "source": "twitter_timeline", - "type": "new", - "data": { - "_id": "4e702f08bcb6f2f6dd06c849", - "in_reply_to_user_id_str": null, - "retweet_count": 0, - "in_reply_to_status_id": null, - "id_str": "113829905090879488", - "contributors": null, - "truncated": false, - "geo": null, - "coordinates": null, - "favorited": false, - "user": { - "default_profile": false, - "contributors_enabled": false, - "profile_use_background_image": true, - "protected": false, - "id_str": "633", - "time_zone": "Eastern Time (US & Canada)", - "profile_background_color": "1A1B1F", - "name": "danah boyd", - "profile_background_image_url": "http://a1.twimg.com/images/themes/theme9/bg.gif", - "profile_image_url_https": "https://si0.twimg.com/profile_images/890666764/MeAtTalkWithHat-Sq_normal.jpg", - "default_profile_image": false, - "following": true, - "utc_offset": -18000, - "profile_image_url": "http://a0.twimg.com/profile_images/890666764/MeAtTalkWithHat-Sq_normal.jpg", - "description": "social media scholar, youth researcher & advocate | Microsoft Research, Harvard Berkman Center | zephoria@zephoria.org", - "show_all_inline_media": false, - "geo_enabled": false, - "friends_count": 734, - "profile_text_color": "666666", - "location": "Boston, MA", - "is_translator": false, - "profile_sidebar_fill_color": "252429", - "follow_request_sent": false, - "profile_background_tile": false, - "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme9/bg.gif", - "url": "http://www.zephoria.org/thoughts/", - "statuses_count": 2471, - "followers_count": 50313, - "screen_name": "zephoria", - "notifications": false, - "profile_link_color": "2FC2EF", - "lang": "en", - "verified": false, - "favourites_count": 44, - "profile_sidebar_border_color": "181A1E", - "id": 633, - "listed_count": 5224, - "created_at": "Thu Jul 13 21:27:23 +0000 2006" - }, - "in_reply_to_user_id": null, - "in_reply_to_screen_name": null, - "possibly_sensitive": false, - "retweeted": false, - "source": "web", - "in_reply_to_status_id_str": null, - "entities": { - "user_mentions": [{ + "idr": "tweet://twitter/timeline?id=twitter#113829905090879488", + "data": { + "_id": "4e702f08bcb6f2f6dd06c849", + "in_reply_to_user_id_str": null, + "retweet_count": 0, + "in_reply_to_status_id": null, + "id_str": "113829905090879488", + "contributors": null, + "truncated": false, + "geo": null, + "coordinates": null, + "favorited": false, + "user": { + "default_profile": false, + "contributors_enabled": false, + "profile_use_background_image": true, + "protected": false, + "id_str": "633", + "time_zone": "Eastern Time (US & Canada)", + "profile_background_color": "1A1B1F", + "name": "danah boyd", + "profile_background_image_url": "http://a1.twimg.com/images/themes/theme9/bg.gif", + "profile_image_url_https": "https://si0.twimg.com/profile_images/890666764/MeAtTalkWithHat-Sq_normal.jpg", + "default_profile_image": false, + "following": true, + "utc_offset": -18000, + "profile_image_url": "http://a0.twimg.com/profile_images/890666764/MeAtTalkWithHat-Sq_normal.jpg", + "description": "social media scholar, youth researcher & advocate | Microsoft Research, Harvard Berkman Center | zephoria@zephoria.org", + "show_all_inline_media": false, + "geo_enabled": false, + "friends_count": 734, + "profile_text_color": "666666", + "location": "Boston, MA", + "is_translator": false, + "profile_sidebar_fill_color": "252429", + "follow_request_sent": false, + "profile_background_tile": false, + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme9/bg.gif", + "url": "http://www.zephoria.org/thoughts/", + "statuses_count": 2471, + "followers_count": 50313, + "screen_name": "zephoria", + "notifications": false, + "profile_link_color": "2FC2EF", + "lang": "en", + "verified": false, + "favourites_count": 44, + "profile_sidebar_border_color": "181A1E", + "id": 633, + "listed_count": 5224, + "created_at": "Thu Jul 13 21:27:23 +0000 2006" + }, + "in_reply_to_user_id": null, + "in_reply_to_screen_name": null, + "possibly_sensitive": false, + "retweeted": false, + "source": "web", + "in_reply_to_status_id_str": null, + "entities": { + "user_mentions": [ + { "name": "Wil Wheaton", "id_str": "1183041", - "indices": [108, 113], + "indices": [ + 108, + 113 + ], "screen_name": "wilw", "id": 1183041 - }], - "hashtags": [], - "urls": [{ - "indices": [116, 135], + } + ], + "hashtags": [], + "urls": [ + { + "indices": [ + 116, + 135 + ], "display_url": "amzn.to/qqvlo4", "url": "http://t.co/zAN4G8S", "expanded_url": "http://amzn.to/qqvlo4" - }] - }, - "id": 113829905090879490, - "place": null, - "text": "Just finished \"Ready Player One.\" YA dystopia mixed w/ 1980s geek nostalgia. Soooo much fun. (Audio read by @wilw ) http://t.co/zAN4G8S", - "created_at": "Wed Sep 14 04:22:18 +0000 2011" - } + } + ] + }, + "id": 113829905090879490, + "place": null, + "text": "Just finished \"Ready Player One.\" YA dystopia mixed w/ 1980s geek nostalgia. Soooo much fun. (Audio read by @wilw ) http://t.co/zAN4G8S", + "created_at": "Wed Sep 14 04:22:18 +0000 2011" } -} +} \ No newline at end of file From fc931d7b7244db9dca7cf0a6e01ced2e4bcdd680 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Tue, 27 Dec 2011 18:18:25 -0600 Subject: [PATCH 26/40] fixed up and tested all the fixtures, very close --- Collections/Timeline/dataIn.js | 68 +++++++--- Collections/Timeline/dataStore.js | 9 +- Collections/Timeline/fixtures/ig.tweet | 2 +- Collections/Timeline/fixtures/link | 7 +- Collections/Timeline/test.js | 173 +++++++++++++++++++++++++ 5 files changed, 232 insertions(+), 27 deletions(-) create mode 100644 Collections/Timeline/test.js diff --git a/Collections/Timeline/dataIn.js b/Collections/Timeline/dataIn.js index 1f768f353..2065509d1 100644 --- a/Collections/Timeline/dataIn.js +++ b/Collections/Timeline/dataIn.js @@ -6,20 +6,13 @@ var url = require('url'); var crypto = require("crypto"); var path = require('path'); -var dataStore, locker, profiles = {}; +var dataStore, locker; // internally we need these for happy fun stuff exports.init = function(l, dStore, callback){ dataStore = dStore; locker = l; - // load our known profiles, require for foursquare checkins, and to tell "me" flag on everything - async.forEach(["foursquare", "instagram"], function(svc, cb){ - var lurl = locker.lockerBase + '/Me/' + svc + '/getCurrent/profile'; - request.get({uri:lurl, json:true}, function(err, resp, arr){ - if(arr && arr.length > 0) profiles[svc] = arr[0]; - return cb(); - }); - }, callback); + callback(); } // manually walk and reindex all possible link sources @@ -143,12 +136,39 @@ exports.processEvent = function(event, callback) var idr = url.parse(event.idr, true); if(!idr || !idr.protocol) return callback("don't understand this data"); // handle links as a special case as we're using them for post-process-deduplication - if(idr.protocol == 'link') return processLink(event, callback); + if(idr.protocol == 'link:') return processLink(event, callback); masterMaster(idr, event.data, callback); } -// figure out what to do with any data +// some data is incomplete, stupid but WTF do you do! +var profiles = {}; function masterMaster(idr, data, callback) +{ + if(idr.protocol == 'checkin:' && idr.pathname.indexOf('checkin') != -1) + { // foursquare checkins api for a person is the same json format but missing the .user, which we need to store the correct contact idr, so we have to fetch/cache that + var svcId = idr.query.id; + if(profiles[svcId]) + { + data.user = profiles[svcId]; + return masterBlaster(idr, data, callback); + } + var lurl = locker.lockerBase + '/Me/'+svcId+'/getCurrent/profile'; + request.get({uri:lurl, json:true}, function(err, resp, arr){ + if(err || !arr || arr.length == 0) + { + logger.error("couldn't fetch profile for "+svcId+" so have to skip "+url.format(idr)); + return callback(); + } + logger.debug("caching profile for "+svcId); + data.user = profiles[svcId] = arr[0]; + return masterBlaster(idr, data, callback); + }); + } + masterBlaster(idr, data, callback); +} + +// figure out what to do with any data +function masterBlaster(idr, data, callback) { if(typeof data != 'object') return callback("missing or bad data"); // logger.debug("MM\t"+url.format(idr)); @@ -230,6 +250,7 @@ function itemMerge(older, newer) // merge existing with delete! function itemMergeHard(a, b, cb) { + logger.debug("hard merge of "+a.ref+" to "+b.ref); var item = itemMerge(a, b); if(!item.responses) item.responses = []; // gather old responses @@ -246,17 +267,15 @@ function itemMergeHard(a, b, cb) // when a processed link event comes in, check to see if it's a long url that could help us de-dup function processLink(event, callback) { - if(!event || !event.obj || !event.obj.encounters) return callback("no encounter"); + if(!event || !event.data || !event.data.encounters) return callback("no encounter"); // process each encounter if there's multiple - async.forEach(event.obj.encounters, function(encounter, cb){ + async.forEach(event.data.encounters, function(encounter, cb){ // first, look up event via/orig key and see if we've processed it yet, if not (for some reason) ignore var key = url.format(getKey(encounter.network, encounter.via)); dataStore.getItemByKey(key,function(err, item){ if(err || !item) return cb(err); // we tag this item with a ref to the link, super handy for cross-collection mashups - var hash = crypto.createHash('md5'); - hash.update(encounter.link); // turn link into hash as an id - itemRef(item, "link://links/#"+hash.digest('hex'), function(err, item){ + itemRef(item, event.idr, function(err, item){ if(err || !item) return cb(err); var u = url.parse(encounter.link); // if foursquare checkin and from a tweet, generate foursquare key and look for it @@ -286,6 +305,16 @@ function newResponse(item, type) } } +// sometimes links can be used as raw keys +// using the hash value here because a ver of mongo will silently barf on some http based keys!!! +function keyUrl(item, link) +{ + var u = url.parse(link); + if(!u || !u.host) return; + // all instagram links across all services seem to be preserved + if(u.host == 'instagr.am') item.keys["url:"+crypto.createHash('md5').update(link).digest('hex')] = item.ref; +} + // extract info from a tweet function itemTwitter(item, tweet) { @@ -322,6 +351,8 @@ function itemTwitter(item, tweet) hash.update(item.text.replace(/ http\:\/\/\S+$/,"")); // cut off appendege links that some apps add (like instagram) item.keys['text:'+hash.digest('hex')] = item.ref; } + // check all links + if(tweet.entities && tweet.entities.urls) tweet.entities.urls.forEach(function(link){keyUrl(item, link.expanded_url);}); // if this is also a reply if(tweet.in_reply_to_status_id_str) @@ -362,6 +393,7 @@ function itemFacebook(item, post) hash.update(item.text.substr(0,130)); // ignore trimming variations item.keys['text:'+hash.digest('hex')] = item.ref; } + if(post.link) keyUrl(item, post.link); // process responses! if(post.comments && post.comments.data) @@ -402,7 +434,7 @@ function itemFoursquare(item, checkin) hash.update(item.text.substr(0,130)); // ignore trimming variations item.keys['text:'+hash.digest('hex')] = item.ref; } - var profile = (checkin.user) ? checkin.user : profiles["foursquare"]; + var profile = checkin.user; item.from.id = 'contact://foursquare/#'+profile.id; item.from.name = profile.firstName + " " + profile.lastName; item.from.icon = profile.photo; @@ -461,6 +493,8 @@ function itemInstagram(item, pic) item.text = "New Picture"; } + if(pic.link) keyUrl(item, pic.link); + // process responses! if(pic.comments && pic.comments.data) { diff --git a/Collections/Timeline/dataStore.js b/Collections/Timeline/dataStore.js index 3a05aa6e6..37b177948 100644 --- a/Collections/Timeline/dataStore.js +++ b/Collections/Timeline/dataStore.js @@ -6,7 +6,7 @@ * Please see the LICENSE file for more information. * */ -var logger = require(__dirname + "/../../Common/node/logger").logger; +var logger = require(__dirname + "/../../Common/node/logger"); var crypto = require("crypto"); var async = require('async'); var lmongoutil = require("lmongoutil"); @@ -32,7 +32,7 @@ exports.getTotalItems = function(callback) { itemCol.count(callback); } exports.getTotalResponses = function(callback) { - respColl.count(callback); + respCol.count(callback); } exports.getAll = function(fields, callback) { @@ -66,7 +66,7 @@ exports.getResponses = function(arg, cbEach, cbDone) { exports.getSince = function(arg, cbEach, cbDone) { if(!arg || !arg.id) return cbDone("no id given"); - findWrap({"_id":{"$gt":lmongoutil.ObjectID(arg.id)}}, {sort:{_id:-1}}, linkCollection, cbEach, cbDone); + findWrap({"_id":{"$gt":lmongoutil.ObjectID(arg.id)}}, {sort:{_id:-1}}, itemCol, cbEach, cbDone); } // arg takes sort/limit/offset/find @@ -106,13 +106,12 @@ exports.addItem = function(item, callback) { for(var i in item.keys) hash.update(i); item.id = hash.digest('hex'); } -// logger.debug("addItem: "+JSON.stringify(item)); var responses = item.responses; if(responses) responses.forEach(function(r){ r.item = item.id; }); delete item.responses; // store responses in their own table delete item._id; // mongo is miss pissypants itemCol.findAndModify({"id":item.id}, [['_id','asc']], {$set:item}, {safe:true, upsert:true, new: true}, function(err, doc){ - if(err || !responses) return callback(err); + if(err || !responses) return callback(err, doc); async.forEach(responses, exports.addResponse, function(err){callback(err, doc);}); // orig caller wants saved item back }); } diff --git a/Collections/Timeline/fixtures/ig.tweet b/Collections/Timeline/fixtures/ig.tweet index 5fc47aa05..b7267ce07 100644 --- a/Collections/Timeline/fixtures/ig.tweet +++ b/Collections/Timeline/fixtures/ig.tweet @@ -75,6 +75,6 @@ "place": null, "retweeted": false, "in_reply_to_user_id": null, - "text": "Model S model http://t.co/PeedUocW" + "text": "Model S ... http://t.co/PeedUocW" } } \ No newline at end of file diff --git a/Collections/Timeline/fixtures/link b/Collections/Timeline/fixtures/link index 6b96bea82..bffc77bc4 100644 --- a/Collections/Timeline/fixtures/link +++ b/Collections/Timeline/fixtures/link @@ -1,10 +1,9 @@ { - "type": "link", - "via": "links", - "timestamp": 1315976215195, + "idr": "link://links/#FOOOOOOOOO", "action": "new", - "obj": { + "data": { "at": 1320220061000, + "id": "FOOOOOOOOO", "favicon": "http://t.co/favicon.ico", "link": "http://t.co/QgLsBzHy", "encounters": [{ diff --git a/Collections/Timeline/test.js b/Collections/Timeline/test.js new file mode 100644 index 000000000..dbd9ad34c --- /dev/null +++ b/Collections/Timeline/test.js @@ -0,0 +1,173 @@ +// BWARE, this is a scratch area used while dev'ing and is pretty ugly as it does a bunch of support work to simulate a working env +// TODO convert to a real unit testing framework +try{ + var lconfig = require('lconfig'); +}catch(E){ + console.error("export NODE_PATH=../../Common/node/"); + process.exit(); +} +var fs = require('fs'); +var url = require('url'); +var request = require('request'); +lconfig.load('../../Config/config.json'); +var lmongoclient = require('../../Common/node/lmongoclient')(lconfig.mongo.host, lconfig.mongo.port, "timelinetest", ["item", "response"]); +var logger = require('../../Common/node/logger'); +var async = require("async"); +var url = require("url"); + +var dataIn = require('./dataIn'); // for processing incoming twitter/facebook/etc data types +var dataStore = require("./dataStore"); // storage/retreival of raw items and responses + +boot(function(){dataStore.clear(false,testFB)}); + +function testFB() +{ + dataIn.processEvent(fixture("facebook.update"), function(err){ + if(err) console.error(err); + count(function(i1, r1){ + dataIn.processEvent(fixture("facebook.update"), function(err){ + if(err) console.error(err); + count(function(i2, r2){ + if(i2 != i1) return console.error("didn't dedup facebook.update"); + if(r2 != r1) return console.error("didn't dedup facebook.update responses"); + console.error("facebook.update OK"); + testFBTW(); + }); + }) + }); + }) +} + +function testFBTW() +{ + dataIn.processEvent(fixture("facebook.tweet"), function(err){ + if(err) console.error(err); + count(function(i1, r1){ + dataIn.processEvent(fixture("tweet.facebook"), function(err){ + if(err) console.error(err); + count(function(i2, r2){ + if(i2 != i1) return console.error("didn't dedup facebook.tweet"); + if(r2 != r1) return console.error("didn't dedup facebook.tweet responses"); + console.error("facebook<->twitter OK"); + testRT(); + }); + }) + }); + }) +} + +function testRT() +{ + count(function(i1, r1){ + dataIn.processEvent(fixture("twitter.rt"), function(err){ + if(err) console.error(err); + count(function(i2, r2){ + if(i2 - i1 != 1) return console.error("didn't process twitter.rt"); + if(r2 - r1 != 1) return console.error("didn't process twitter.rt response"); + console.error("ReTweet OK"); + testTWReply() + }); + }) + }); +} + +function testTWReply() +{ + count(function(i0, r0){ + dataIn.processEvent(fixture("twitter.reply.orig"), function(err){ + if(err) console.error(err); + count(function(i1, r1){ + dataIn.processEvent(fixture("twitter.reply"), function(err){ + if(err) console.error(err); + count(function(i2, r2){ + if(i2 != i1 && i2 - i0 != 1) return console.error("didn't process twitter.reply properly"); + if(r2 - r1 != 1 || r2 - r0 != 2) return console.error("didn't dedup twitter.reply responses"); + console.error("twitter reply OK"); + test4sqLink(); + }); + }) + }); + }) + }); +} + +function test4sqLink() +{ + count(function(i0, r0){ + dataIn.processEvent(fixture("foursquare.recents"), function(err){ + if(err) console.error(err); + count(function(i1, r1){ + dataIn.processEvent(fixture("twitter.timeline"), function(err){ + if(err) console.error(err); + count(function(i2, r2){ + dataIn.processEvent(fixture("link"), function(err){ + if(err) console.error(err); + count(function(i3, r3){ + if(i2 - i0 != 2 || i2 - i3 != 1) return console.error("didn't dedup with link properly"); + if(r3 - r0 != 0) return console.error("found responses and shouldn't have"); + console.error("link based dedup OK"); + testIG(); + }); + }) + }); + }) + }); + }) + }); +} + +function testIG() +{ + count(function(i0, r0){ + dataIn.processEvent(fixture("ig.instagram"), function(err){ + if(err) console.error(err); + count(function(i1, r1){ + dataIn.processEvent(fixture("ig.fb"), function(err){ + if(err) console.error(err); + count(function(i2, r2){ + dataIn.processEvent(fixture("ig.tweet"), function(err){ + if(err) console.error(err); + count(function(i3, r3){ + if(i3 - i0 != 1) return console.error("didn't dedup instagram properly"); + if(r3 - r0 != 0) return console.error("found responses and shouldn't have"); + console.error("instagram dedup OK"); + }); + }) + }); + }) + }); + }) + }); +} + + + +function count(callback) +{ + dataStore.getTotalItems(function(err, items){ + dataStore.getTotalResponses(function(err, resp){ + console.error("Items: "+items+" Responses: "+resp); + callback(items, resp); + }) + }) +} +function fixture(name) +{ + return JSON.parse(fs.readFileSync(__dirname+"/fixtures/"+name)); +} + +function boot(callback){ + var tdir = __dirname+"/test"; + try{ + fs.mkdirSync(tdir,0755); + }catch(E){} + process.chdir(tdir); + lmongoclient.connect(function(mongo) { + // initialize all our libs + dataStore.init(mongo.collections.item,mongo.collections.response); + dataIn.init({}, dataStore, function(){ + console.error("booted up"); + callback(); + }); + }); +} From cd4a4f9f569628f5f0bffab1705fc74827985cb0 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Wed, 28 Dec 2011 03:22:18 -0600 Subject: [PATCH 27/40] added instagram-4sq deduping (plus time based), reorgd the code some, and more testing, nearly ready --- Collections/Links/links.js | 1 - Collections/Timeline/dataIn.js | 113 +++++++---------------- Collections/Timeline/fixtures/ig.4sq | 106 +++++++++++++++++++++ Collections/Timeline/sync.js | 88 ++++++++++++++++++ Collections/Timeline/test.js | 15 +-- Collections/Timeline/timeline.collection | 3 +- Collections/Timeline/timeline.js | 10 +- 7 files changed, 246 insertions(+), 90 deletions(-) create mode 100644 Collections/Timeline/fixtures/ig.4sq create mode 100644 Collections/Timeline/sync.js diff --git a/Collections/Links/links.js b/Collections/Links/links.js index dc45c3734..0d267f723 100644 --- a/Collections/Links/links.js +++ b/Collections/Links/links.js @@ -168,7 +168,6 @@ app.get('/', function(req, res) { if(req.query['stream'] == "true") { res.writeHead(200, {'content-type' : 'application/jsonstream'}); - options = {}; // exclusive } dataStore.getLinks(options, function(item) { if(req.query['stream'] == "true") return res.write(JSON.stringify(item)+'\n'); diff --git a/Collections/Timeline/dataIn.js b/Collections/Timeline/dataIn.js index 2065509d1..b27934d97 100644 --- a/Collections/Timeline/dataIn.js +++ b/Collections/Timeline/dataIn.js @@ -15,74 +15,13 @@ exports.init = function(l, dStore, callback){ callback(); } -// manually walk and reindex all possible link sources -exports.update = function(locker, type, callback) { - dataStore.clear(type, function(){ - callback(); - var types = (type) ? [type] : ['home/facebook', 'tweets/twitter', 'checkin/foursquare', 'feed/instagram']; - locker.providers(types, function(err, services) { - if (!services) return; - async.forEachSeries(services, function(svc, cb) { - logger.debug("processing "+svc.id); - if(svc.provides.indexOf('home/facebook') >= 0) { - getData("home/facebook", svc.id, cb); - } else if(svc.provides.indexOf('tweets/twitter') >= 0) { - async.forEachSeries(["tweets/twitter", "timeline/twitter", "mentions/twitter", "related/twitter"], function(type, cb2){ getData(type, svc.id, cb2) }, cb); - } else if(svc.provides.indexOf('checkin/foursquare') >= 0) { - async.forEachSeries(["recents/foursquare", "checkin/foursquare"], function(type, cb2){ getData(type, svc.id, cb2) }, cb); - } else if(svc.provides.indexOf('feed/instagram') >= 0) { - async.forEachSeries(["photo/instagram", "feed/instagram"], function(type, cb2){ getData(type, svc.id, cb2) }, cb); - } else { - cb(); - } - }, function(err){ - // process links too - lutil.streamFromUrl(locker.lockerBase+'/Me/links/?full=true&limit=100&stream=true', function(link, cb){ - processLink({data:link}, cb); - }, function(err){ - if(err) logger.error(err); - logger.debug("done with update"); - }); - }); - }); - }); -} - -// go fetch data from sources to bulk process -function getData(type, svcId, callback) -{ - var subtype = type.substr(0, type.indexOf('/')); - var lurl = locker.lockerBase + '/Me/' + svcId + '/getCurrent/' + subtype + "?limit=100&sort=_id&order=-1&stream=true"; - var tot = 0; - lutil.streamFromUrl(lurl, function(a, cb){ - tot++; - var idr = getIdr(type, svcId, a); - masterMaster(idr, a, cb); - }, function(err){ - logger.debug("processed "+tot+" items from "+lurl); - callback(err); - }); -} - -// generate unique id for any item based on it's event -//> u.parse("type://network/context?id=account#46234623456",true); -//{ protocol: 'type:', -// slashes: true, -// host: 'network', -// hostname: 'network', -// href: 'type://network/context?id=account#46234623456', -// hash: '#46234623456', -// search: '?id=account', -// query: { id: 'account' }, -// pathname: '/context' } -function getIdr(type, via, data) +// take an idr and turn it into a generic network-global key +// this could be network-specific transformation and need data +function idr2key(idr, data) { - var r = {slashes:true}; - r.host = type.substr(type.indexOf('/')+1); - r.pathname = type.substr(0, type.indexOf('/')); - r.query = {id: via}; // best proxy of account id right now - idrHost(r, data); - return url.parse(url.format(r)); // make sure it's consistent + delete idr.query; delete idr.search; // not account specific + idr.pathname = '/'; // ids are generic across any context + return url.parse(url.format(idr)); } // internal util breakout @@ -109,15 +48,7 @@ function idrHost(r, data) r.protocol = 'photo'; } } - -// take an idr and turn it into a generic network-global key -// this could be network-specific transformation and need data -function idr2key(idr, data) -{ - delete idr.query; delete idr.search; // not account specific - idr.pathname = '/'; // ids are generic across any context - return url.parse(url.format(idr)); -} +exports.idrHost = idrHost; // useful to get key from raw data directly (like from a via, not from an event) function getKey(network, data) @@ -144,7 +75,7 @@ exports.processEvent = function(event, callback) var profiles = {}; function masterMaster(idr, data, callback) { - if(idr.protocol == 'checkin:' && idr.pathname.indexOf('checkin') != -1) + if(idr.protocol == 'checkin:' && idr.pathname.indexOf('checkin') != -1 && idr.query.id) { // foursquare checkins api for a person is the same json format but missing the .user, which we need to store the correct contact idr, so we have to fetch/cache that var svcId = idr.query.id; if(profiles[svcId]) @@ -163,9 +94,11 @@ function masterMaster(idr, data, callback) data.user = profiles[svcId] = arr[0]; return masterBlaster(idr, data, callback); }); + return; } masterBlaster(idr, data, callback); } +exports.masterMaster = masterMaster; // figure out what to do with any data function masterBlaster(idr, data, callback) @@ -294,6 +227,7 @@ function processLink(event, callback) }); }, callback); } +exports.processLink = processLink; // give a bunch of sane defaults function newResponse(item, type) @@ -315,6 +249,20 @@ function keyUrl(item, link) if(u.host == 'instagr.am') item.keys["url:"+crypto.createHash('md5').update(link).digest('hex')] = item.ref; } +// add key(s) based on a timestamp+-window (all in seconds) +function keyTime(item, at, window, prefix) +{ + var keys = {}; + var wlen = window.toString().length; + var alen = at.toString().length; + keys[at.toString().substr(0,alen-wlen)]=true; + at += window; + keys[at.toString().substr(0,alen-wlen)]=true; + at -= (window*2); + keys[at.toString().substr(0,alen-wlen)]=true; + Object.keys(keys).forEach(function(key){ item.keys[prefix+key] = item.ref; }); +} + // extract info from a tweet function itemTwitter(item, tweet) { @@ -352,7 +300,7 @@ function itemTwitter(item, tweet) item.keys['text:'+hash.digest('hex')] = item.ref; } // check all links - if(tweet.entities && tweet.entities.urls) tweet.entities.urls.forEach(function(link){keyUrl(item, link.expanded_url);}); + if(tweet.entities && tweet.entities.urls) tweet.entities.urls.forEach(function(link){if(link.expanded_url) keyUrl(item, link.expanded_url);}); // if this is also a reply if(tweet.in_reply_to_status_id_str) @@ -434,6 +382,11 @@ function itemFoursquare(item, checkin) hash.update(item.text.substr(0,130)); // ignore trimming variations item.keys['text:'+hash.digest('hex')] = item.ref; } + // foursquare preserves no other metadata than the shout in a cross-post, so we fall back to timestamp window :( + if(checkin.photos && checkin.photos.items) checkin.photos.items.forEach(function(photo){ + if(!photo.source || photo.source.name != "Instagram") return; + keyTime(item, photo.createdAt, 5, "igts:"); + }); var profile = checkin.user; item.from.id = 'contact://foursquare/#'+profile.id; item.from.name = profile.firstName + " " + profile.lastName; @@ -475,7 +428,7 @@ function itemInstagram(item, pic) { item.pri = 4; // the source item.first = item.last = pic.created_time * 1000; - if(pic.caption) item.text = pic.caption.text; + if(pic.caption && pic.caption.length > 0) item.text = pic.caption.text; if(pic.user) { item.from.id = 'contact://instagram/#'+pic.user.id; @@ -491,6 +444,8 @@ function itemInstagram(item, pic) item.keys['text:'+hash.digest('hex')] = item.ref; }else{ item.text = "New Picture"; + // if there's no caption, we have to key the timestamp to catch foursquare cross-posting since it has no other metadata :( + keyTime(item, pic.created_time, 5, "igts:"); } if(pic.link) keyUrl(item, pic.link); diff --git a/Collections/Timeline/fixtures/ig.4sq b/Collections/Timeline/fixtures/ig.4sq new file mode 100644 index 000000000..e0ac671ef --- /dev/null +++ b/Collections/Timeline/fixtures/ig.4sq @@ -0,0 +1,106 @@ +{ + "idr": "checkin://foursquare/recents?id=foursquare#4eb0dfbbe3002b5426f05a51", + "action": "new", + "data": { + "id": "4eb0dfbbe3002b5426f05a51", + "createdAt": 1320214459, + "type": "checkin", + "shout": "11.1.11, 11:11", + "timeZone": "America/Los_Angeles", + "entities": [], + "user": { + "id": "1064267", + "firstName": "Jason", + "lastName": "Cavnar", + "photo": "https://playfoursquare.s3.amazonaws.com/userpix_thumbs/NVPLSYYLJHBHSIEW.jpg", + "gender": "male", + "homeCity": "Palo Alto, CA", + "relationship": "friend" + }, + "venue": { + "id": "4daf691893a037b7b06ee236", + "name": "SINGLY", + "contact": { + "twitter": "singlyinc" + }, + "location": { + "address": "777 Florida St", + "lat": 37.75926, + "lng": -122.410651, + "postalCode": "94110", + "city": "San Francisco", + "state": "CA", + "country": "USA" + }, + "categories": [{ + "id": "4bf58dd8d48988d125941735", + "name": "Tech Startup", + "pluralName": "Tech Startups", + "shortName": "Tech Startup", + "icon": "https://foursquare.com/img/categories/building/default.png", + "parents": ["Professional & Other Places", "Offices"], + "primary": true + }], + "verified": true, + "stats": { + "checkinsCount": 449, + "usersCount": 51, + "tipCount": 1 + }, + "url": "http://singly.com" + }, + "source": { + "name": "Instagram", + "url": "http://instagram.com" + }, + "photos": { + "count": 1, + "items": [{ + "id": "4eb0dfbcdab4a8a58a48dd35", + "createdAt": 1316385389, + "url": "https://playfoursquare.s3.amazonaws.com/pix/PBRGCC2IKYJVJN41TOI1SNQIUJY2U2DUOFTIOP5FNYZ4ZXWG.jpg", + "sizes": { + "count": 4, + "items": [{ + "url": "https://playfoursquare.s3.amazonaws.com/pix/PBRGCC2IKYJVJN41TOI1SNQIUJY2U2DUOFTIOP5FNYZ4ZXWG.jpg", + "width": 612, + "height": 612 + }, + { + "url": "https://playfoursquare.s3.amazonaws.com/derived_pix/PBRGCC2IKYJVJN41TOI1SNQIUJY2U2DUOFTIOP5FNYZ4ZXWG_300x300.jpg", + "width": 300, + "height": 300 + }, + { + "url": "https://playfoursquare.s3.amazonaws.com/derived_pix/PBRGCC2IKYJVJN41TOI1SNQIUJY2U2DUOFTIOP5FNYZ4ZXWG_100x100.jpg", + "width": 100, + "height": 100 + }, + { + "url": "https://playfoursquare.s3.amazonaws.com/derived_pix/PBRGCC2IKYJVJN41TOI1SNQIUJY2U2DUOFTIOP5FNYZ4ZXWG_36x36.jpg", + "width": 36, + "height": 36 + }] + }, + "source": { + "name": "Instagram", + "url": "http://instagram.com" + }, + "user": { + "id": "1064267", + "firstName": "Jason", + "lastName": "Cavnar", + "photo": "https://playfoursquare.s3.amazonaws.com/userpix_thumbs/NVPLSYYLJHBHSIEW.jpg", + "gender": "male", + "homeCity": "Palo Alto, CA", + "relationship": "friend" + }, + "visibility": "public" + }] + }, + "comments": { + "count": 0, + "items": [] + } + } +} \ No newline at end of file diff --git a/Collections/Timeline/sync.js b/Collections/Timeline/sync.js new file mode 100644 index 000000000..ba232d864 --- /dev/null +++ b/Collections/Timeline/sync.js @@ -0,0 +1,88 @@ +var async = require('async'); +var logger = require(__dirname + "/../../Common/node/logger"); +var lutil = require('lutil'); +var url = require('url'); + +var dataStore, dataIn, locker; + +// internally we need these for happy fun stuff +exports.init = function(l, dStore, dIn, callback){ + dataStore = dStore; + dataIn = dIn; + locker = l; + callback(); +} + +// manually walk and reindex all possible link sources +exports.update = function(locker, type, callback) { + dataStore.clear(type, function(){ + callback(); + var types = (type) ? [type] : ['home/facebook', 'tweets/twitter', 'checkin/foursquare', 'feed/instagram']; + locker.providers(types, function(err, services) { + if (!services) return; + async.forEachSeries(services, function(svc, cb) { + logger.debug("processing "+svc.id); + if(svc.provides.indexOf('home/facebook') >= 0) { + getData("home/facebook", svc.id, cb); + } else if(svc.provides.indexOf('tweets/twitter') >= 0) { + async.forEachSeries(["tweets/twitter", "timeline/twitter", "mentions/twitter", "related/twitter"], function(type, cb2){ getData(type, svc.id, cb2) }, cb); + } else if(svc.provides.indexOf('checkin/foursquare') >= 0) { + async.forEachSeries(["recents/foursquare", "checkin/foursquare"], function(type, cb2){ getData(type, svc.id, cb2) }, cb); + } else if(svc.provides.indexOf('feed/instagram') >= 0) { + async.forEachSeries(["photo/instagram", "feed/instagram"], function(type, cb2){ getData(type, svc.id, cb2) }, cb); + } else { + cb(); + } + }, function(err){ + if(type) return logger.debug("done with update for "+type); + // process links too + var tot = 0; + lutil.streamFromUrl(locker.lockerBase+'/Me/links/?full=true&limit=100&stream=true', function(link, cb){ + tot++; + dataIn.processLink({data:link}, cb); + }, function(err){ + if(err) logger.error(err); + logger.debug(tot+" links processed, done with update"); + }); + }); + }); + }); +} + +// go fetch data from sources to bulk process +function getData(type, svcId, callback) +{ + var subtype = type.substr(0, type.indexOf('/')); + var lurl = locker.lockerBase + '/Me/' + svcId + '/getCurrent/' + subtype + "?limit=100&sort=_id&order=-1&stream=true"; + var tot = 0; + lutil.streamFromUrl(lurl, function(a, cb){ + tot++; + var idr = getIdr(type, svcId, a); + dataIn.masterMaster(idr, a, cb); + }, function(err){ + logger.debug("processed "+tot+" items from "+lurl); + callback(err); + }); +} + +// generate unique id for any item based on it's event +//> u.parse("type://network/context?id=account#46234623456",true); +//{ protocol: 'type:', +// slashes: true, +// host: 'network', +// hostname: 'network', +// href: 'type://network/context?id=account#46234623456', +// hash: '#46234623456', +// search: '?id=account', +// query: { id: 'account' }, +// pathname: '/context' } +function getIdr(type, via, data) +{ + var r = {slashes:true}; + r.host = type.substr(type.indexOf('/')+1); + r.pathname = type.substr(0, type.indexOf('/')); + r.query = {id: via}; // best proxy of account id right now + dataIn.idrHost(r, data); + return url.parse(url.format(r),true); // make sure it's consistent +} + diff --git a/Collections/Timeline/test.js b/Collections/Timeline/test.js index dbd9ad34c..6826ce751 100644 --- a/Collections/Timeline/test.js +++ b/Collections/Timeline/test.js @@ -127,11 +127,14 @@ function testIG() count(function(i2, r2){ dataIn.processEvent(fixture("ig.tweet"), function(err){ if(err) console.error(err); - count(function(i3, r3){ - if(i3 - i0 != 1) return console.error("didn't dedup instagram properly"); - if(r3 - r0 != 0) return console.error("found responses and shouldn't have"); - console.error("instagram dedup OK"); - }); + dataIn.processEvent(fixture("ig.4sq"), function(err){ + if(err) console.error(err); + count(function(i3, r3){ + if(i3 - i0 != 1) return console.error("didn't dedup instagram properly"); + if(r3 - r0 != 0) return console.error("found responses and shouldn't have"); + console.error("instagram dedup OK"); + }); + }) }) }); }) @@ -146,7 +149,7 @@ function count(callback) { dataStore.getTotalItems(function(err, items){ dataStore.getTotalResponses(function(err, resp){ - console.error("Items: "+items+" Responses: "+resp); +// console.error("Items: "+items+" Responses: "+resp); callback(items, resp); }) }) diff --git a/Collections/Timeline/timeline.collection b/Collections/Timeline/timeline.collection index a4927130e..53115be9a 100644 --- a/Collections/Timeline/timeline.collection +++ b/Collections/Timeline/timeline.collection @@ -6,8 +6,9 @@ "handle":"timeline", "provides":["item"], "mongoCollections": ["item","response"], - "events":[["recents/foursquare","/events"] + "events":[["checkin/foursquare","/events"] ,["tweet/twitter","/events"] + ,["photo/instagram","/events"] ,["related/twitter","/events"] ,["post/facebook","/events"] ,["link","/events"] diff --git a/Collections/Timeline/timeline.js b/Collections/Timeline/timeline.js index afdc0a9e0..a851a01dc 100644 --- a/Collections/Timeline/timeline.js +++ b/Collections/Timeline/timeline.js @@ -21,6 +21,7 @@ var url = require("url"); var dataIn = require('./dataIn'); // for processing incoming twitter/facebook/etc data types var dataStore = require("./dataStore"); // storage/retreival of raw items and responses +var sync = require('./sync'); // for manual updating/resyncing of data from synclets var lockerInfo; var express = require('express'), @@ -103,7 +104,7 @@ app.get('/ref', function(req, res) { }); app.get('/update', function(req, res) { - dataIn.update(locker, req.query.type, function(){ + sync.update(locker, req.query.type, function(){ res.writeHead(200); res.end('Extra mince!'); }); @@ -119,6 +120,7 @@ app.post('/events', function(req, res) { // handle asyncadilly dataIn.processEvent(req.body, function(err){ + if(err) logger.error(err); if(err) return res.send(err, 500); res.send('ok'); }); @@ -155,8 +157,10 @@ process.stdin.on('data', function(data) { // initialize all our libs dataStore.init(mongo.collections.item,mongo.collections.response); dataIn.init(locker, dataStore, function(){ - app.listen(lockerInfo.port, 'localhost', function() { - process.stdout.write(data); + sync.init(locker, dataStore, dataIn, function(){ + app.listen(lockerInfo.port, 'localhost', function() { + process.stdout.write(data); + }); }); }); }); From 8ef2d4ac4c9259731dcba92687abe64ebcb47e08 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Wed, 28 Dec 2011 12:53:58 -0600 Subject: [PATCH 28/40] add facebook story support --- Collections/Timeline/dataIn.js | 1 + 1 file changed, 1 insertion(+) diff --git a/Collections/Timeline/dataIn.js b/Collections/Timeline/dataIn.js index b27934d97..a94e75c74 100644 --- a/Collections/Timeline/dataIn.js +++ b/Collections/Timeline/dataIn.js @@ -334,6 +334,7 @@ function itemFacebook(item, post) if(post.name) item.title = post.name; if(post.message) item.text = post.message; if(!post.message && post.caption) item.text = post.caption; + if(!post.message && !post.caption && post.story) item.text = post.story; // should we only add a text key if we can detect it's a tweet? we're de-duping any fb post essentially if(item.text) { From ef1961cf9d0dc05b0d7a2a1e45696b76bee89371 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Wed, 28 Dec 2011 13:56:42 -0600 Subject: [PATCH 29/40] easier to test it, can expand responses --- Apps/HelloTimeline/img/facebook.png | Bin 0 -> 7056 bytes Apps/HelloTimeline/img/foursquare.png | Bin 0 -> 2045 bytes Apps/HelloTimeline/img/instagram.png | Bin 0 -> 2822 bytes Apps/HelloTimeline/img/twitter.png | Bin 0 -> 6269 bytes Apps/HelloTimeline/script.js | 71 ++++++++++++++++++++++++-- 5 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 Apps/HelloTimeline/img/facebook.png create mode 100644 Apps/HelloTimeline/img/foursquare.png create mode 100644 Apps/HelloTimeline/img/instagram.png create mode 100644 Apps/HelloTimeline/img/twitter.png diff --git a/Apps/HelloTimeline/img/facebook.png b/Apps/HelloTimeline/img/facebook.png new file mode 100644 index 0000000000000000000000000000000000000000..355ac84faf03b7e73f79c85d8dedd7620fd0666a GIT binary patch literal 7056 zcmeHMc{tSV*SAI1Y-LMLL)K=ohp|O=+2bKHX2#597-j}zdu-YFND5^s6j?$%ZOC4- zl(IyYr|d$=&O7Mo@m%lIulJAl_g`~ebA9jgJ?DJRxj*OJ=eq8hNRx9qtW5k&R8&-~ zdb(O>lqYoOdyt-riW+dBg@YpSka5Tyc1xI?jx1eAY z=LD>UHObn@Pzix|0mFA}z+@LUiZ>M%OpWXYM>wNM07n!W>#72reOwI$V38_7D>);m zk((w8gVpsRpv-;FSs;9z5sFBlnko~FtV98DL6P79vJ1|Ys6{^?FE1~!mkbzBKtrSy6%`>+X^6Bmh++XE zdb^U~WRNRSXa`~sLkmSj5U_3}EZ!Bcg9&%UdyrIsK#Csl4IGL1j_u|_!0jR<5fBs( z<$`i05g}4wDad|%3KM2nG=@a{?&eDT#xjL(2pR51ae+bzyEgyubi?3Dcp?Vx_V1qG z)w}-tL+ZJa5VDlGJA{7srR)l^n=FxpgzUH2b>FSiUi9w?D_Nl&%}H?5?#903IO6bV zO8!VJf`men@Cdw<6AA@JqIMJbo|BS>2ML2G5KZtz><*h04tGcfz>o-~y{+zwd#r2V zNPE(rh{Ntst%QTSqE&!o5EA7C_rQ^Ws?YC!EBVetM2M`Rywewo07q|tF%IjAA^_^>+ zLc)@8sC_a1%II77{b1veA8z7rJlNk=vm{c^w*`veLj3=V^h0de{ytLD+~ZIaPdQfz zlAfy*e)rh?(*~=7B%CFa;7Gz>$LfFUHL)b`A8XAhlW4c0et>R*z#}Pbzjq-11)N0q z5x4~kXN4zFX41d@t^2o z`um!Na;4m@yeOBbgVAHMl)G7qo|c9MxzBN9ds~fGzJ><4sJXUPR#qnCx3UtX5qL3h zc&z;viy?q90Z^RETX!Gy)`7pC!2tIAQsVG}aRi64+9vfvE$4Q?CX&1`Drh~@J!b5n zD%4DXM}OHaoIc7DgPV4!9aIZ|Qqi#sowEwMP~Ti%U(?poGPkm2M=!-F9yYct52Jfu zOG_H*YjdMlaaM*3(52pl`PR*C3@FgUsa?a6hXkSPOG}dYRNk#k)XLJI0(E8Aj`Lm` zJx*OVth)VmOEARfwP6IYT2q*c=HRr1$8w<;^)q-dlt)UMW|Lkx@Cr+FoazIj&!HY+ zHv7Z18R1}PiMt!LgnAR3RsXWXoR8%^GU-v^Fi%Hy_3s-Fqm^Cd|hpc?&auu1( z*Y)u%pU;a6!?|A~BA^=^i;Ngaq38GGZ#{8z_%!0=Ft*+>H$9~mrX2r4I=`7wW*mh_ zuB$BvK$R7IbtE5Xi-CNf#&uU`rVC+>-rVJLwZ%8%B<^jg#j`oSXYx#`5R-?CWe{3);Mm*)> zDNFu^lh!Y`jD**g7^_+iZRt?n&Z-C3<;*!y^W~hlH$SQSPShFBToFFXO(&8c-nH$1 zs$eO@;`CU4P;`!&g%9>_UnnAZ8g2hYAL4Ce^Hf&1@s{&8O~?6}Q>=6vPauxqwX|Z(GRW2 zzTK4+)#Gl--nh^l=uVG5wBFZxkd<|@z69{)^(&dD2iUxOPq{m^WLn42aJHFZY?bo` zyq?NSkFRFOH_I)OMQMT8`qM7<=H-;x`lmz63rc7PPZYgX!dwk%;jPs^aUw|P$b<2j z*3QOBEs2?>2C0F{Lv5G>GWfNL##*4b&;~A0+#L}6y4>4+k^OdzAdl~AseVYAa`|If z{mlVEA1|M!ggAp@eX@4+kqYZ_ET3`AKyddmCSRPvRQRSKD0PTeNw-TuigrN;qOFz;3Ptkb7&1KsCk8@0~KYpw18&{T^hQW zsFhEIx)g%S%S#T^9tqBHK-61soGoF#YiWX`(Jt*st!HnIPp?##c{cm1e^kEmn(U>m zukw!!-7=CL#xkt@_>y$2v(D3oT~jKHW~E-O#`CTu3jXzqZN)^RKXmP zTxnKy^4;>Eoi1%r`bzVt|HuunZi3dazK7_L;agup99;A=<%yY9{8>07P}odgB`Pk% zKqf=UU)$ui-8A3DQW4ub)Hf!2p%X#Z5~vESf3weRsx0p4Elo;Sn5=J@lKEJACZlB5 zE*`unqFY-r9(34FJ=?2HA-|JX?2yw3CW$=Q3@fI~%hbb|=3s1TqET!`HbWR+g(613 zwz|kgZ)Q0DD1(B8{4G7!v3peqJ;G(PKg*ex^Y$OR_hx>y=m>7695eYLID!AB>Etv& z!z=QsGGXj-A1-wwNdEAeb0&lKxUa`22D!JNA1w60;ZIvRh3{;RgujM4zbtDLEaokc z-smleqGL*Hkm z%@$?8sKu}5KZrbx@M`xg%5j?UjJ{qq&}3ON>Q+X@%JwdHHg8O|qi1dySlgZUi0sTu<>L(UuW~o`2bNA!tNj==*e24=NZWtEkj8`StAGbQuc&pS< zg5FB?n1bK=Sx@?8DH{{E2>)is>omCNtP|oBHK29{%Pf)fEF#CzN7@peWhr;tJs}XX?8KjIPj1m zhs!umjbbwTS?Quka=g|<&&$TM?JHZnK6mQ)edBa4%{snG6t_dIL3_q4`L>-m+X6~K z0d(VSllD_$*(=w32Ym5n1z26pF8z6W4XwiCiNTpGLky1dwG13t2hXgFu857O4W+GQ znk?RT&{)h`6K1l(AWXF+Z7i;&^xvrO&(-_f)D}Zjb$U?Qn>Ot)7Qy^-df;*fu`OM( zdT_QCbZhARMOwC_zMsfX2PUe^s-CQEI29|*Uz8?2Id7{hE4SJp7(0Ex-8i5hG;lx- zR6csk=W9J}cYwV~rPT)X#2-vXJcrXDkL`}i4v*5XA16XOj=hkKU7A=80jp1!Q-RArd)Pdtw{Hce?(`&Lvip-0tFMJzo7UzQ-X z*{vDivo8XJ+$*^amZKgW`BU~a98@i5z$pgHDe-8meP66b4XK7!dVNe-@}YMMrTijY zf9VX7#1s_NarO^0jgmZC4TaFQgM3kvL!C_=*tfE|E2=2;jU>d}N}|(5uFCohYGrx@ zi1TK%S%`l4uy?6kN=WOa_=r!M^w6+~I^9pE*IXhJqQv3A5tNr@GYMaIun?deX0j)hXac zm)qaypeNY6t;V)5LjoRiubKy6kU-&}S_^LW?VY8s;aj58pAW9QTD3VnnyCb?uw(64 zN@*)JUA~*CbH);hsjie41L)6gI91uzrwpfg1nQ6boAa<8Rb1uGWy_7q@iP|pD5;3d zJ{s_6DV2%u2}I(}xj{DLuAu=go#8EChWDy3w{g$Yj$a6zz4kD*VoW#ty0C5CNCTg% zu-TyTR?1la&ALGiwhjOJP4CC0Fa^?q=hB^x?pH!+B&aXs^IuM4Fts70x16!rFQS79 zon)@H3eP#^uKrW4T%+%bV(uO4rM33dv-H+Ln1=lsUb-c;-o$u#$_VFs+kU<3%vE)N zndI${{YEH|umo_@?1-I%VXe3TEviAo>JC#@pSmcgATqG%CsFH^ij$CEryq zVpK6`6-A9d8o(q+#_2j+n}y1ya;S`M8`J1a=1Q=caJ2|zM8rt>t!aiyPBC%l{WdA0 zNPT_6S?mo@T)hvE582{uBX3_b=_n$`gh3Tl%UySlh7WJl<#*~8&O_K!Hd3fgoPn!} z$w;Q0){I#n65kePP&jdt%H$c1$Z}_sCVGngbzS|_8q+ba*P&3q4@gXT4lye;;cgck z*{SPtJ#UCkN%TdK9m{hT^Kc)~`*}fRHs1@cX+TRZ3#Rp{k3o~+=iFB=CuhU)!%Vp3 zN4zNoTvxz%XluMfKK?cadw3pS6OJB*JNZ->N5B-1!S5G)4RuR8cWpg*eP`tAP}J!2 z$_8=tObKm7ryy?+Pn>PTFt?UoQa{sN`T4cE52Aw_zy$koq}iycgx+1Yyp!PM367W# zuY$~HoO$wQM&9~vJh!^~_C1<0Q#;k>5g!;f>*1k({E>65nx8Cpq(8U7h{b)kyilnm zDe;)d8g((7&}Xe|(f9k$MWtQsY-ZHqU6=mKgs39oiDLn8&ad*kOwz7KX0!o!{v*>n Mdrqt9j6?AM06M3OIsgCw literal 0 HcmV?d00001 diff --git a/Apps/HelloTimeline/img/foursquare.png b/Apps/HelloTimeline/img/foursquare.png new file mode 100644 index 0000000000000000000000000000000000000000..28ed5e3e4df15d75459ea68274c9fb1b31095c4b GIT binary patch literal 2045 zcmV771tI1?##ULyv_Ub5**tx2Gt}E5TI(3(xz#e4Wf`uib5h8sjAZc3xE2f zKM?<`_Fo$*KR{};NR`@DqOBsOWNVfx${tK0HrT}mgYm+{Z=a>-+_`t&yl10a>v-nP za=-7K^WAgL2&nP!D{oJ>C}xP+J;8#6cRNg=i*$M)06csQG?C|7Nch@UQ84oV~ho{KV5g zJ|rG^`K^g9-JQpt7#qS;Ex_zr6$RIq#={V70w$RuLZ1yqP{b!!<`aq^ghx-E^GI8U zw4scRu7to}CeLGfY4wnmps;W6?ZW4)RV>Pp!~UowH6eL2jmtR1l&D>0n+4P2?jz$F-gD*~qaZ76UZfhZpYKzL0A0!hN9 zb3sRs@~@-8j&{?eig$%!;A|RLmO)p5) zt|H-j0sehz83%96F@^zOgQMr7md7tQlPGW+!OiUi+3)=6mz zU`60M^s$%9#P-bG8dlb7QH0u(7Vg~Afwoj)od|XQIs8I}ADmqR59s*C)GA)M^Ck(- z!=r1K8w83#quOUK7xCq1^B6xng{^ZJVU{cK9S4gyZ^G+m1~GE`0Pfq?9V0+OlamGw z;5t1Yo?XVSo>T__ME)Tv20ESyXdm4u06v4Z~T^BA4Ki2vko z!&_nu_mhyyHBWkO(*&*|BH>AJ`tjcBW!&1~;K+SL>#CzAL#D*V1WFYTr@mOg!RaY< zWYe%Li&X2OSSrCZP2}?hRBJBwesv0eZ_QwHE`^idEXfI%c<-0TT%7s;1+`t0mlVc>%d(64^`!7J2Sb0ZCX1`1C@xT0^;1fo)SUnL3A8ve{_D zKYcJKNmxXJoWmPmU%{3(dqZ#@*U4E!R)AQMnOiDjs8B?b@@*$=SY!#RT&bjk7Mso! zmdbg6zU7sM2^X$aB#9ako}9mqBwf1^_*ruF_a9yeHDD-(>%x{R79gA!c~KZbMW?dt3S6q&E|pD} z%5<4l0f*mpy=YggQZzOi8a&%#uDkc@=Zp9?f$!edffPMwan)kw>v}NIoy1Hkh2G_5 zcqBd+As-9HlC*NUN@=4*%T+~)3o_TFv}0n&ST|1Oci`Ul|BDs^_qm&)(DQR?(uR|9aQ83nMfXS# z4sY`C&bez6=(qbu)`7Fw7<~IkJBIs`viJmQ&td1S8mFMCso30~hVg@2aMH0cczQ~z zy|-9IDU(L~_#hsA{vKp|+av%39^N~Et+^!nTkQsLj?B?FXXW{&Ra#xW$$Gga4KcP% z0sp9XGLT#9D3RFIxfvJ6Z^Gna5g(OY{ z@+3Bkke8Vs+e~yjl_s$9!B#xBrytMWJ%FCpmMEWk@{Nj1{C-!~#)EhCgvAMg+^YG= ztB9KwqlS6}yr`4WT}{YRs+A^q^85YBsW8VqjOY8{Yv0IquE8JL8G?IiK0Pl-Kw?W4 zLPWh=okZPOIBW(@^u(S<61=o@zS;9j`Dz`UJU1<|ZoY5>4BItJ7b4xqg*_ai4F$1X zjXNikNF3PRhrfNifUDQ5vaozYJp+5MhxVMGWeJRJIuAd6ZN6MR zu&X1B{d({vL!nZB zLCUZFkN-H9%VZ`F4EAHZtChMRi_C2JOcSwhf#wfX?VSER(+x}f6SYQrNzHZf)lwd> z&tAl0p>TZlm-`MyEk^4fPE3$=kI=#$4tq4?J3h@f9yk0%=e}Hq&Yd9z9?k!1zx+J^ bzW@UO)gqSF%=n2500000NkvXXu0mjflV#xp literal 0 HcmV?d00001 diff --git a/Apps/HelloTimeline/img/instagram.png b/Apps/HelloTimeline/img/instagram.png new file mode 100644 index 0000000000000000000000000000000000000000..d06b4aad3c8a762a5aba67f46764fd1fa0dec16e GIT binary patch literal 2822 zcmV+h3;FbkP);Vsy=ac_ z!#X>S`i2}%>vdQxugA{VG_+So!}HO6j;~}T_+!8b!g&U#1d+^|dwqjkSr#%w!`Of4 zr;a^%_|e(nD~TuE0nt3JU~?C<+YI zvZ>D_1i~~hJy2$Iq641i$(;;N3$+^gLJ>bd^5a{5sM&8Eek6Sfp!86B;`EDeV*8fO zh&eWt>tkCY2{jhO8#;s1`Q7NY7hw5iu*+Nm6tKw#6+{XIN(noG_;Y$;5$7gn1%OSg zFq0m!JJgrLaJm<%WCBn9^vK$Jt^Uo!kALSeF^K;?_`n1EUi#ZH3=d=wwJlf{Eub{H zD2d%WqFB3Y17gFwp)ChCJz>!X7H#P8l@K^|FF(h_re`B`<%KUy&toRHgj}JF>G?c8 zA3>lk%;bxt6(H4-Ku65MhZiRwY7!%W*7S8hQY{q`A!gW6kN=VJ89RcZejD+D9yq-N za0rkVwHj)f1&b`_s|sCZRAb&V@L0qVmo+?>*7lI@S%X+K0*4K!?>jq^c<-Gvkrghp zl)8$k<{MZ99#L*dXfQ4%JNTZ_iQ=%K%@i>aD73W0w)x}|6^1WAWYeaK*7RLhv^JB? zqFSpV-Q7hjY6*y_;~<;QuM)KitqKvha9W|MC)q9R7;c`7`Dp~x(G-p9vlN=aqlr zgL7vwHr!{()}+HI>5tsZ0kbL7L!x#CnAR&3V>9e^t>K|msgd|R(Lr~A1}|K<73Gg= z7;V(BrSn;wbsHEP8^`t=Z^qNl9LHZzypE~aS;|W_(FzN6+vrAo;d8fR-xZ_y%X5Fg z&XECOOqf@JL}8uSt<3{vo4B8qX?Ha@hQigI{O}8{q6+rzH2`Y+X;!>QA+uLMM(ea;CRHmpN>a2-)=Av-&V<42zp0dq35$^X~i za4mjx3eZKueeBym#B9MvdQ$@NbP{Fv zLwLDGxJ4fp2|bnUBwa;NDi&bJ0{r6lf5eTuuEN&Q5fn=$ab3U>82IwuBf<0LxWr{9 ziO_wX(92(Yq9rX~Mx#-jd-ohp{QVu+$uv}?L6c>PyJ$28H)#<#hs2&D*31Sjt(?o2 z@TZqw!LDnzQ~s&8_LYjpZsXA&^X*Z_5yV646DP$m+sgN`7~XpOUCfY{dgBQaVivWm zi>_D)qCtcfjZ+qHphjgP(bJ7oXAivIB&uo}XD&`t=o_Qzl#4i*@w$GP>NFXxwOXAR z8K)8;T0j|KL7)vNVk?)+qg2i#)hMA_wxR0{3S|NT!Am zuLLM~8VzEeHR!so>|siaX$>4s1!@%_0uV?~BLn6kq1SJf1h&krS5Pf2!75Rrqc+5e z@O_+#V~G0}5|KCpyMYoFQLpYo+YU)U$@5%*L)%G5zUh^eCFLhroftXL0zU|wEwAb5 z@ZynWb)>pFq>ZRoNjs$orTq#Cm~>{vX}EB8L`rqOK!Q&8kxS^gMjZ(%9S+S0l9hQ! z+S2FRW?D_t!3LK$XyPJaq79JrSsEpZkdgI6$n>wJ&anocB9fzJagjQ>L<*i=M-hez zD&yn^4@+|&pfEL$^*hF?dE%zd50^Cs8q8F)(`<{fW&+Sn-6$5FFTJgdCeSs+&}}#0 zfO4^nM!AkiBmqt3C6}M0Z2b|kOYzXM|%2C2g|>;+iP(oIgdOYr>F3@O%-FR45T$Fko7$xVHIz3?fQ z9Q^V<*t|Z2h3PpIvrF(Qu4M8WiMY5_Kz=5V#hDy#+qE4B?%anuMOG{kqcy!SNJ1Bd zZZhm#Mnlr?`hM%?AWU3MSK4&`8C*Arvu~ckLtniY&mBL3Kfm}7%+D;Cdx>7|I5!UU z$MO29 zSFv~3I6h7;nVgzNez73czQ4B{>j%2iG>c1r6EQ1_2f~QsHkbQ%YjxO@ zw;T$`2o3I0T4$^#mCCr!rS_wvvlBPnvKJR7-p9rB=P}&ViB0{z5&*S&jq*il+A5#ub6vK;ESa zYGV@=Y0V-jd5m2|#>sPHl)v1W2VSVBpw&sVzr{etq8d#tXA~U>xC7{Qjc;Ywd zkx{&M>J))m5%lzcgJXMJ2!&K6s0J-?V z1m1Y-98@y({S_MGGt!cd-##@t`F!xZXK>}ts}WBmmc`usp+KuP=FEGhU5AD*=NKc_ zAe`fAFEDjW1VQ?noB9YZoj8e&*X(f*9y%l;g5@ZiDE`h|txPM&+)glm|CnmkJ% z9O_YN)5ln0;JKb08<#UhcpslAxS}%Z z$D+3VMFKToS^CmnzD#WREGs_|1~0tw+k0xuFkL=fw$R9TAB zbg_Y0P$3i%jerzES82K|sC1N0z5wd7=UZRDAK&}WoRjn1^UTaOb6+#}oO36MuFhLw zQtDC=2n1$tM{pB7F^gXbF$hEmAzCjl5Y#{$FVLMq1;dCe0AfjDkN|{zAkhzS1BjIH z;CcWra6U!z@B+P@98JiKKooJ&2E`6!3cMi@yg8dmBnJQ>f&}={f=rRaI@8&iR>zi2xuF0Eq~8AU!C=gl&pk!Zi`B7hj{1 zh$Ry+z!bThl$Vn$!kWPX5QZo$3ad}XY{Vgqa44J+b|cnM4}ryCHls0{&^U~~!DbVT zk%^%J;`0Y-CWXMWC{zAh$F<5?R17l-jGzN>tV)X?U z`XS*#Ad#&f6tZ>^Vi|(~gpgS@CP-rhAr>)-Bt|G`ibM+Zh|k~@@>gtTD2u*?Od+EI zIuHm1fgxxElmU9By?_Zfnx8)y^3^RU9j?vP3XiRKT{-Ip8`;cp>z;wR#o)53*jMX za|$CYB!GrTFP4u^`y6FS`Vt0@{(BOOIgwca5oEAdB;^hS|2-+Q|5K1JZc9N{qAo@O zX&@a~5#vinpS$lH8%KY06My01|Eij2h~RvC0Ia}}|F1~j#Ky1eBNOXo4y_r2b43R2 zgQ$$9WAk?#nk9v0!v={I)|X@TzxCEMF#OwEH^C%YYN&6ZyOJ3cLEA4Mh<^YFS>FQp z0O&gzEWu3rM_-#jBJDrvJy~D0O9fl*z!d{UP>tnxfCEL#J?>53ji)tZBv8;K{_}c{2RtQ(U@4$M`$`r?85J>dKFe6?_~^ zGK)xpsK3>EB(=$Bym`HL*6_jOBykCdh+NH`;*d_D#zwjdjG+tZ3hPcMy=`{zmNb=v zVN^>n2GR5NH6qVPtq*0yym{<*jia`Jn1%M;ALhWL-i^gRUAX$Ry7DKPasEuC%ri&6 z+R@q$`5<6wOguLxGc9Jm_22P)6{`&~WfQ8W*mIdNqci6=$-($Jze1a7%^zd}_VG!N zqCTGK`6H#`)P^Zou)?br$K6$bSR9b4dB}HkHLpGfCE5DYv5?oZ_3DUd=EKz}$N{&$ zH^Uq5`aT<5Wq)j--6g*0G4xm=_i>Fxt*a-(k3TERMchi0#v)^tFUl#&(h)pfQ{xjJ zQxz`jYj8@#QLW~^%qa}S!q#%Kh=N72fP)nvp2%kHhbM)<;8-smZCg zZAKpRXJ^E&;$K18tSVQbb@0=#63-5CMN~4gU(Jp2ghW-&$77LsKDM@Ta6P>Fe5|&E zFQKw}%sd}(OeI3~9W2wm)4B&TS0&iQ2`;l%UfBW3u5HgIQusA5M!mqaM9a2nDA zO}(6^H4Qzx{!$0x3eJyA@zSemz^LlJ$dJAsqd#-CPI^?!u4}SFA}d#WZEC2a++iPr zx7xjRn=7k|2};_0*cAL_P!zRrVM4R^g@m+)=sFzlqoa(Oc{D0~I;lqu1J{0$DzU5b zC%r42gA2mJrcJ-KS3;GI5~5X}NL@3iiTy27f+y1$Jv}}&1u2vcPFbEVydn=Bkwg!kH+>r;v__#R1@dtNX00h z>P?(XpgfLCcI58Qu~!({U&!TOQyTMS4sy-K^hI^pYFtPt?7A8Zr`>hoEoGNbH00h+ zCx1mt_`J@GKs~L%GhT8zqDk4}+g;YEqhnLnuoUD)Tt4;JYTip|j#ZWCUg))yduJm@ z7(q<*YO=5Qw#8bV-Zhx6WAuA#&FZcX4OG7yB^es6hhy~8raO7Jy6#Or;QZwKF=J%s zqbqKvq2nWi3 zEn2*EXj91^WB+Y(Ink}Wq578OYXz};TW`+0$jwFVE-UPt%RNoM=1;}%eYj<$Yd-V9 zwfON=TWaa+BO7wD8_Q@BJe|F|EoM_)Y8mTPUZo`xbFcbL9Qaw&!y`AhUN#&yj?&z) zy;Evrco6NHu9r~ScJg>j|2TtCe#AcW!`6|utrGEYW@6H6F+G)WD6LVt&^r`;piK9T z_FXCb)gH6);Z@+gLjb~RR`ROXe8<9Kt!iFwSmtpu?m@un(AmP2BBc-E6Mf_UD(jiD4Jk+bu_-xX z2g(!PdHI8}V$mV5yH@EXrG2=qtxqS5K7Si4(eWgIw|s<)q?ugC*cz%*Qh7urEF#(;Cz1JRK&y}R3{|eUf;3As?Ym2Qm z#U}My$;HLy>$e6eUr$1{I*BGB4j4Kv%zw9oM!Eba$QM(p{cC($DDeM8b{PwKjuwCQsL z`E}&=96GM-)^w?qkwZCfhcs4dTe1Gx#;&luRg(pJl7ukru7HB>oB7;?-;?dcHsjvA zh+j0TW$%C;6Sbe*^YPLE4tEOE>qo`Dka;+MVR)w$Rz>=e~k#%gQZn1*Mz`61PYo1%{0#|I^>_Fuez zUd2MGJfYJ*Sf%W#ypZk2wdgbwj2t&uya(~ z*(b2b;_{#n-2bM!_RfeD!7nmvAo^?kv?wKTgV&7gxkE#x0fp|D?FO3;Cc$ka(NVYM z@=jo7+mc5_8O$;IVV7CupDg2EIHX1oUCkI6=`nT?i@k7gZ)Rcv#L+&vhD##(5b#!dLU-jp(kL0>r=Ph(h_YLMeZ0@z*5HEKc)_vcz-nA|7 zd>&Tead|=;OcT=x$(aa6oV}`^vu-&-t+cmVDg#DzGi1*8-*EB`u&n;+V>Ad!bSY;w_Oqs-gPbh z<=tS|#SiV#r(DYXR8RfFb9?MPCr1S&wDQ`mR7fY%ta0rHdZi~tneUs>7Q8%}u}#)- zzzO+4g2wNE+;ZE&N|%D}y?y@qiSs-7e*2Xj$6fbYpE=?x?'; for(var i in data) { var p = data[i]; - html += "("+p.refs.length+":"+p.comments+":"+p.ups+") "+p.from.name+": " + p.text + ' .js
    '; + p.refs.forEach(function(ref){ html += icon(ref); }); + var resp = p.comments + p.ups; + html += ""+p.from.name+": " + p.text + ''; + html += ' .js'; + if(resp > 0) html += " "; + html += '
    '; } $("#test").append(html); }); } + +function loadResp(id) +{ + $.getJSON(baseUrl + '/Me/timeline/getResponses',{item:id}, function(data) { + $("#"+id).html(""); + if(!data || !data.length) return; + var ups = " ups: "; + var coms = ""; + for(var i in data) + { + var r = data[i]; + if(r.type == "up") + { + ups += icon(r.ref); + ups += " "+r.from.name+", "; + } + if(r.type == "comment") + { + coms += "
    " + coms += icon(r.ref); + coms += " "+r.from.name+": "; + coms += r.text; + } + } + $("#"+id).append(ups+coms); + }); +} + +function icon(ref) +{ + var start = ref.indexOf('//')+2; + var net = ref.substr(start,ref.indexOf('/',start+2)-start); + if(net == "links") return ""; + return ""; + +} + +function ago(at) +{ + var tip = ''; + var timeDiff = Date.now() - at; + if (timeDiff < 60000) { + tip += 'last updated less than a minute ago'; + } else if (timeDiff < 3600000) { + tip += 'last updated ' + Math.floor(timeDiff / 60000) + ' minutes ago'; + } else if (timeDiff < 43200000) { + tip += 'last updated over an hour ago'; + } else if (timeDiff < 43800000) { + tip += 'last updated ' + Math.floor(timeDiff / 3600000) + ' hours ago'; + } else { + var d = new Date; + d.setTime(at); + tip += 'last updated ' + d.toString(); + } + return tip; +} \ No newline at end of file From d0ccd95bad4b01dd979983cb25b85b91fe5aed72 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Wed, 28 Dec 2011 13:57:15 -0600 Subject: [PATCH 30/40] more facebook fuzzies --- Collections/Timeline/dataIn.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Collections/Timeline/dataIn.js b/Collections/Timeline/dataIn.js index a94e75c74..e1ac96a1b 100644 --- a/Collections/Timeline/dataIn.js +++ b/Collections/Timeline/dataIn.js @@ -333,8 +333,9 @@ function itemFacebook(item, post) } if(post.name) item.title = post.name; if(post.message) item.text = post.message; - if(!post.message && post.caption) item.text = post.caption; - if(!post.message && !post.caption && post.story) item.text = post.story; + if(!item.text && post.description) item.text = post.description; + if(!item.text && post.caption) item.text = post.caption; + if(!item.text && post.story) item.text = post.story; // should we only add a text key if we can detect it's a tweet? we're de-duping any fb post essentially if(item.text) { @@ -429,7 +430,7 @@ function itemInstagram(item, pic) { item.pri = 4; // the source item.first = item.last = pic.created_time * 1000; - if(pic.caption && pic.caption.length > 0) item.text = pic.caption.text; + if(pic.caption && pic.caption.text.length > 0) item.text = pic.caption.text; if(pic.user) { item.from.id = 'contact://instagram/#'+pic.user.id; From 1704fd5aec85ac506e47cbfb89dbe347361c7b0b Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Wed, 28 Dec 2011 14:37:25 -0600 Subject: [PATCH 31/40] adding event generation support --- Collections/Timeline/dataStore.js | 10 ++++++++-- Collections/Timeline/timeline.js | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Collections/Timeline/dataStore.js b/Collections/Timeline/dataStore.js index 37b177948..3a4a502a0 100644 --- a/Collections/Timeline/dataStore.js +++ b/Collections/Timeline/dataStore.js @@ -7,14 +7,16 @@ * */ var logger = require(__dirname + "/../../Common/node/logger"); +var lutil = require(__dirname + "/../../Common/node/lutil"); var crypto = require("crypto"); var async = require('async'); var lmongoutil = require("lmongoutil"); -var itemCol, respCol; +var itemCol, respCol, locker; -exports.init = function(iCollection, rCollection) { +exports.init = function(iCollection, rCollection, l) { + locker = l; itemCol = iCollection; itemCol.ensureIndex({"id":1},{unique:true, background:true},function() {}); itemCol.ensureIndex({"keys":1},{background:true},function() {}); @@ -100,17 +102,20 @@ function findWrap(a,b,c,cbEach,cbDone){ // insert new (fully normalized) item, generate the id here and now exports.addItem = function(item, callback) { + var etype = "update"; if(!item.id) { // first time an item comes in, make a unique id for it var hash = crypto.createHash('md5'); for(var i in item.keys) hash.update(i); item.id = hash.digest('hex'); + etype = "new"; } var responses = item.responses; if(responses) responses.forEach(function(r){ r.item = item.id; }); delete item.responses; // store responses in their own table delete item._id; // mongo is miss pissypants itemCol.findAndModify({"id":item.id}, [['_id','asc']], {$set:item}, {safe:true, upsert:true, new: true}, function(err, doc){ + if(locker) locker.ievent(lutil.idrNew("item","timeline",doc.id),doc,etype); // let happen independently if(err || !responses) return callback(err, doc); async.forEach(responses, exports.addResponse, function(err){callback(err, doc);}); // orig caller wants saved item back }); @@ -134,6 +139,7 @@ exports.delItem = function(id, callback) { if(!id || id.length < 10) return callback("no or invalid id to del: "+id); itemCol.remove({id:id}, function(err){ if(err) return callback(err); + if(locker) locker.ievent(lutil.idrNew("item","timeline",doc.id),doc,"delete"); // let happen independently respCol.remove({item:id}, callback); }); } diff --git a/Collections/Timeline/timeline.js b/Collections/Timeline/timeline.js index a851a01dc..eae0d6e15 100644 --- a/Collections/Timeline/timeline.js +++ b/Collections/Timeline/timeline.js @@ -155,7 +155,7 @@ process.stdin.on('data', function(data) { locker.connectToMongo(function(mongo) { // initialize all our libs - dataStore.init(mongo.collections.item,mongo.collections.response); + dataStore.init(mongo.collections.item,mongo.collections.response, locker); dataIn.init(locker, dataStore, function(){ sync.init(locker, dataStore, dataIn, function(){ app.listen(lockerInfo.port, 'localhost', function() { From 6888fa5d4922b870ab1f458264472dc4fed47c3b Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Wed, 28 Dec 2011 14:47:57 -0600 Subject: [PATCH 32/40] letting things happen more fluidly, and enabling deep update again --- Collections/Timeline/sync.js | 3 ++- Collections/Timeline/timeline.js | 4 +--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Collections/Timeline/sync.js b/Collections/Timeline/sync.js index ba232d864..33839ce60 100644 --- a/Collections/Timeline/sync.js +++ b/Collections/Timeline/sync.js @@ -37,6 +37,7 @@ exports.update = function(locker, type, callback) { if(type) return logger.debug("done with update for "+type); // process links too var tot = 0; + // this is experimental and not sure if it's useful yet lutil.streamFromUrl(locker.lockerBase+'/Me/links/?full=true&limit=100&stream=true', function(link, cb){ tot++; dataIn.processLink({data:link}, cb); @@ -53,7 +54,7 @@ exports.update = function(locker, type, callback) { function getData(type, svcId, callback) { var subtype = type.substr(0, type.indexOf('/')); - var lurl = locker.lockerBase + '/Me/' + svcId + '/getCurrent/' + subtype + "?limit=100&sort=_id&order=-1&stream=true"; + var lurl = locker.lockerBase + '/Me/' + svcId + '/getCurrent/' + subtype + "?all=true&stream=true"; var tot = 0; lutil.streamFromUrl(lurl, function(a, cb){ tot++; diff --git a/Collections/Timeline/timeline.js b/Collections/Timeline/timeline.js index eae0d6e15..bc301e41d 100644 --- a/Collections/Timeline/timeline.js +++ b/Collections/Timeline/timeline.js @@ -117,12 +117,10 @@ app.post('/events', function(req, res) { res.end('bad data'); return; } - + res.send('ok'); // handle asyncadilly dataIn.processEvent(req.body, function(err){ if(err) logger.error(err); - if(err) return res.send(err, 500); - res.send('ok'); }); }); From 005eb7070473fb135f2b6106123aad6e8dd5bf54 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Thu, 29 Dec 2011 01:21:26 -0600 Subject: [PATCH 33/40] safely serialize things, if stuff truly happens in parallel the dedup may not catch --- Collections/Timeline/dataIn.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Collections/Timeline/dataIn.js b/Collections/Timeline/dataIn.js index e1ac96a1b..c7622dde3 100644 --- a/Collections/Timeline/dataIn.js +++ b/Collections/Timeline/dataIn.js @@ -73,6 +73,7 @@ exports.processEvent = function(event, callback) // some data is incomplete, stupid but WTF do you do! var profiles = {}; +var mbQ = async.queue(masterBlaster, 1); // only process in series function masterMaster(idr, data, callback) { if(idr.protocol == 'checkin:' && idr.pathname.indexOf('checkin') != -1 && idr.query.id) @@ -81,7 +82,7 @@ function masterMaster(idr, data, callback) if(profiles[svcId]) { data.user = profiles[svcId]; - return masterBlaster(idr, data, callback); + return mbQ.push({idr:idr, data:data}, callback); } var lurl = locker.lockerBase + '/Me/'+svcId+'/getCurrent/profile'; request.get({uri:lurl, json:true}, function(err, resp, arr){ @@ -92,17 +93,19 @@ function masterMaster(idr, data, callback) } logger.debug("caching profile for "+svcId); data.user = profiles[svcId] = arr[0]; - return masterBlaster(idr, data, callback); + return mbQ.push({idr:idr, data:data}, callback); }); return; } - masterBlaster(idr, data, callback); + mbQ.push({idr:idr, data:data}, callback); } exports.masterMaster = masterMaster; // figure out what to do with any data -function masterBlaster(idr, data, callback) +function masterBlaster(arg, callback) { + var idr = arg.idr; + var data = arg.data; if(typeof data != 'object') return callback("missing or bad data"); // logger.debug("MM\t"+url.format(idr)); var ref = url.format(idr); @@ -344,6 +347,7 @@ function itemFacebook(item, post) item.keys['text:'+hash.digest('hex')] = item.ref; } if(post.link) keyUrl(item, post.link); + if(!item.text && post.type == "photo") item.text = "New Photo"; // process responses! if(post.comments && post.comments.data) From 0828ea340670b874db5714bd6464f45332ae0ce7 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Thu, 19 Jan 2012 01:30:28 -0600 Subject: [PATCH 34/40] bring up to locker upstream current, fix logger references and generic pathing --- Collections/Timeline/dataIn.js | 3 ++- Collections/Timeline/dataStore.js | 5 ++-- Collections/Timeline/package.json | 33 ++++++++++++++++++++++++ Collections/Timeline/sync.js | 3 ++- Collections/Timeline/timeline.collection | 16 ------------ Collections/Timeline/timeline.js | 9 ++++--- 6 files changed, 45 insertions(+), 24 deletions(-) create mode 100644 Collections/Timeline/package.json delete mode 100644 Collections/Timeline/timeline.collection diff --git a/Collections/Timeline/dataIn.js b/Collections/Timeline/dataIn.js index c7622dde3..58bcf14d9 100644 --- a/Collections/Timeline/dataIn.js +++ b/Collections/Timeline/dataIn.js @@ -1,6 +1,6 @@ var request = require('request'); var async = require('async'); -var logger = require(__dirname + "/../../Common/node/logger"); +var logger; var lutil = require('lutil'); var url = require('url'); var crypto = require("crypto"); @@ -12,6 +12,7 @@ var dataStore, locker; exports.init = function(l, dStore, callback){ dataStore = dStore; locker = l; + logger = l.logger; callback(); } diff --git a/Collections/Timeline/dataStore.js b/Collections/Timeline/dataStore.js index 3a4a502a0..32bf13392 100644 --- a/Collections/Timeline/dataStore.js +++ b/Collections/Timeline/dataStore.js @@ -6,8 +6,8 @@ * Please see the LICENSE file for more information. * */ -var logger = require(__dirname + "/../../Common/node/logger"); -var lutil = require(__dirname + "/../../Common/node/lutil"); +var logger; +var lutil = require("lutil"); var crypto = require("crypto"); var async = require('async'); var lmongoutil = require("lmongoutil"); @@ -17,6 +17,7 @@ var itemCol, respCol, locker; exports.init = function(iCollection, rCollection, l) { locker = l; + logger = l.logger; itemCol = iCollection; itemCol.ensureIndex({"id":1},{unique:true, background:true},function() {}); itemCol.ensureIndex({"keys":1},{background:true},function() {}); diff --git a/Collections/Timeline/package.json b/Collections/Timeline/package.json new file mode 100644 index 000000000..184996ebe --- /dev/null +++ b/Collections/Timeline/package.json @@ -0,0 +1,33 @@ +{ + "author": "Singly ", + "name": "timeline", + "description": "Timeline Collection", + "version": "0.0.1", + "repository": { + "title": "Timeline", + "run": "node timeline.js", + "author": "nerds", + "update": "auto", + "github": "https://github.com/LockerProject/Locker", + "type": "collection", + "handle": "timeline", + "provides": [ + "item" + ], + "mongoCollections": [ + "item", "response" + ], + "events":[["checkin/foursquare","/events"] + ,["tweet/twitter","/events"] + ,["photo/instagram","/events"] + ,["related/twitter","/events"] + ,["post/facebook","/events"] + ,["link","/events"] + ] + }, + "engines": { + "node": ">=0.4.9" + }, + "dependencies": {}, + "devDependencies": {} +} diff --git a/Collections/Timeline/sync.js b/Collections/Timeline/sync.js index 33839ce60..c3b781474 100644 --- a/Collections/Timeline/sync.js +++ b/Collections/Timeline/sync.js @@ -1,5 +1,5 @@ var async = require('async'); -var logger = require(__dirname + "/../../Common/node/logger"); +var logger; var lutil = require('lutil'); var url = require('url'); @@ -10,6 +10,7 @@ exports.init = function(l, dStore, dIn, callback){ dataStore = dStore; dataIn = dIn; locker = l; + logger = l.logger; callback(); } diff --git a/Collections/Timeline/timeline.collection b/Collections/Timeline/timeline.collection deleted file mode 100644 index 53115be9a..000000000 --- a/Collections/Timeline/timeline.collection +++ /dev/null @@ -1,16 +0,0 @@ -{ - "title":"Timeline", - "desc":"A collection of socially shared items from various networks.", - "run":"node timeline.js", - "status":"stable", - "handle":"timeline", - "provides":["item"], - "mongoCollections": ["item","response"], - "events":[["checkin/foursquare","/events"] - ,["tweet/twitter","/events"] - ,["photo/instagram","/events"] - ,["related/twitter","/events"] - ,["post/facebook","/events"] - ,["link","/events"] - ] -} diff --git a/Collections/Timeline/timeline.js b/Collections/Timeline/timeline.js index bc301e41d..094030989 100644 --- a/Collections/Timeline/timeline.js +++ b/Collections/Timeline/timeline.js @@ -12,10 +12,9 @@ var fs = require('fs'), url = require('url'), request = require('request'), - locker = require('../../Common/node/locker.js'), - lconfig = require('../../Common/node/lconfig'); -lconfig.load('../../Config/config.json'); -var logger = require('logger'); + locker = require('locker.js'), + lconfig = require('lconfig'); +var logger; var async = require("async"); var url = require("url"); @@ -150,6 +149,8 @@ process.stdin.on('data', function(data) { process.exit(1); } process.chdir(lockerInfo.workingDirectory); + lconfig.load('../../Config/config.json'); + locker.logger = logger = require('logger'); locker.connectToMongo(function(mongo) { // initialize all our libs From 46864fc57d4ac3ad3e9a20e051796a9753d2f9ed Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Thu, 19 Jan 2012 01:51:17 -0600 Subject: [PATCH 35/40] updating test app and removing synclet depreciated code --- Apps/HelloTimeline/manifest.app | 8 -------- Apps/HelloTimeline/package.json | 30 ++++++++++++++++++++++++++++++ Apps/prolific_posters | 1 + Common/node/ldatastore.js | 6 ++---- 4 files changed, 33 insertions(+), 12 deletions(-) delete mode 100644 Apps/HelloTimeline/manifest.app create mode 100644 Apps/HelloTimeline/package.json create mode 160000 Apps/prolific_posters diff --git a/Apps/HelloTimeline/manifest.app b/Apps/HelloTimeline/manifest.app deleted file mode 100644 index 7f8f323c1..000000000 --- a/Apps/HelloTimeline/manifest.app +++ /dev/null @@ -1,8 +0,0 @@ -{ - "title":"Hello Timeline", - "desc":"A simple way to view your timeline.", - "status":"stable", - "static":"true", - "uses":["facebook", "twitter", "foursquare", "instagram"], - "handle":"hellotimeline" -} \ No newline at end of file diff --git a/Apps/HelloTimeline/package.json b/Apps/HelloTimeline/package.json new file mode 100644 index 000000000..205ee6a6c --- /dev/null +++ b/Apps/HelloTimeline/package.json @@ -0,0 +1,30 @@ +{ + "author": "Singly, Inc. ", + "name": "hellotimeline", + "description": "A simple way to view your timeline.", + "version": "0.0.1", + "homepage": "https://github.com/LockerProject/locker", + "repository": { + "type": "app", + "title": "Hello Timeline", + "status": "stable", + "static": true, + "handle": "hellotimeline", + "update": true, + "uses": { + "services": [ + "facebook", + "foursquare", + "instagram", + "twitter" + ] + }, + "github": "https://github.com/LockerProject/locker", + "url": "" + }, + "engines": { + "node": ">=0.4.9" + }, + "dependencies": {}, + "devDependencies": {} +} diff --git a/Apps/prolific_posters b/Apps/prolific_posters new file mode 160000 index 000000000..591b54d68 --- /dev/null +++ b/Apps/prolific_posters @@ -0,0 +1 @@ +Subproject commit 591b54d68061810e45b8a0604cc0abcc42463677 diff --git a/Common/node/ldatastore.js b/Common/node/ldatastore.js index 92a40c1a0..d5e6f8e61 100644 --- a/Common/node/ldatastore.js +++ b/Common/node/ldatastore.js @@ -16,12 +16,10 @@ var IJOD = require('ijod').IJOD , colls = {} , mongoIDs = {} ; -var syncManager = require('lsyncmanager'); -var synclets; +var svcMan = require('lservicemanager'); exports.init = function(owner, callback) { if (mongo[owner]) return callback(); - synclets = syncManager.synclets(); lmongo.init(owner, [], function(_mongo) { mongo[owner] = _mongo; colls[owner] = mongo[owner].collections[owner]; @@ -116,7 +114,7 @@ exports.getCurrent = function(owner, type, id, callback) { var parts = type.split("_"); var idname = "id"; // BEWARE THE mongoId s appendage WEIRDNESS! #techdebt - if(parts.length == 2 && synclets && synclets[parts[0]] && synclets[parts[0]].mongoId && synclets[parts[0]].mongoId[parts[1]+'s']) idname = synclets[parts[0]].mongoId[parts[1]+'s']; + if(parts.length == 2 && svcMan.map(parts[0]) && svcMan.map(parts[0]).mongoId && svcMan.map(parts[0]).mongoId[parts[1]+'s']) idname = svcMan.map(parts[0]).mongoId[parts[1]+'s']; var id2 = {}; id2[idname] = id; or.push(id2); From aa565f5f721b673e6a825c5d11028d129621253b Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Tue, 31 Jan 2012 01:08:10 -0600 Subject: [PATCH 36/40] basic work load scheduling management signals by core --- Common/node/lconfig.js | 3 +++ Common/node/levents.js | 2 +- Common/node/lservicemanager.js | 20 ++++++++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/Common/node/lconfig.js b/Common/node/lconfig.js index 16a6c5984..e63820e61 100644 --- a/Common/node/lconfig.js +++ b/Common/node/lconfig.js @@ -10,6 +10,7 @@ //just a place for lockerd.js to populate config info var fs = require('fs'); var path = require('path'); +var os = require('os'); exports.load = function(filepath) { var config = {}; @@ -63,6 +64,8 @@ exports.load = function(filepath) { exports.ui = config.ui || 'dashboardv3'; exports.quiesce = config.quiesce || 650000; exports.dashboard = config.dashboard; + exports.workWarn = config.workWarn || os.cpus().length; + exports.workStop = config.workStop || os.cpus().length * 3; } function setBase() { diff --git a/Common/node/levents.js b/Common/node/levents.js index 6c288a11a..5395c2b98 100644 --- a/Common/node/levents.js +++ b/Common/node/levents.js @@ -85,7 +85,7 @@ exports.fireEvent = function(idr, action, obj) { } exports.displayListeners = function(type) { - return eventListeners[type]; + return eventListeners[type] || []; } function findListenerPosition(type, id, cb) { diff --git a/Common/node/lservicemanager.js b/Common/node/lservicemanager.js index ab6d82e20..15729947d 100644 --- a/Common/node/lservicemanager.js +++ b/Common/node/lservicemanager.js @@ -9,6 +9,7 @@ var fs = require("fs"); var path = require("path"); +var os = require("os"); var lconfig = require("lconfig"); var crypto = require("crypto"); var util = require("util"); @@ -454,3 +455,22 @@ function checkForShutdown() { shuttingDown(); shuttingDown = null; } + +// create signal events for services to listen to +var workLast = null; +var workSig = ""; // need to re-signal if listeners change +function workCheck() +{ + var newSig = levents.displayListeners("work/me").join(" "); + if(newSig != workSig) workLast = null; + workSig = newSig; + if(newSig == "") return; // no listeners! + var work = "start"; + var load = os.loadavg(); + if(load[0] > lconfig.workWarn) work = "warn"; + if(load[0] > lconfig.workStop) work = "stop"; + if(work == workLast) return; // no changes! + workLast = work; + levents.fireEvent("work://me/#"+work,"new",{}); +} +setInterval(workCheck, 10000); // 10s granularity \ No newline at end of file From da311660f7c781d0abd06fc73f785e019a582e66 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Thu, 2 Feb 2012 01:22:38 -0600 Subject: [PATCH 37/40] in-progress reorg for dynamic updating --- Apps/HelloTimeline/index.html | 1 + Apps/HelloTimeline/package.json | 1 + Apps/HelloTimeline/script.js | 13 +++++++++++++ Collections/Timeline/dataStore.js | 3 +-- Collections/Timeline/sync.js | 17 ++++++++++++++++- Collections/Timeline/timeline.js | 26 ++++++++------------------ 6 files changed, 40 insertions(+), 21 deletions(-) diff --git a/Apps/HelloTimeline/index.html b/Apps/HelloTimeline/index.html index 73cdbcc4d..d096e6364 100644 --- a/Apps/HelloTimeline/index.html +++ b/Apps/HelloTimeline/index.html @@ -22,6 +22,7 @@ +

    Forever Timeless

      diff --git a/Apps/HelloTimeline/package.json b/Apps/HelloTimeline/package.json index 205ee6a6c..c6a5a0d27 100644 --- a/Apps/HelloTimeline/package.json +++ b/Apps/HelloTimeline/package.json @@ -9,6 +9,7 @@ "title": "Hello Timeline", "status": "stable", "static": true, + "hidden": true, "handle": "hellotimeline", "update": true, "uses": { diff --git a/Apps/HelloTimeline/script.js b/Apps/HelloTimeline/script.js index 48a8e3ac2..f9c9d5eba 100755 --- a/Apps/HelloTimeline/script.js +++ b/Apps/HelloTimeline/script.js @@ -11,9 +11,22 @@ $(function() { $("#moar").click( function(){ offset += 50; loadStuff(); + loadStatus(); }); }); +var max = 20; +function loadStatus() +{ + $.getJSON(baseUrl + '/Me/timeline/state',{}, function(data) { + if(!data) return $("#status").text("timeline failed :("); + if(data.ready == 1) return $("#status").text(""); + $("#status").text("timeline is indexing..."); + if(max-- <= 0) return $("#status").text("plz reload"); + window.setTimeout(loadStatus, 10000); // poll for a bit + }); +} + function loadStuff(){ $.getJSON(baseUrl + '/Me/timeline/',{limit:50, offset:offset}, function(data) { if(!data || !data.length) return; diff --git a/Collections/Timeline/dataStore.js b/Collections/Timeline/dataStore.js index 32bf13392..5a1a3a7a0 100644 --- a/Collections/Timeline/dataStore.js +++ b/Collections/Timeline/dataStore.js @@ -26,8 +26,7 @@ exports.init = function(iCollection, rCollection, l) { respCol.ensureIndex({"item":1},{background:true},function() {}); } -exports.clear = function(flag, callback) { - if(flag) return callback(); +exports.clear = function(callback) { itemCol.drop(function(){respCol.drop(callback)}); } diff --git a/Collections/Timeline/sync.js b/Collections/Timeline/sync.js index c3b781474..0aac8990b 100644 --- a/Collections/Timeline/sync.js +++ b/Collections/Timeline/sync.js @@ -12,10 +12,25 @@ exports.init = function(l, dStore, dIn, callback){ locker = l; logger = l.logger; callback(); + var js; + try { js = JSON.parse(fs.readFileSync('state.json')); } catch(E) {} + if(js && js.ready == 1) return; // already sync'd + if(js) exports.sync(l, js, function(err){ if(err) logger.error(err); }); + // starting from scratch, make sure clear, set initial list + dataStore.clear(function(){ + js = {types:[ + "facebook/getCurrent/home", + "twitter/getCurrent/tweets", "twitter/getCurrent/timeline", "twitter/getCurrent/mentions", "twitter/getCurrent/related", + "foursquare/getCurrent/recents", "foursquare/getCurrent/checkin", + "instagram/getCurrent/photo", "instagram/getCurrent/feed", + "links/" + ]}; + exports.sync(l, js, function(err){ if(err) logger.error(err); }); + }); } // manually walk and reindex all possible link sources -exports.update = function(locker, type, callback) { +exports.sync = function(js, callback) { dataStore.clear(type, function(){ callback(); var types = (type) ? [type] : ['home/facebook', 'tweets/twitter', 'checkin/foursquare', 'feed/instagram']; diff --git a/Collections/Timeline/timeline.js b/Collections/Timeline/timeline.js index 094030989..5683330d1 100644 --- a/Collections/Timeline/timeline.js +++ b/Collections/Timeline/timeline.js @@ -65,17 +65,14 @@ app.get('/', function(req, res) { app.get('/state', function(req, res) { dataStore.getTotalItems(function(err, countInfo) { if(err) return res.send(err, 500); - dataStore.getLastObjectID(function(err, lastObject) { - if(err) return res.send(err, 500); - var objId = "000000000000000000000000"; - if (lastObject) objId = lastObject._id.toHexString(); - var updated = new Date().getTime(); - try { - var js = JSON.parse(fs.readFileSync('state.json')); - if(js && js.updated) updated = js.updated; - } catch(E) {} - res.send({ready:1, count:countInfo, updated:updated, lastId:objId}); - }); + var updated = new Date().getTime(); + var js; + try { + js = JSON.parse(fs.readFileSync('state.json')); + } catch(E) {} + if(!js) js = {ready:0}; + js.count = countInfo; + res.send(js); }); }); @@ -102,13 +99,6 @@ app.get('/ref', function(req, res) { }); }); -app.get('/update', function(req, res) { - sync.update(locker, req.query.type, function(){ - res.writeHead(200); - res.end('Extra mince!'); - }); -}); - app.post('/events', function(req, res) { if (!req.body.idr || !req.body.data){ logger.error('5 HUNDO bad data:',JSON.stringify(req.body)); From 92c035a1bcb1b80728ad3206d55937beb7832456 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Thu, 2 Feb 2012 03:44:41 -0600 Subject: [PATCH 38/40] background syncing seems to be working now --- Collections/Timeline/sync.js | 138 ++++++++++++++++++------------- Collections/Timeline/timeline.js | 16 ++++ Common/node/lservicemanager.js | 2 +- 3 files changed, 96 insertions(+), 60 deletions(-) diff --git a/Collections/Timeline/sync.js b/Collections/Timeline/sync.js index 0aac8990b..5e6b5dfc5 100644 --- a/Collections/Timeline/sync.js +++ b/Collections/Timeline/sync.js @@ -3,7 +3,7 @@ var logger; var lutil = require('lutil'); var url = require('url'); -var dataStore, dataIn, locker; +var dataStore, dataIn, locker, state; // internally we need these for happy fun stuff exports.init = function(l, dStore, dIn, callback){ @@ -12,76 +12,96 @@ exports.init = function(l, dStore, dIn, callback){ locker = l; logger = l.logger; callback(); - var js; - try { js = JSON.parse(fs.readFileSync('state.json')); } catch(E) {} - if(js && js.ready == 1) return; // already sync'd - if(js) exports.sync(l, js, function(err){ if(err) logger.error(err); }); + loadState(function(){ + if(state.ready == 1) return; // nothing to do + locker.listen("work/me", "/work"); + exports.sync(); + }); +} + +var stopped; +exports.work = function(type) +{ + stopped = false; + if(type == "start") exports.sync(); + if(type == "stop") stopped = true; + if(type == "warn") state.sleep = 10000; +} + +function loadState(callback) +{ + try { state = JSON.parse(fs.readFileSync('state.json')); } catch(E) {} + if(state) return callback(); // starting from scratch, make sure clear, set initial list + logger.error("syncing from a new state"); dataStore.clear(function(){ - js = {types:[ + state = {sleep:500, types:[ "facebook/getCurrent/home", "twitter/getCurrent/tweets", "twitter/getCurrent/timeline", "twitter/getCurrent/mentions", "twitter/getCurrent/related", "foursquare/getCurrent/recents", "foursquare/getCurrent/checkin", "instagram/getCurrent/photo", "instagram/getCurrent/feed", "links/" ]}; - exports.sync(l, js, function(err){ if(err) logger.error(err); }); - }); -} - -// manually walk and reindex all possible link sources -exports.sync = function(js, callback) { - dataStore.clear(type, function(){ + saveState(); callback(); - var types = (type) ? [type] : ['home/facebook', 'tweets/twitter', 'checkin/foursquare', 'feed/instagram']; - locker.providers(types, function(err, services) { - if (!services) return; - async.forEachSeries(services, function(svc, cb) { - logger.debug("processing "+svc.id); - if(svc.provides.indexOf('home/facebook') >= 0) { - getData("home/facebook", svc.id, cb); - } else if(svc.provides.indexOf('tweets/twitter') >= 0) { - async.forEachSeries(["tweets/twitter", "timeline/twitter", "mentions/twitter", "related/twitter"], function(type, cb2){ getData(type, svc.id, cb2) }, cb); - } else if(svc.provides.indexOf('checkin/foursquare') >= 0) { - async.forEachSeries(["recents/foursquare", "checkin/foursquare"], function(type, cb2){ getData(type, svc.id, cb2) }, cb); - } else if(svc.provides.indexOf('feed/instagram') >= 0) { - async.forEachSeries(["photo/instagram", "feed/instagram"], function(type, cb2){ getData(type, svc.id, cb2) }, cb); - } else { - cb(); - } - }, function(err){ - if(type) return logger.debug("done with update for "+type); - // process links too - var tot = 0; - // this is experimental and not sure if it's useful yet - lutil.streamFromUrl(locker.lockerBase+'/Me/links/?full=true&limit=100&stream=true', function(link, cb){ - tot++; - dataIn.processLink({data:link}, cb); - }, function(err){ - if(err) logger.error(err); - logger.debug(tot+" links processed, done with update"); - }); - }); - }); }); } -// go fetch data from sources to bulk process -function getData(type, svcId, callback) +function saveState() { - var subtype = type.substr(0, type.indexOf('/')); - var lurl = locker.lockerBase + '/Me/' + svcId + '/getCurrent/' + subtype + "?all=true&stream=true"; - var tot = 0; - lutil.streamFromUrl(lurl, function(a, cb){ - tot++; - var idr = getIdr(type, svcId, a); - dataIn.masterMaster(idr, a, cb); - }, function(err){ - logger.debug("processed "+tot+" items from "+lurl); - callback(err); + lutil.atomicWriteFileSync("state.json", JSON.stringify(state)); +} + +// see if there's any syncing work to do in the background, only running one at a time +var syncTimeout; +var running; +exports.sync = function() { + // if any timer for future sync, zap it + if(syncTimeout) clearTimeout(syncTimeout); + syncTimeout = false; + // don't run if told not to + if(stopped) return; + // don't run if we're already running + if(running) return; + running = true; + // load fresh just to be safe + loadState(function(){ + // if nothing to sync, go away + if(!state) return logger.error("invalid state!"); + if(state.ready == 1) return; + // see if we're all done! + if(state.types.length == 0) + { + running = false; + state.ready = 1; + return saveState(); + } + if(!state.current) + { + state.current = {}; + state.current.type = state.types.shift(); + state.current.offset = 0; + } + var cnt = 0; + var lurl = locker.lockerBase + '/Me/' + state.current.type + "?full=true&stream=true&limit=250&offset="+state.current.offset; + logger.error("syncing "+lurl); + lutil.streamFromUrl(lurl, function(a, cb){ + cnt++; + (state.current.type == "link/") ? dataIn.processLink({data:a}, cb) : dataIn.masterMaster(getIdr(state.current.type, a), a, cb); + }, function(err){ + running = false; + if(err) return logger.error("sync failed, stopping: ",err); + // no data n no error, we hit the end, move on! + state.current.offset += 250; + if(cnt == 0) delete state.current; + saveState(); + // come back again darling + syncTimeout = setTimeout(exports.sync, state.sleep); + }) }); } + // generate unique id for any item based on it's event //> u.parse("type://network/context?id=account#46234623456",true); //{ protocol: 'type:', @@ -93,12 +113,12 @@ function getData(type, svcId, callback) // search: '?id=account', // query: { id: 'account' }, // pathname: '/context' } -function getIdr(type, via, data) +function getIdr(type, data) { var r = {slashes:true}; - r.host = type.substr(type.indexOf('/')+1); - r.pathname = type.substr(0, type.indexOf('/')); - r.query = {id: via}; // best proxy of account id right now + r.host = type.substr(0, type.indexOf('/')); + r.pathname = type.substr(type.lastIndexOf('/')+1); + r.query = {id: r.host}; // best proxy of account id right now dataIn.idrHost(r, data); return url.parse(url.format(r),true); // make sure it's consistent } diff --git a/Collections/Timeline/timeline.js b/Collections/Timeline/timeline.js index 5683330d1..6d84e3340 100644 --- a/Collections/Timeline/timeline.js +++ b/Collections/Timeline/timeline.js @@ -89,6 +89,22 @@ app.get('/id/:id', function(req, res) { dataStore.getItem(req.param('id'), function(err, doc) { return (err != null || !doc) ? res.send(err, 500) : res.send(doc); }); }); +// this would only be useful if something changes or removes state.json +app.get('/sync', function(req, res) { + res.send(true); + sync.sync(); // happens async +}); + +// sync.init() optionally listens for work events, funnel back +app.post('/work', function(req, res) { + if (!req.body.idr || !req.body.data){ + logger.error('bad work data: ',JSON.stringify(req.body)); + return res.send('bad data', 500); + } + res.send('ok'); + sync.work(req.body.data.work); +}); + app.get('/ref', function(req, res) { var idr = url.parse(req.query.id); if(!idr || !idr.hash) return res.send("missing or invalid id",500); diff --git a/Common/node/lservicemanager.js b/Common/node/lservicemanager.js index 15729947d..ef15390d7 100644 --- a/Common/node/lservicemanager.js +++ b/Common/node/lservicemanager.js @@ -471,6 +471,6 @@ function workCheck() if(load[0] > lconfig.workStop) work = "stop"; if(work == workLast) return; // no changes! workLast = work; - levents.fireEvent("work://me/#"+work,"new",{}); + levents.fireEvent("work://me/#"+work,"new",{work:work}); } setInterval(workCheck, 10000); // 10s granularity \ No newline at end of file From 8c5f3d32ca570c80f80e3399757a9c26108e006f Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Thu, 2 Feb 2012 07:19:16 -0600 Subject: [PATCH 39/40] better status update in test app, update links to enable querying of just encounters, and some better state handling during sync --- Apps/HelloTimeline/script.js | 5 +++-- Collections/Links/links.js | 26 ++++++++++++++++++++++++++ Collections/Timeline/sync.js | 16 ++++++++++------ 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/Apps/HelloTimeline/script.js b/Apps/HelloTimeline/script.js index f9c9d5eba..457f13c64 100755 --- a/Apps/HelloTimeline/script.js +++ b/Apps/HelloTimeline/script.js @@ -8,10 +8,10 @@ var offset=0; $(function() { // be careful with the limit, some people have large datasets ;) loadStuff(); + loadStatus(); $("#moar").click( function(){ offset += 50; loadStuff(); - loadStatus(); }); }); @@ -21,7 +21,8 @@ function loadStatus() $.getJSON(baseUrl + '/Me/timeline/state',{}, function(data) { if(!data) return $("#status").text("timeline failed :("); if(data.ready == 1) return $("#status").text(""); - $("#status").text("timeline is indexing..."); + var name = (data.current) ? data.current.type + " at "+data.current.offset : ""; + $("#status").text("timeline is indexing "+name); if(max-- <= 0) return $("#status").text("plz reload"); window.setTimeout(loadStatus, 10000); // poll for a bit }); diff --git a/Collections/Links/links.js b/Collections/Links/links.js index 7c8c447f2..d29fed0b3 100644 --- a/Collections/Links/links.js +++ b/Collections/Links/links.js @@ -208,6 +208,32 @@ app.get('/', function(req, res) { }); }); +// get just encounters raw! +// expose way to get raw links and encounters +app.get('/encounters', function(req, res) { + var options = {sort:{"_id":-1}}; + if(!req.query["all"]) options.limit = 20; // default 20 unless all is set + if (req.query.limit) { + options.limit = parseInt(req.query.limit); + } + if (req.query.offset) { + options.offset = parseInt(req.query.offset); + } + if(req.query['stream'] == "true") + { + res.writeHead(200, {'content-type' : 'application/jsonstream'}); + } + var results = []; + dataStore.getEncounters(options, function(item) { + if(req.query['stream'] == "true") return res.write(JSON.stringify(item)+'\n'); + results.push(item); + }, function(err){ + if(err) logger.error(err); + if(req.query['stream'] == "true") return res.end(); + return res.send(results); + }); +}); + // expose way to get the list of encounters from a link id app.get('/encounters/:id', function(req, res) { var encounters = []; diff --git a/Collections/Timeline/sync.js b/Collections/Timeline/sync.js index 5e6b5dfc5..f98d08c57 100644 --- a/Collections/Timeline/sync.js +++ b/Collections/Timeline/sync.js @@ -2,6 +2,7 @@ var async = require('async'); var logger; var lutil = require('lutil'); var url = require('url'); +var fs = require('fs'); var dataStore, dataIn, locker, state; @@ -23,14 +24,17 @@ var stopped; exports.work = function(type) { stopped = false; - if(type == "start") exports.sync(); + if(type == "start") { + state.sleep = 500; + exports.sync(); + } if(type == "stop") stopped = true; if(type == "warn") state.sleep = 10000; } function loadState(callback) { - try { state = JSON.parse(fs.readFileSync('state.json')); } catch(E) {} + try { state = JSON.parse(fs.readFileSync('state.json')); } catch(E) { logger.error("state.json failed ",E); } if(state) return callback(); // starting from scratch, make sure clear, set initial list logger.error("syncing from a new state"); @@ -40,7 +44,7 @@ function loadState(callback) "twitter/getCurrent/tweets", "twitter/getCurrent/timeline", "twitter/getCurrent/mentions", "twitter/getCurrent/related", "foursquare/getCurrent/recents", "foursquare/getCurrent/checkin", "instagram/getCurrent/photo", "instagram/getCurrent/feed", - "links/" + "links/encounters" ]}; saveState(); callback(); @@ -70,7 +74,7 @@ exports.sync = function() { if(!state) return logger.error("invalid state!"); if(state.ready == 1) return; // see if we're all done! - if(state.types.length == 0) + if(state.types.length == 0 && !state.current) { running = false; state.ready = 1; @@ -83,11 +87,11 @@ exports.sync = function() { state.current.offset = 0; } var cnt = 0; - var lurl = locker.lockerBase + '/Me/' + state.current.type + "?full=true&stream=true&limit=250&offset="+state.current.offset; + var lurl = locker.lockerBase + '/Me/' + state.current.type + "?stream=true&limit=250&offset="+state.current.offset; logger.error("syncing "+lurl); lutil.streamFromUrl(lurl, function(a, cb){ cnt++; - (state.current.type == "link/") ? dataIn.processLink({data:a}, cb) : dataIn.masterMaster(getIdr(state.current.type, a), a, cb); + (state.current.type.indexOf("link/") == 0) ? dataIn.processLink({data:a}, cb) : dataIn.masterMaster(getIdr(state.current.type, a), a, cb); }, function(err){ running = false; if(err) return logger.error("sync failed, stopping: ",err); From 860f739c32cbaad48db9e3d5a6283c2440b783a3 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Sat, 4 Feb 2012 14:36:33 -0600 Subject: [PATCH 40/40] starting to migrate to sqlite-cursor since mongo can't handle the timeline --- Collections/Timeline/dataStore.js | 34 +++++++++++++++++++++++++++++-- Collections/Timeline/timeline.js | 12 ++++++----- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/Collections/Timeline/dataStore.js b/Collections/Timeline/dataStore.js index 5a1a3a7a0..59c607113 100644 --- a/Collections/Timeline/dataStore.js +++ b/Collections/Timeline/dataStore.js @@ -11,11 +11,15 @@ var lutil = require("lutil"); var crypto = require("crypto"); var async = require('async'); var lmongoutil = require("lmongoutil"); +var lsql = require('sqlite-cursor'); -var itemCol, respCol, locker; +var itemCol, respCol; +var locker; +var mItem, mResp, mKey; -exports.init = function(iCollection, rCollection, l) { + +exports.init = function(iCollection, rCollection, l, callback) { locker = l; logger = l.logger; itemCol = iCollection; @@ -24,6 +28,32 @@ exports.init = function(iCollection, rCollection, l) { respCol = rCollection; respCol.ensureIndex({"id":1},{unique:true, background:true},function() {}); respCol.ensureIndex({"item":1},{background:true},function() {}); + async.series([ + function(cb) { + lsql.connectToDB("./timeline.sqlite", cb); + }, + function(cb) { + mKey = new lsql.Model("key", { + key:{type:lsql.Types.String, primaryKey:true}, + item:lsql.Types.String + }); + console.error() + mKey.create(cb); + }, + function(cb) { + mItem = new lsql.Model("item", { + item:{type:lsql.Types.String, primaryKey:true}, + json:lsql.Types.String + }); + mItem.create(cb); + }, + function(cb) { + mResp = new lsql.Model("response", { + item:{type:lsql.Types.String}, + json:lsql.Types.String + }); + mResp.create(cb); + }], callback); } exports.clear = function(callback) { diff --git a/Collections/Timeline/timeline.js b/Collections/Timeline/timeline.js index 6d84e3340..c8e732abf 100644 --- a/Collections/Timeline/timeline.js +++ b/Collections/Timeline/timeline.js @@ -160,11 +160,13 @@ process.stdin.on('data', function(data) { locker.connectToMongo(function(mongo) { // initialize all our libs - dataStore.init(mongo.collections.item,mongo.collections.response, locker); - dataIn.init(locker, dataStore, function(){ - sync.init(locker, dataStore, dataIn, function(){ - app.listen(lockerInfo.port, 'localhost', function() { - process.stdout.write(data); + dataStore.init(mongo.collections.item,mongo.collections.response, locker, function(err){ + if(err) logger.error("datastore init failed: ",err); + dataIn.init(locker, dataStore, function(){ + sync.init(locker, dataStore, dataIn, function(){ + app.listen(lockerInfo.port, 'localhost', function() { + process.stdout.write(data); + }); }); }); });