diff -r 1a77b44dfa7d -r d60efd677b50 tweetcast/nodejs/server/node-direct.js --- a/tweetcast/nodejs/server/node-direct.js Tue Oct 25 18:26:39 2011 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,265 +0,0 @@ -READ_OLD_TWEETS = true; -RECORD_NEW_TWEETS = true; -TWEET_FILE_DIR = './'; -TWEET_FILE_START = 'tweets-'; -TWEET_FILE_EXT = '.txt'; -TRACKING_KEYWORD = '#p2'; - -var fs = require('fs'), - https = require('https'), - io = require('socket.io') - .listen(8000), - tweets = [], - arcs = [], - tweet_ids = [], - date_struct = [], - date_levels = [ - 3600 * 1000, - 15 * 60 * 1000, - 5 * 60 * 1000, - 60 * 1000 - ], - annkw = { - 'positive' : '++', - 'negative' : '--', - 'reference' : '==', - 'question' : '??' - }; - -function populateDateStruct(level, start) { - if (typeof start == "object") { - start = start.valueOf(); - } - var end = start + date_levels[level], - struct = { - "level" : level, - "start" : new Date(start), - "end" : new Date(end) - }; - if (level < date_levels.length - 1) { - struct.slices = []; - var newstart = start; - while (newstart < end) { - struct.slices.push(populateDateStruct(level + 1, newstart)); - newstart += date_levels[level + 1]; - } - } else { - struct.tweets = []; - struct.annotations = {}; - } - return struct; -} - -function insertIntoDateStruct(slices, tweet) { - var creadate = new Date(tweet.created_at); - for (var i in slices) { - if (creadate < slices[i].end) { - if (slices[i].slices) { - insertIntoDateStruct(slices[i].slices, tweet); - } else { - slices[i].tweets.push(tweet.pos); - if (tweet.annotations.length) { - var ann = tweet.annotations[0]; - if (slices[i].annotations[ann]) { - slices[i].annotations[ann].push(tweet.pos); - } else { - slices[i].annotations[ann] = [ tweet.pos ]; - } - } - } - break; - } - } -} - -function getSliceContent(slice) { - if (slice.slices) { - var twz = [], - annotations = {}; - for (var i in slice.slices) { - var data = getSliceContent(slice.slices[i]); - twz = twz.concat(data.tweets); - for (var j in data.annotations) { - if (annotations[j]) { - annotations[j] = annotations[j].concat(data.annotations[j]); - } else { - annotations[j] = data.annotations[j]; - } - } - } - } else { - twz = slice.tweets; - annotations = slice.annotations; - } - return { "tweets" : twz, "annotations" : annotations } -}; - -function flattenDateStruct(slices, target_level) { - var current_level = slices[0].level, - result = []; - if (current_level < target_level) { - if (slices[0].slices) { - for (var i in slices) { - result = result.concat(flattenDateStruct(slices[i].slices, target_level)); - } - } - } - else { - for (var i in slices) { - var data = getSliceContent(slices[i]); - result.push({ - "start" : slices[i].start, - "end" : slices[i].end, - "tweets" : data.tweets, - "annotations" : data.annotations - }); - } - } - return result; -} - -function trimFDS(slices) { - while (slices[0].tweets.length == 0) { - slices.splice(0,1); - } - while (slices[slices.length - 1].tweets.length == 0) { - slices.pop(); - } - return slices; -} - -function addToList(tweet) { - if (tweet_ids.indexOf(tweet.id) != -1) { - console.log("Error: Tweet already in list"); - return; - } - tweet.pos = tweets.length; - tweets.push(tweet); - tweet_ids.push(tweet.id); - var creadate = new Date(tweet.created_at); - if (!date_struct.length) { - date_struct = [ populateDateStruct(0, date_levels[0] * parseInt(creadate / date_levels[0])) ] - } - while (creadate > date_struct[date_struct.length - 1].end) { - date_struct.push( populateDateStruct(0, date_struct[date_struct.length - 1].end.valueOf()) ); - } - insertIntoDateStruct(date_struct, tweet); - if (tweet.in_reply_to_status_id) { - var ref = tweet_ids.indexOf(tweet.in_reply_to_status_id) - if (ref != -1) { - arcs.push({ - "from" : tweet.pos, - "to" : ref, - "type" : "reply" - }); - } - } - if (tweet.retweeted_status) { - var ref = tweet_ids.indexOf(tweet.retweeted_status.id_str) - if (ref != -1) { - arcs.push({ - "from" : tweet.pos, - "to" : ref, - "type" : "retweet" - }); - } - } -} - -function textids(object) { - for (var key in object) { - if (key.substr(-2) == 'id') { - object[key] = object[key + '_str']; - delete object[key + '_str']; - } - } -} - -function readTweetsFromFile() { - var dir = fs.readdirSync(TWEET_FILE_DIR), - fileName = null; - for (var i in dir) { - if ( dir[i].indexOf( TWEET_FILE_START + encodeURIComponent(TRACKING_KEYWORD) ) == 0 ) { - var oldtweets = fs.readFileSync(dir[i], 'utf8').split('\n'), - tweetscopied = 0; - for (var j in oldtweets) { - if (oldtweets[j].length > 1) { - addToList(JSON.parse(oldtweets[j])); - tweetscopied++; - } - } - console.log(tweetscopied, "tweets copied from", dir[i]); - console.log(arcs); - } - } -} - -function callBackNewTweets(chunk) { - var newdata = chunk.split('\r\n'); - for (var i in newdata) { - if (newdata[i].length > 0) { - var tweet = JSON.parse(newdata[i]), - annotations = []; - for (var a in annkw) { - if (tweet.text.indexOf(annkw[a]) != -1) { - annotations.push(a); - } - } - tweet.annotations = annotations; - textids(tweet); - textids(tweet.user); - - addToList(tweet); - var txt = JSON.stringify(tweet)+'\n'; - writestream.write(txt); - } - } - console.log("New tweets received. We now have", tweets.length, "tweets in memory"); - io.sockets.emit('tweetSummary', { tweetcount : tweets.length }); -} - -/* MAIN CODE */ - -if (READ_OLD_TWEETS) { - readTweetsFromFile(); -} - -if (RECORD_NEW_TWEETS) { - var now = new Date(), - fileTimestamp = (1900+now.getYear())+"-"+("0"+(1+now.getMonth())).slice(-2)+"-"+("0"+now.getDate()).slice(-2)+"."+("0"+now.getHours()).slice(-2)+"-"+("0"+now.getMinutes()).slice(-2), - fileName = TWEET_FILE_DIR + TWEET_FILE_START + encodeURIComponent(TRACKING_KEYWORD) + '-' + fileTimestamp + TWEET_FILE_EXT, - writestream = fs.createWriteStream(fileName, { flags: 'w', encoding: 'utf-8' }); - console.log('Writing to',fileName); - var req = https.request({ - host: "stream.twitter.com", - path: "/1/statuses/filter.json", - method: "POST", - headers: { - 'Authorization': 'Basic cmFwaHY6N3czMzdMZkMyM2dF', - 'Content-Type': 'application/x-www-form-urlencoded' - } - }, function(res) { - console.log('STATUS: ' + res.statusCode); - console.log('HEADERS: ' + JSON.stringify(res.headers)); - res.setEncoding('utf8'); - res.on('data', callBackNewTweets); - }); - - req.write('track=' + encodeURIComponent(TRACKING_KEYWORD)); - req.end(); -} - -io.set('log level', 0); -io.sockets.on('connection', function(socket) { - console.log("New connection from", socket.handshake.address.address,"id :", socket.id); - socket.emit('tweetSummary', { tweetcount : tweets.length }); - socket.on('getTweets', function(data) { - console.log('getTweets from',socket.id); - data.tweets = tweets.slice(Math.max(0, data.from), data.to); - socket.emit('tweets', data); - }); - socket.on('getTimeline', function(data) { - socket.emit('timeline', { "timeline" : trimFDS(flattenDateStruct(date_struct, data.level)), "arcs" : arcs }); - console.log('getTimeline from',socket.id); - }); -}); \ No newline at end of file