--- a/tweetcast/nodejs/node-direct.js Wed Dec 07 19:28:46 2011 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,459 +0,0 @@
-/* DEFAULT CONFIGURATION */
-
-//var TWITTER_USER = 'materiauxnum',
-// TWITTER_PASS = 'm473r14ux7w337',
-var TWITTER_USER = 'raphv',
- TWITTER_PASS = '7w337LfC23gE',
- RECORD_NEW_TWEETS = true,
- commit_script = '';
-
-/* CALLING COMMON CONFIGURATION FILE */
-
-var conf_file = flagOption('-c', 'conf.js');
-
-myLog('Reading Configuration from ' + conf_file);
-
-var fs = require('fs');
-eval(fs.readFileSync(conf_file,'utf8'));
-
-if (typeof sqlfile == "undefined") {
- sqlfile = __dirname + '/tweets-' + encodeURIComponent(tracking_keyword) + '.sqlite'
- }
-
-/* FUNCTIONS */
-
-function flagOption(flag, defaultValue) {
- var flagPos = process.argv.indexOf(flag);
- return ( flagPos != -1 && flagPos < process.argv.length - 1) ? process.argv[flagPos + 1] : defaultValue;
-}
-
-function myLog() {
- var args = ["\033[1;34m["+new Date().toLocaleTimeString()+"]\033[0m"];
- for (var i in arguments) {
- args.push(arguments[i]);
- }
- console.log.apply(console, args);
-}
-
-function annotationMap(callback, options) {
- var includeDefault = ( options && options.includeDefault ? options.includeDefault : false );
- var returnObject = ( options && options.returnObject ? options.returnObject : false );
- res = (returnObject ? {} : []);
- for (var i in annotations) {
- if (i != "default" || includeDefault) {
- var el = callback(i, annotations[i])
- if (returnObject) {
- res[i] = el;
- } else {
- res.push(el);
- }
- }
- }
- return res;
-}
-
-function createTables() {
-
- var requete = "CREATE TABLE IF NOT EXISTS tweets ( pos INTEGER PRIMARY KEY, tweet_id TEXT UNIQUE, created_at INTEGER, json TEXT"
- + annotationMap(function(a) { return ', a_' + a + ' INTEGER' }).join("")
- + " );\n"
- + "CREATE TABLE IF NOT EXISTS tweet_refs ( id INTEGER PRIMARY KEY, from_id TEXT, to_id TEXT, ref_type TEXT );\n"
- + "CREATE INDEX IF NOT EXISTS idx_created_at ON tweets ( created_at );";
- db.executeScript(requete, function(err) {
- if (err) { myLog("SQLITE error",err.stack); }
- getSendLastPos();
- });
-}
-
-function commitReference(from_id, to_id, ref_type) {
- commit_script += 'INSERT OR IGNORE INTO tweet_refs ( from_id, to_id, ref_type ) VALUES ( "' + from_id + '", "' + to_id + '", "' + ref_type + '" );\n';
-}
-
-function commitTweet(data) {
-
- var tweet = JSON.parse(data),
- ann = [];
-
- if (!tweet.id) {
- myLog("Error: Could not parse data",data);
- return;
- }
-
- try {
- textids(tweet);
- for (var j in keys_to_delete) {
- delete tweet[keys_to_delete[j]];
- }
- textids(tweet.user);
- for (var j in user_keys_to_delete) {
- delete tweet.user[user_keys_to_delete[j]];
- }
- if (tweet.retweeted_status) {
- textids(tweet.retweeted_status);
- for (var j in keys_to_delete) {
- delete tweet.retweeted_status[keys_to_delete[j]];
- }
- }
- annotationMap(function(i, annotation) {
- for (var j in annotation.keywords) {
- if (tweet.text.search(annotation.keywords[j]) != -1) {
- ann.push(i);
- break;
- }
- }
- });
- tweet.annotations = ann;
- tweet.created_at = new Date(tweet.created_at);
-// myLog("Time delta :",(new Date() - tweet.created_at) / 1000);
- } catch (err) {
- myLog("Error while processing tweet",err.stack);
- }
-
- if (tweet.in_reply_to_status_id) {
- commitReference( tweet.id, tweet.in_reply_to_status_id, "reply" );
- }
- if (tweet.retweeted_status) {
- commitReference( tweet.id, tweet.retweeted_status.id, "retweet" );
- }
- commit_script += 'INSERT INTO tweets ( tweet_id, created_at, json '
- + annotationMap(function(a) { return ', a_' + a }).join("")
- + ' ) VALUES ( "'
- + tweet.id
- + '", '
- + tweet.created_at.valueOf()
- + ', "'
- + JSON.stringify(tweet).replace(/\"/g, '""')
- + '"'
- + annotationMap(function(a) {
- return ann.indexOf(a) == -1 ? ', 0' : ', 1'
- }).join("")
- + ' );\n';
-}
-
-function callBackNewTweets(chunk) {
- var newdata = chunk.split('\r\n');
- for (var i in newdata) {
- if (newdata[i].length > 0) {
- commitTweet(newdata[i]);
- }
- }
-// myLog("Data received - length :",chunk.length);
-}
-
-function requestTwitter() {
- myLog("Requesting Twitter to track keyword(s): "+tracking_keyword);
- var req = https.request({
- host: "stream.twitter.com",
- path: "/1/statuses/filter.json",
- method: "POST",
- headers: {
- 'Authorization': 'Basic ' + new Buffer( TWITTER_USER + ":" + TWITTER_PASS ).toString('base64'),
- 'Content-Type': 'application/x-www-form-urlencoded'
- }
- }, function(res) {
- myLog('Reply from stream.twitter.com: ' + res.statusCode);
- myLog('Headers: ' + JSON.stringify(res.headers));
- res.setEncoding('utf8');
- res.on('data', callBackNewTweets);
- res.on('end', function() {
- myLog('End Twitter Connection — Trying to reconnect');
- requestTwitter();
- });
- });
-
- req.write('track=' + encodeURIComponent(tracking_keyword));
- req.socket.setTimeout(60000);
- req.socket.on('timeout', function() {
- myLog('TimeOut - Trying to reconnect');
- requestTwitter();
- });
- req.end();
-}
-
-function getSendLastPos() {
- db.execute("SELECT MAX(pos) lastpos FROM tweets", function (err, results) {
- if (err) { myLog("SQLITE error",err.stack); }
- if (results[0].lastpos != lastpos) {
- lastpos = results[0].lastpos ? results[0].lastpos : 0;
- try {
- io.sockets.emit('tweetSummary', {
- tweetcount : lastpos
- });
- } catch(err) {
- myLog("SOCKET.IO error while Broadcasting tweetSummary",err.stack);
- }
- }
- });
-}
-
-function commitTweets() {
- if (commit_script != '') {
- var requete = commit_script;
- commit_script = '';
- // console.log(requete);
- // var reqd = new Date();
- db.executeScript(requete, function (err) {
- if (err) { myLog("SQLITE error",err.stack); }
- // myLog("Commit took",(new Date() - reqd),"ms");
- getSendLastPos();
- });
- }
-}
-
-function getSendTweetPosByDate(date, socket) {
- db.execute("SELECT pos, created_at, ABS(created_at-" + date + ") AS dist FROM tweets ORDER BY dist ASC LIMIT 0,9", function (err, results) {
- if (err) { myLog("SQLITE error",err.stack); }
- if (results.length) {
- try {
- socket.emit('tweetPosByDate', {
- tweetpos : results[0].pos,
- date : results[0].created_at
- });
- } catch(err) {
- myLog("SOCKET.IO error while sending tweetPosByDate",err.stack);
- }
- }
- });
-}
-
-function getSendLinkedTweets(pos, socket) {
-// myLog("request for tweets linked to",pos);
- db.execute("SELECT A.pos pos_a, A.tweet_id id_a, B.pos pos_b, B.tweet_id id_b, ref_type, ABS(B.created_at - A.created_at) delta FROM tweets A, tweets B, tweet_refs WHERE id_a = from_id AND id_b = to_id AND (pos_a = ? OR pos_b = ?) ORDER BY delta ASC LIMIT 0, 10", [ pos, pos ], function(err, results) {
- if (err) { myLog("SQLITE error: ",err.stack); }
- var struct = {
- "tweetpos" : pos,
- "referencing" : [],
- "referenced_by" : []
- };
- for (var i in results) {
- if (results[i].pos_a == pos) {
- struct.referencing.push({
- "pos" : results[i].pos_b,
- "ref_type" : results[i].ref_type
- });
- } else {
- struct.referenced_by.push({
- "pos" : results[i].pos_a,
- "ref_type" : results[i].ref_type
- });
- }
- }
- try {
- socket.emit('linkedTweets', struct);
- } catch(err) {
- myLog("SOCKET.IO error while sending linkedTweets: ",err.stack);
- }
- });
-}
-
-function getSendTweets(posList, socket) {
-// myLog("request for tweets ("+posList.join(',')+") from "+socket.id);
- db.execute("SELECT * FROM tweets WHERE pos IN ( " + posList.join(',') + " )", function (err, results) {
- if (err) { myLog("SQLITE error",err.stack); }
- try {
- socket.emit('tweets',
- results.map(function(line) {
- var tw = JSON.parse(line.json);
- tw.pos = line.pos;
- return tw;
- })
- );
- } catch (err) {
- myLog("SOCKET.IO error while sending tweets",err.stack);
- }
- });
-}
-
-function getSendTimeline(data, socket) {
-// myLog("request for timeline (",data.level, data.full,") from "+socket.id);
- var lvl = date_levels[data.level],
- requete = "SELECT COUNT(*) AS nb, "
- + lvl
- + "*ROUND(created_at/"
- + lvl
- + ") AS tranche"
- + annotationMap(function (a) { return " , SUM(a_" + a + ") AS s_" + a }).join("")
- + " FROM tweets GROUP BY tranche ORDER BY tranche DESC LIMIT 0," + ( data.full ? "50" : "1" );
- db.execute(requete, function (err, results) {
- if (err) { myLog("SQLITE error",err.stack); }
- if (!results.length) {
- return;
- }
- var tbl = [],
- lastend = parseInt(results[results.length - 1].tranche);
- for (var i = results.length - 1; i >= 0; i--) {
- var start = parseInt(results[i].tranche);
- while (start > lastend) {
- var struct = { "start": lastend, "tweets" : 0, "annotations" : {} };
- lastend += lvl;
- struct.end = lastend;
- tbl.push(struct);
- }
- lastend += lvl;
- var struct = {
- "start" : start,
- "end" : lastend,
- "tweets" : results[i].nb,
- "annotations" : annotationMap(function (a) {
- return results[i]['s_'+a];
- },{returnObject: true})
- }
- tbl.push(struct);
- }
- try {
- socket.emit('timeline', {
- "full" : data.full,
- "level" : data.level,
- "data" : tbl
- });
- } catch (err) {
- myLog("SOCKET.IO error while sending timeline",err.stack);
- }
- });
-}
-
-function textids(object) {
- for (var key in object) {
- // Workaround for Unicode bug in socket.io.
-
- if (typeof object[key] == "string") {
- var tmp = '';
- for (var i = 0; i < object[key].length; i++) {
- tmp += ( object[key].charCodeAt(i) < 128 ? object[key].charAt(i) : "&#" + object[key].charCodeAt(i) + ";" );
- }
- object[key] = tmp;
- }
-
- if (key.substr(-2) == 'id') {
- object[key] = object[key + '_str'];
- delete object[key + '_str'];
- }
-
- }
-}
-
-function httpHandler(req, res) {
- myLog("HTTP Request for URL "+req.url);
- var url = ( req.url == "/config" ? conf_file : __dirname + "/client" + req.url + ( req.url[req.url.length - 1] == "/" ? "index.html" : "" ) );
- fs.readFile( url, function(err, data) {
- if (err) {
- myLog("Error 404");
- res.writeHead(404);
- return res.end('File not found');
- }
- res.writeHead(200);
- res.end(data);
- });
-}
-
-/* Initialization */
-
-var http = require('http'),
- https = require('https'),
- sqlite = require('sqlite'),
- socketio = require('socket.io'),
- tweets = [],
- lastpos = 0,
- arcs = [],
- tweet_ids = [],
- date_struct = [],
- date_levels = [
- 3600 * 1000,
- 15 * 60 * 1000,
- 5 * 60 * 1000,
- 60 * 1000,
- 15 * 1000
- ],
- keys_to_delete = [
- 'in_reply_to_screen_name',
- 'in_reply_to_user_id',
- 'retweeted',
- 'place',
- 'geo',
- 'source',
- 'contributors',
- 'coordinates',
- 'retweet_count',
- 'favorited',
- 'truncated',
- 'possibly_sensitive'
- ],
- user_keys_to_delete = [
- 'default_profile_image',
- 'show_all_inline_media',
- 'contributors_enabled',
- 'profile_sidebar_fill_color',
- 'created_at',
- 'lang',
- 'time_zone',
- 'profile_sidebar_border_color',
- 'follow_request_sent',
- 'profile_background_image_url',
- 'profile_background_image_url_https',
- 'followers_count',
- 'description',
- 'url',
- 'geo_enabled',
- 'profile_use_background_image',
- 'default_profile',
- 'following',
- 'profile_text_color',
- 'is_translator',
- 'favourites_count',
- 'listed_count',
- 'friends_count',
- 'profile_link_color',
- 'protected',
- 'location',
- 'notifications',
- 'profile_image_url_https',
- 'statuses_count',
- 'verified',
- 'profile_background_color',
- 'profile_background_tile',
- 'utc_offset'
- ],
- app = http.createServer(httpHandler),
- io = socketio.listen(app),
- db = new sqlite.Database();
-
-/* MAIN CODE */
-
-app.listen(app_port);
-myLog("Listening on port: "+app_port);
-myLog("Opening SQLITE file: "+sqlfile);
-db.open(sqlfile , function(err) {
- if (err) { myLog("SQLITE error",err.stack); }
- createTables();
-});
-
-setInterval(commitTweets,500);
-setInterval(function(){myLog("Still alive, tweet count",lastpos)}, 60000);
-
-if (RECORD_NEW_TWEETS) {
- requestTwitter();
-}
-
-io.set('log level', 0);
-io.sockets.on('connection', function(socket) {
- myLog("New connection from", socket.handshake.address.address, "with id=", socket.id);
- try {
- socket.emit('tweetSummary', { tweetcount : lastpos });
- } catch (err) {
- myLog("SOCKET.IO error while sending tweetSummary",err.stack);
- }
- socket.on('updateTweets', function(data) {
- if (data.tweets.length) {
- getSendTweets(data.tweets, socket);
- }
- });
- socket.on('updateTimeline', function(data) {
- getSendTimeline(data, socket);
- });
- socket.on('tweetPosByDate', function(data) {
- getSendTweetPosByDate(data.date, socket);
- });
- socket.on('linkedTweets', function(data) {
- getSendLinkedTweets(data.tweetpos, socket);
- });
-});
\ No newline at end of file