# HG changeset patch # User Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com> # Date 1319560028 -7200 # Node ID d60efd677b503c79d50463f011383ec3f9fc8081 # Parent 1a77b44dfa7d2406a4389c9bed5ca1e113f37ca6# Parent 5f83c21dee69a205a9c67fe5a50043225c1b61fa Merge with 9321f0a58b041ce1db18e2437be94be4677de391 diff -r 1a77b44dfa7d -r d60efd677b50 tweetcast/nodejs/client/css/style.css --- a/tweetcast/nodejs/client/css/style.css Tue Oct 25 18:26:39 2011 +0200 +++ b/tweetcast/nodejs/client/css/style.css Tue Oct 25 18:27:08 2011 +0200 @@ -28,33 +28,45 @@ /* Liste de Tweets */ #tweetlist { - position: absolute; left: 50px; top: 50px; width: 240px; height: 600px; list-style: none; overflow: hidden; border: 1px solid #999; + position: absolute; left: 50px; top: 50px; width: 250px; height: 600px; list-style: none; border: 1px solid #999; +} + +li.tweet, li.placeholder { + border: 1px solid; overflow: hidden; float: left; margin: -1px 0 0 -1px; border-color: #ccc; } li.tweet { - border-width: 1px; border-style: none solid solid none; border-color: #ccc; overflow: hidden; float: left; + position: relative; background: #fff; } -li.tweet.full { - width: 240px; height: 99px; +li.full { + width: 240px; height: 119px; border-right: 10px solid #ff0; +} + +li.half { + width: 250px; height: 49px; } -li.tweet.half { - width: 240px; height: 49px; +li.quarter { + width: 124px; height: 44px; } -li.tweet.quarter { - width: 119px; height: 49px; +li.icons { + width: 24px; height: 24px; } -li.tweet.icons { - width: 29px; height: 29px; +li.placeholder { + height: 0; visibility: hidden; } .tweet h4 { font-weight: bold; } +.quarter p { + word-break: break-all; +} + .full h4, .full p, .half p { margin: 5px; } @@ -68,55 +80,94 @@ } .quarter p { - font-size: 10px; margin: 2px 2px 2px 28px; + margin: 2px; font-size: 10px; } -img.tweet_profile_image { - float: left; opacity: .7; filter:alpha(opacity=70); +img.profile_image { + float: left; } -.full .tweet_profile_image { +.full .profile_image { margin: 5px; width: 48px; height: 48px; } -.half .tweet_profile_image { +.half .profile_image { margin: 5px; width: 39px; height: 39px; } -.quarter .tweet_profile_image { - margin: 2px; width: 25px; height: 25px; +.quarter .profile_image { + margin: 2px; width: 22px; height: 22px; } - .icons .tweet_profile_image { - margin: 3px; width: 23px; height: 23px; +.icons .profile_image { + margin: 2px; width: 20px; height: 20px; } -p.tweet_created_at { +.full .created_at { font-style: italic; color: #999; } -.half h4, .half p.tweet_created_at, .quarter h4, .quarter p.tweet_created_at, .icons h4, .icons p { +.half h4, .half p.created_at, .quarter h4, .quarter p.created_at, .icons h4, .icons p { display: none } -.a_positive { - background: #c5e7cd; +.full .tweet_text { + clear: left; +} + +.annotations { + position: absolute; margin: 0; padding: 0; top: 0; left: 0; width: 100%; height: 100%; +} + +/*.icons .annotations { + position: absolute; width: 6px; height: 6px; right: 2px; bottom: 2px; border: 1px solid #ccc; z-index: 3; +}*/ + +div.annotation { + float: left; height: 100%; +} + +div.twmain { + position: absolute; top: 0; left: 0; width: 100%; height: 100%; z-index: 2; overflow: hidden; +} + +.full .twmain { } -.a_negative { - background: #f6ced0; +.half .twmain { + opacity: .9 +} + +.half.fade .twmain { + opacity: .8 } -.a_reference { - background: #ecedc1; +.quarter .twmain { + opacity: .7 } -.a_question { - background: #bfdbec; +.quarter.fade .twmain { + opacity: .6 +} + +.icons .twmain { + opacity: .5 +} + +.icons.fade .twmain { + opacity: .4 } /* timeline */ +#timeline, #timewindow { + position: absolute; top: 50px; left: 301px; height: 600px; width: 220px; border: 1px solid #999; +} + #timeline { - position: absolute; top: 50px; left: 291px; height: 600px; width: 160px; border: 1px solid #999; + z-index: 42 +} + +#timewindow { + z-index: 47 } \ No newline at end of file diff -r 1a77b44dfa7d -r d60efd677b50 tweetcast/nodejs/client/index.html --- a/tweetcast/nodejs/client/index.html Tue Oct 25 18:26:39 2011 +0200 +++ b/tweetcast/nodejs/client/index.html Tue Oct 25 18:27:08 2011 +0200 @@ -4,20 +4,17 @@ Polemic Tweet “TweetCast” - + + + -

Niveau de zoom (liste de tweets) 1 - 2 - 3 - 4

-

Niveau de zoom (timeline) 1 - 2 - 3 - 4

- -
+ +
+
\ No newline at end of file diff -r 1a77b44dfa7d -r d60efd677b50 tweetcast/nodejs/client/js/jquery.mousewheel.min.js --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tweetcast/nodejs/client/js/jquery.mousewheel.min.js Tue Oct 25 18:27:08 2011 +0200 @@ -0,0 +1,11 @@ +/* Copyright (c) 2009 Brandon Aaron (http://brandonaaron.net) + * Dual licensed under the MIT (http://www.opensource.org/licenses/mit-license.php) + * and GPL (http://www.opensource.org/licenses/gpl-license.php) licenses. + * Thanks to: http://adomas.org/javascript-mouse-wheel/ for some pointers. + * Thanks to: Mathias Bank(http://www.mathias-bank.de) for a scope bug fix. + * + * Version: 3.0.2 + * + * Requires: 1.2.2+ + */ +(function(c){var a=["DOMMouseScroll","mousewheel"];c.event.special.mousewheel={setup:function(){if(this.addEventListener){for(var d=a.length;d;){this.addEventListener(a[--d],b,false)}}else{this.onmousewheel=b}},teardown:function(){if(this.removeEventListener){for(var d=a.length;d;){this.removeEventListener(a[--d],b,false)}}else{this.onmousewheel=null}}};c.fn.extend({mousewheel:function(d){return d?this.bind("mousewheel",d):this.trigger("mousewheel")},unmousewheel:function(d){return this.unbind("mousewheel",d)}});function b(f){var d=[].slice.call(arguments,1),g=0,e=true;f=c.event.fix(f||window.event);f.type="mousewheel";if(f.wheelDelta){g=f.wheelDelta/120}if(f.detail){g=-f.detail/3}d.unshift(f,g);return c.event.handle.apply(this,d)}})(jQuery); \ No newline at end of file diff -r 1a77b44dfa7d -r d60efd677b50 tweetcast/nodejs/client/js/script.js --- a/tweetcast/nodejs/client/js/script.js Tue Oct 25 18:26:39 2011 +0200 +++ b/tweetcast/nodejs/client/js/script.js Tue Oct 25 18:27:08 2011 +0200 @@ -1,60 +1,61 @@ var socket, - paper, + tlPaper, + twPaper, tweetData = { "tweetcount" : 0, - "position" : -1, + "position" : 0, + "followLast" : true, "zoomLevel" : 3, - "timeLevel" : 2 + "timeLevel" : 2, + "tweets" : [], + "posIndex" : [], + "tlChanged" : true }, - zoomLevels = [ + displaySplitting = [ { - "description" : "160 tweets per page", - "className" : "icons", - "displayCount" : 160 - }, - { - "description" : "24 tweets per page", - "className" : "quarter", - "displayCount" : 24 - }, - { - "description" : "12 tweets per page", - "className" : "half", - "displayCount" : 12 + positions : [ 10, 20, 22, 24, 25, 26, 27, 28, 29, 31, 33, 43, 53 ], + classNames : [ 'icons fade', 'icons', 'quarter fade', 'quarter', 'half fade', 'half', 'full', 'half', 'half fade', 'quarter', 'quarter fade', 'icons', 'icons fade' ] }, { - "description" : "6 tweets per page", - "className" : "full", - "displayCount" : 6 + positions : [ 1, 3, 5, 7, 13, 33, 53 ], + classNames : [ 'full', 'half', 'half fade', 'quarter', 'quarter fade', 'icons', 'icons fade' ] } ], - colors = { - 'positive' : "#1D973D", - 'reference' : "#C5A62D", - 'negative' : "#CE0A15", - 'question' : "#036AAE" - }, - timeWindow; + blockUpdate = false, + waitUpdate = true, + wheeldelta = 0; + +function placeHolder(className) { + return '
  • '; +} -function tweetToHtml(tweet) { - html = '
  • '; + html += '>'; + if (tweet.annotations.length) { + html += '
    '; + for (var i in tweet.annotations) { + html += '
    '; + } + html += '
    '; + } + html += '
    '; a_user = ''; if (tweet.user.profile_image_url) { - html += a_user + ''; + html += a_user + ''; } - html += '

    ' + a_user + '@' + tweet.user.screen_name + '

    ' + new Date(tweet.created_at).toLocaleTimeString() + '

    '; + html += '

    ' + a_user + '@' + tweet.user.screen_name + '

    ' + new Date(tweet.created_at).toLocaleTimeString() + '

    '; lastend = 0; txt = ''; entities = []; - for (var i in tweet.entities.hashtag) { + for (var i in tweet.entities.hashtags) { entities.push({ - "start" : tweet.entities.hashtag[i].indices[0], - "end" : tweet.entities.hashtag[i].indices[1], - "html" : '#' + tweet.entities.hashtag[i].text + '' + "start" : tweet.entities.hashtags[i].indices[0], + "end" : tweet.entities.hashtags[i].indices[1], + "html" : '#' + tweet.entities.hashtags[i].text + '' }); } for (var i in tweet.entities.urls) { @@ -77,110 +78,246 @@ lastend = entities[i].end; } txt += tweet.text.substring(lastend); - html += txt + '

    • '; + html += txt + '

    '; return html; } -function displayTweets() { - var to = tweetData.position + 1, - from = Math.max(0, to - zoomLevels[tweetData.zoomLevel].displayCount); - socket.emit('getTweets',{ "from": from, "to": to, "callback" : "display" }); + +function getUpdate() { + tweetData.posToDisplay = []; + if (tweetData.followLast) { + for (var i = tweetData.tweetcount; i >= tweetData.tweetcount - 52; i--) { + tweetData.posToDisplay.push( i > 0 ? i : -1 ); + } + tweetData.end = tweetData.tweetcount; + tweetData.start = Math.max(1, tweetData.end - 52); + } else { + for (var i = tweetData.position + 26; i >= tweetData.position - 26; i--) { + tweetData.posToDisplay.push( i > 0 && i <= tweetData.tweetcount ? i : -1 ); + } + tweetData.end = Math.min(tweetData.tweetcount, tweetData.position + 26 ); + tweetData.start = Math.max(1, tweetData.position - 26); + } + var tweetsToGet = []; + for (var i = tweetData.start; i <= tweetData.end; i++) { + if (!tweetByPos(i)) { + tweetsToGet.push(i); + } + } + if (tweetsToGet.length) { + socket.emit('updateTweets', { + "tweets" : tweetsToGet + }); + } else { + drawTweetList(); + //drawDisplay(); + } + if (tweetData.tlChanged) { + socket.emit('updateTimeline', { + "level" : tweetData.timeLevel + }); + } else { + drawTimeLine(); + } } -function setZoom(level) { - tweetData.zoomLevel = Math.max(0, Math.min( zoomLevels.length - 1 , level ) ); - displayTweets(); +function tweetByPos(pos) { + var index = tweetData.posIndex.indexOf(pos); + return (index == -1 ? false : tweetData.tweets[index]); +} + +function delayedUpdate() { + blockUpdate = false; + if (waitUpdate) { + updateDisplay(); + } +} + +function updateDisplay() { + if (blockUpdate) { + waitUpdate = true; + } else { + waitUpdate = false; + getUpdate(); + blockUpdate = true; + setTimeout(delayedUpdate, 100); + } } function setTimeZoom(level) { - tweetData.timeLevel = Math.max(0, Math.min( 3, level )); - getTimeline(); + if (level >= 0 && level <= 4) { + tweetData.timeLevel = level; + tweetData.tlChanged = true; + updateDisplay(); + } } -function getTimeline() { - socket.emit('getTimeline',{"level":tweetData.timeLevel}); +function drawTweetList() { + html = ''; + var i = 0; + while (i < tweetData.posIndex.length) { + if (tweetData.posToDisplay.indexOf(tweetData.posIndex[i]) == -1) { + tweetData.posIndex.splice(i,1); + tweetData.tweets.splice(i,1); + } else { + i++; + } + } + for (var i in tweetData.posToDisplay) { + var ds = displaySplitting[tweetData.followLast ? 1 : 0]; + for (var j in ds.positions) { + if (ds.positions[j] > i) { + var className = ds.classNames[j]; + break; + } + } + html += ( tweetData.posToDisplay[i] != -1 ? tweetToHtml(tweetByPos(tweetData.posToDisplay[i]), className) : placeHolder(className) ); + } + $("#tweetlist").html(html); + if (tweetData.followLast) { + $("#tweet_" + tweetData.position).fadeIn(500); + } + drawTimeWindow(); +} + +function drawTimeWindow() { + twPaper.clear(); + if (!tweetData.timeline || !tweetData.timeline.length) return; + + var dtfintl = new Date ( tweetData.timeline[ tweetData.timeline.length - 1 ].end ), + dtdebtl = new Date ( tweetData.timeline[0].start ), + dtfintw = new Date( tweetByPos( tweetData.end ).created_at ), + dtdebtw = new Date( tweetByPos( tweetData.start ).created_at ), + scY = 600 / ( dtfintl - dtdebtl ); + twPaper.rect( 0, scY * ( dtfintl - dtfintw ), 300, scY * ( dtfintw - dtdebtw )).attr({"stroke":"none","fill":"#8080ff","fill-opacity":.2}); + var dtcour = new Date( tweetByPos( tweetData.position ).created_at ), + posY = scY * ( dtfintl - dtcour ); + twPaper.path("M0 "+posY+"L300 "+posY).attr({"stroke":"#ff0"}); } -function showTimeWindow() { - if (timeWindow) { - timeWindow.remove(); - timeWindow = null; +function drawTimeLine() { + tlPaper.clear(); + if (!tweetData.timeline || !tweetData.timeline.length) return; + tweetData.tlTweetRects = []; + var scaleY = 600 / tweetData.timeline.length, + max = 0; + for (var i = 0; i < tweetData.timeline.length; i++) { + max = Math.max(max, tweetData.timeline[i].tweets); } - if (tweetData.tweetsOnDisplay && tweetData.timelineOnDisplay) { - var dtfintl = new Date ( tweetData.timelineOnDisplay[ tweetData.timelineOnDisplay.length - 1 ].end ), - dtdebtl = new Date ( tweetData.timelineOnDisplay[0].start ), - dtfintw = new Date( tweetData.tweetsOnDisplay[0].created_at ), - dtdebtw = new Date( tweetData.tweetsOnDisplay[ tweetData.tweetsOnDisplay.length - 1 ].created_at ), - scaleY = 600 / ( dtfintl - dtdebtl ); - timeWindow = paper.rect( 0, scaleY * ( dtfintl - dtfintw ), 600, scaleY * ( dtfintw - dtdebtw )).attr({"stroke":"#000080","fill":"#8080ff","fill-opacity":.2}); + var scaleX = 160 / max; + + // dessin de l'axe vertical + + tlPaper.path("M160 0L160 600").attr({"stroke":"#ccc"}); + + // dessin de la date de début + + tlPaper.text(165, 592, new Date(tweetData.timeline[0].start).toLocaleTimeString()).attr({ "text-anchor" : "start", "font-size": "12px" }); + + // dessin de la date de fin + + tlPaper.text(165, 7, new Date(tweetData.timeline[tweetData.timeline.length - 1].end).toLocaleTimeString()).attr({ "text-anchor" : "start", "font-size": "12px" }); + for (var i = 0; i < tweetData.timeline.length; i++) { + var posY = 600 - (i * scaleY); + + // Si on est à une demi-heure, on trace un axe secondaire + heure + + if (i && !(new Date(tweetData.timeline[i].start).valueOf() % 1800000)) { + tlPaper.path("M0 "+posY+"L165 "+posY).attr({"stroke":"#ccc"}); + tlPaper.text(165, posY, new Date(tweetData.timeline[i].start).toLocaleTimeString()).attr({ "text-anchor" : "start", "font-size": "12px" }); + } + var anz = { + "default" :tweetData.timeline[i].tweets + }; + for (var j in tweetData.timeline[i].annotations) { + anz.default -= tweetData.timeline[i].annotations[j]; + anz[j] = tweetData.timeline[i].annotations[j]; + } + var posX = 0; + for (var j in anz) { + var largX = scaleX * anz[j]; + tlPaper.rect(posX, 600 - scaleY * (i+1), largX, scaleY).attr({"stroke": "none", "fill": annotations[j].colors.timeline}); + posX += largX; + } } + + drawTimeWindow(); +/* for (var i = 0; i < tweetData.arcs.length; i++) { + var x1 = scaleX * (tmptw[tweetData.arcs[i].from].x + .5), + x2 = scaleX * (tmptw[tweetData.arcs[i].to].x + .5), + y1 = 600 - scaleY * (tmptw[tweetData.arcs[i].from].y + .5), + y2 = 600 - scaleY * (tmptw[tweetData.arcs[i].to].y + .5), + d = "M"+x1+" "+y1+"C"; + if (y1 == y2) { + d += x1+" "+(y1 - 30)+" "+x2+" "+(y2 - 30); + } else { + d += (x1 + 60)+" "+y1+" "+(x2 + 60)+" "+y2; + } + paper.path(d+" "+x2+" "+y2); + } */ } $(document).ready(function() { - paper = Raphael("timeline", 160, 600); - socket = io.connect('http://' + S_IO_HOST + ':' + S_IO_PORT ); + tlPaper = Raphael("timeline", 220, 600); + twPaper = Raphael("timewindow", 220, 600); + socket = io.connect('http://' + document.location.hostname ); socket.on('tweetSummary', function (data) { - if (tweetData.position == tweetData.tweetcount - 1) { - tweetData.position = data.tweetcount - 1; - displayTweets(); - getTimeline(); + if (tweetData.tweetcount != data.tweetcount) { + tweetData.tweetcount = data.tweetcount; + tweetData.tlChanged = true; + if (tweetData.followLast) { + tweetData.position = data.tweetcount; + } + updateDisplay(); } - tweetData.tweetcount = data.tweetcount; }); socket.on('tweets', function (data) { - switch (data.callback) { - case "display": - tweetData.tweetsOnDisplay = data.tweets; - tweetData.tweetsOnDisplay.reverse(); - html = ''; - for (var i in tweetData.tweetsOnDisplay) { - html += tweetToHtml(tweetData.tweetsOnDisplay[i]); - } - $("#tweetlist").html(html); - showTimeWindow(); - break; + for (var i in data) { + if (tweetData.posIndex.indexOf(data[i].pos) == -1) { + tweetData.tweets.push(data[i]); + tweetData.posIndex.push(data[i].pos); + } } + drawTweetList(); }); socket.on('timeline', function (data) { - tweetData.timelineOnDisplay = data.timeline; - paper.clear(); - timeWindow = null; - var max = 0; - for (var i in data.timeline) { - max = Math.max(max, data.timeline[i].tweets.length); - } - var scaleX = 160 / max, - scaleY = 600 / data.timeline.length, - tmptw = []; - for (var i = 0; i < tweetData.tweetcount; i++) { - tmptw.push({}); - } - for (var i = 0; i < data.timeline.length; i++) { - for (var j = 0; j < data.timeline[i].tweets.length; j++) { - var coul = "#585858"; - for (var k in data.timeline[i].annotations) { - if (data.timeline[i].annotations[k].indexOf(data.timeline[i].tweets[j]) != -1) { - coul = colors[k]; - } - } - tmptw[data.timeline[i].tweets[j]].y = i; - tmptw[data.timeline[i].tweets[j]].x = j; - paper.rect(scaleX * j + (scaleX > 3 ? .5 : 0), 600 - ((i+1) * scaleY) + (scaleY > 3 ? .5 : 0), scaleX - (scaleX > 3 ? 1 : 0), scaleY - (scaleY > 3 ? 1 : 0)).attr({"stroke":"none","fill":coul}); + tweetData.timeline = data; + drawTimeLine(); + }); + socket.on('display', function (data) { + tweetData.tlChanged = false; + for (var i in data.tweets) { + if (tweetData.posIndex.indexOf(data.tweets[i].pos) == -1) { + tweetData.tweets.push(data.tweets[i]); + tweetData.posIndex.push(data.tweets[i].pos); } } - for (var i = 0; i < data.arcs.length; i++) { - var x1 = scaleX * (tmptw[data.arcs[i].from].x + .5), - x2 = scaleX * (tmptw[data.arcs[i].to].x + .5), - y1 = 600 - scaleY * (tmptw[data.arcs[i].from].y + .5), - y2 = 600 - scaleY * (tmptw[data.arcs[i].to].y + .5), - d = "M"+x1+" "+y1+"C"; - if (y1 == y2) { - d += x1+" "+(y1 - 60)+" "+x2+" "+(y2 - 60); + if (data.timeline) { + tweetData.timeline = data.timeline + } + tweetData.arcs = data.arcs; + drawDisplay(); + }); + $("#tweetlist").mousewheel(function(e, d) { + wheeldelta += d; + if (Math.abs(wheeldelta) >= 1) { + tweetData.position = Math.min( tweetData.tweetcount, Math.max(1, parseInt(wheeldelta) + tweetData.position ) ); + tweetData.followLast = (tweetData.position == tweetData.tweetcount); + updateDisplay(); + wheeldelta = 0; + } + return false; + }); + $("#timewindow").mousewheel(function(e, d) { + wheeldelta += d; + if (Math.abs(wheeldelta) >= 1) { + if (wheeldelta > 0) { + setTimeZoom(tweetData.timeLevel + 1); } else { - d += (x1 + 60)+" "+y1+" "+(x2 + 60)+" "+y2; + setTimeZoom(tweetData.timeLevel - 1); } - paper.path(d+" "+x2+" "+y2); + wheeldelta = 0; } - showTimeWindow(); + return false; }); }); \ No newline at end of file diff -r 1a77b44dfa7d -r d60efd677b50 tweetcast/nodejs/conf.js --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tweetcast/nodejs/conf.js Tue Oct 25 18:27:08 2011 +0200 @@ -0,0 +1,40 @@ +/* CONFIGURATION FILE USED BY BOTH CLIENT AND SERVER */ + +var app_port = 8000, + tracking_keyword = "Obama,Bieber", + annotations = { + "default" : { + "colors" : { + "tweet" : "#ffffff", + "timeline" : "#585858" + } + }, + "positive" : { + "keywords" : [ /\+\+/ ], + "colors" : { + "tweet" : "#c5e7cd", + "timeline" : "#1D973D" + } + }, + "negative" : { + "keywords" : [ /\-\-/ ], + "colors" : { + "tweet" : "#f6ced0", + "timeline" : "#CE0A15" + } + }, + "reference" : { + "keywords" : [ /\=\=/ ], + "colors" : { + "tweet" : "#ecedc1", + "timeline" : "#C5A62D" + } + }, + "question" : { + "keywords" : [ /\?\?/ ], + "colors" : { + "tweet" : "#bfdbec", + "timeline" : "#036AAE" + } + } + } \ No newline at end of file diff -r 1a77b44dfa7d -r d60efd677b50 tweetcast/nodejs/node-direct.js --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tweetcast/nodejs/node-direct.js Tue Oct 25 18:27:08 2011 +0200 @@ -0,0 +1,314 @@ +/* CALLING COMMON CONFIGURATION FILE */ + +console.log('Reading Configuration from conf.js'); + +var fs = require('fs'); +eval(fs.readFileSync(__dirname + '/conf.js','utf8')); + +/* SERVER-SIDE ONLY CONFIGURATION */ + +sqlfile = __dirname + '/tweets-' + encodeURIComponent(tracking_keyword) + '.sqlite'; +TWITTER_USER = 'materiauxnum'; +TWITTER_PASS = 'm473r14ux7w337'; +RECORD_NEW_TWEETS = true; + +/* FUNCTIONS */ + +function annotationMap(callback, options) { + var includeDefault = ( options && options.includeDefault ? options.includeDefault : false ); + var returnObject = ( options && options.returnObject ? options.returnObject : false ); + res = (returnObject ? {} : []); + for (var i in annotations) { + if (i != "default" || includeDefault) { + var el = callback(i, annotations[i]) + if (returnObject) { + res[i] = el; + } else { + res.push(el); + } + } + } + return res; +} + +function createTables() { + + var requete = "CREATE TABLE IF NOT EXISTS tweets ( pos INTEGER PRIMARY KEY, tweet_id TEXT UNIQUE, created_at INTEGER, json TEXT" + annotationMap(function(a) { return ', a_' + a + ' INTEGER' }).join("") + " )"; + db.execute(requete, function(err) { + if (err) throw err; + db.execute("CREATE INDEX IF NOT EXISTS idx_created_at ON tweets ( created_at )", function(err) { if (err) throw err; }); + getSendLastPos(); + }); + + db.execute("CREATE TABLE IF NOT EXISTS tweet_refs ( id INTEGER PRIMARY KEY, from_id TEXT, to_id TEXT, ref_type TEXT )", function(err) { if (err) throw err; }); +} + +function commitReference(from_id, to_id, ref_type) { + db.execute( + "INSERT INTO tweet_refs ( from_id, to_id, ref_type ) VALUES ( ?, ?, ? )", + [ from_id, to_id, ref_type ], + function(err) { if (err) throw err; } + ); +} + +function commitTweet(data) { + + var tweet = JSON.parse(data), + ann = []; + + textids(tweet); + for (var j in keys_to_delete) { + delete tweet[keys_to_delete[j]]; + } + textids(tweet.user); + for (var j in user_keys_to_delete) { + delete tweet.user[user_keys_to_delete[j]]; + } + if (tweet.retweeted_status) { + textids(tweet.retweeted_status); + for (var j in keys_to_delete) { + delete tweet.retweeted_status[keys_to_delete[j]]; + } + } + annotationMap(function(i, annotation) { + for (var j in annotation.keywords) { + if (tweet.text.search(annotation.keywords[j]) != -1) { + ann.push(i); + break; + } + } + }); + tweet.annotations = ann; + tweet.created_at = new Date(tweet.created_at); + + if (tweet.in_reply_to_status_id) { + commitReference( tweet.id, tweet.in_reply_to_status_id, "reply" ); + } + if (tweet.retweeted_status) { + commitReference( tweet.id, tweet.retweeted_status.id, "retweet" ); + } + db.execute( + "INSERT INTO tweets ( tweet_id, created_at, json " + + annotationMap(function(a) { return ', a_' + a }).join("") + + " ) VALUES ( ?, ?, ? " + + annotationMap(function(a) { return ', ?' }).join("") + + " )", + [ tweet.id, tweet.created_at.valueOf(), JSON.stringify(tweet) ].concat(annotationMap(function(a) { return ann.indexOf(a) == -1 ? 0 : 1 })), + function(err) { + if (err) throw err; + getSendLastPos(); + } + ); +} + +function callBackNewTweets(chunk) { + var newdata = chunk.split('\r\n'); + for (var i in newdata) { + if (newdata[i].length > 0) { + commitTweet(newdata[i]); + } + } + console.log("New tweets received"); +} + +function getSendLastPos() { + db.execute("SELECT MAX(pos) lastpos FROM tweets", function (err, results) { + if (err) throw err; + lastpos = results[0].lastpos ? results[0].lastpos : 0; + console.log("Broadcasting last pos = ",lastpos); + io.sockets.emit('tweetSummary', { + tweetcount : lastpos + }); + }); +} + + +function getSendTweets(posList, socket) { + console.log("request for tweets ("+posList.join(',')+") from "+socket.id); + db.execute("SELECT * FROM tweets WHERE pos IN ( " + posList.join(',') + " )", function (err, results) { + if (err) throw err; + socket.emit('tweets', + results.map(function(line) { + var tw = JSON.parse(line.json); + tw.pos = line.pos; + return tw; + }) + ); + }); +} + +function getSendTimeline(level, socket) { + console.log("request for timeline ("+level+") from "+socket.id); + var lvl = date_levels[level], + requete = "SELECT COUNT(*) AS nb, " + + lvl + + "*ROUND(created_at/" + + lvl + + ") AS tranche" + + annotationMap(function (a) { return " , SUM(a_" + a + ") AS s_" + a }).join("") + + " FROM tweets GROUP BY tranche ORDER BY tranche DESC LIMIT 0,50"; + db.execute(requete, function (err, results) { + if (err) throw err; + var tbl = [], + lastend = parseInt(results[results.length - 1].tranche); + for (var i = results.length - 1; i >= 0; i--) { + var start = parseInt(results[i].tranche); + while (start > lastend) { + var struct = { "start": lastend, "tweets" : 0, "annotations" : {} }; + lastend += lvl; + struct.end = lastend; + tbl.push(struct); + } + lastend += lvl; + var struct = { + "start" : start, + "end" : lastend, + "tweets" : results[i].nb, + "annotations" : annotationMap(function (a) { + return results[i]['s_'+a]; + },{returnObject: true}) + } + tbl.push(struct); + } + socket.emit('timeline', tbl); + }); +} + +function textids(object) { + for (var key in object) { + if (key.substr(-2) == 'id') { + object[key] = object[key + '_str']; + delete object[key + '_str']; + } + } +} + +function httpHandler(req, res) { + console.log("HTTP Request for URL "+req.url); + var url = __dirname + ( req.url == "/conf.js" ? "" : "/client" ) + req.url + ( req.url[req.url.length - 1] == "/" ? "index.html" : "" ); + fs.readFile( url, function(err, data) { + if (err) { + console.log("Error 404"); + res.writeHead(404); + return res.end('File not found'); + } + res.writeHead(200); + res.end(data); + }); +} + +/* Initialization */ + +var http = require('http'), + https = require('https'), + sqlite = require('sqlite'), + socketio = require('socket.io'), + tweets = [], + lastpos = 0, + arcs = [], + tweet_ids = [], + date_struct = [], + date_levels = [ + 3600 * 1000, + 15 * 60 * 1000, + 5 * 60 * 1000, + 60 * 1000, + 15 * 1000 + ], + keys_to_delete = [ + 'in_reply_to_screen_name', + 'in_reply_to_user_id', + 'retweeted', + 'place', + 'geo', + 'source', + 'contributors', + 'coordinates', + 'retweet_count', + 'favorited', + 'truncated', + 'possibly_sensitive' + ], + user_keys_to_delete = [ + 'default_profile_image', + 'show_all_inline_media', + 'contributors_enabled', + 'profile_sidebar_fill_color', + 'created_at', + 'lang', + 'time_zone', + 'profile_sidebar_border_color', + 'follow_request_sent', + 'profile_background_image_url', + 'profile_background_image_url_https', + 'followers_count', + 'description', + 'url', + 'geo_enabled', + 'profile_use_background_image', + 'default_profile', + 'following', + 'profile_text_color', + 'is_translator', + 'favourites_count', + 'listed_count', + 'friends_count', + 'profile_link_color', + 'protected', + 'location', + 'notifications', + 'profile_image_url_https', + 'statuses_count', + 'verified', + 'profile_background_color', + 'profile_background_tile', + 'utc_offset' + ], + app = http.createServer(httpHandler), + io = socketio.listen(app), + db = new sqlite.Database(); + +/* MAIN CODE */ + +app.listen(app_port); +console.log("Listening on port: "+app_port); +console.log("Opening SQLITE file: "+sqlfile); +db.open(sqlfile , function(err) { + if (err) throw err; + createTables(); +}); + +if (RECORD_NEW_TWEETS) { + console.log("Requesting Twitter to track keyword(s): "+tracking_keyword); + var req = https.request({ + host: "stream.twitter.com", + path: "/1/statuses/filter.json", + method: "POST", + headers: { + 'Authorization': 'Basic ' + new Buffer( TWITTER_USER + ":" + TWITTER_PASS ).toString('base64'), + 'Content-Type': 'application/x-www-form-urlencoded' + } + }, function(res) { + console.log('Reply from stream.twitter.com: ' + res.statusCode); + console.log('Headers: ' + JSON.stringify(res.headers)); + res.setEncoding('utf8'); + res.on('data', callBackNewTweets); + }); + + req.write('track=' + encodeURIComponent(tracking_keyword)); + req.end(); +} + +io.set('log level', 0); +io.sockets.on('connection', function(socket) { + console.log("New connection from" + socket.handshake.address.address + " with id=" + socket.id); + socket.emit('tweetSummary', { tweetcount : tweets.length }); + socket.on('updateTweets', function(data) { + if (data.tweets.length) { + getSendTweets(data.tweets, socket); + } + }); + socket.on('updateTimeline', function(data) { + getSendTimeline(data.level, socket); + }); +}); \ No newline at end of file diff -r 1a77b44dfa7d -r d60efd677b50 tweetcast/nodejs/server/node-direct.js --- a/tweetcast/nodejs/server/node-direct.js Tue Oct 25 18:26:39 2011 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,265 +0,0 @@ -READ_OLD_TWEETS = true; -RECORD_NEW_TWEETS = true; -TWEET_FILE_DIR = './'; -TWEET_FILE_START = 'tweets-'; -TWEET_FILE_EXT = '.txt'; -TRACKING_KEYWORD = '#p2'; - -var fs = require('fs'), - https = require('https'), - io = require('socket.io') - .listen(8000), - tweets = [], - arcs = [], - tweet_ids = [], - date_struct = [], - date_levels = [ - 3600 * 1000, - 15 * 60 * 1000, - 5 * 60 * 1000, - 60 * 1000 - ], - annkw = { - 'positive' : '++', - 'negative' : '--', - 'reference' : '==', - 'question' : '??' - }; - -function populateDateStruct(level, start) { - if (typeof start == "object") { - start = start.valueOf(); - } - var end = start + date_levels[level], - struct = { - "level" : level, - "start" : new Date(start), - "end" : new Date(end) - }; - if (level < date_levels.length - 1) { - struct.slices = []; - var newstart = start; - while (newstart < end) { - struct.slices.push(populateDateStruct(level + 1, newstart)); - newstart += date_levels[level + 1]; - } - } else { - struct.tweets = []; - struct.annotations = {}; - } - return struct; -} - -function insertIntoDateStruct(slices, tweet) { - var creadate = new Date(tweet.created_at); - for (var i in slices) { - if (creadate < slices[i].end) { - if (slices[i].slices) { - insertIntoDateStruct(slices[i].slices, tweet); - } else { - slices[i].tweets.push(tweet.pos); - if (tweet.annotations.length) { - var ann = tweet.annotations[0]; - if (slices[i].annotations[ann]) { - slices[i].annotations[ann].push(tweet.pos); - } else { - slices[i].annotations[ann] = [ tweet.pos ]; - } - } - } - break; - } - } -} - -function getSliceContent(slice) { - if (slice.slices) { - var twz = [], - annotations = {}; - for (var i in slice.slices) { - var data = getSliceContent(slice.slices[i]); - twz = twz.concat(data.tweets); - for (var j in data.annotations) { - if (annotations[j]) { - annotations[j] = annotations[j].concat(data.annotations[j]); - } else { - annotations[j] = data.annotations[j]; - } - } - } - } else { - twz = slice.tweets; - annotations = slice.annotations; - } - return { "tweets" : twz, "annotations" : annotations } -}; - -function flattenDateStruct(slices, target_level) { - var current_level = slices[0].level, - result = []; - if (current_level < target_level) { - if (slices[0].slices) { - for (var i in slices) { - result = result.concat(flattenDateStruct(slices[i].slices, target_level)); - } - } - } - else { - for (var i in slices) { - var data = getSliceContent(slices[i]); - result.push({ - "start" : slices[i].start, - "end" : slices[i].end, - "tweets" : data.tweets, - "annotations" : data.annotations - }); - } - } - return result; -} - -function trimFDS(slices) { - while (slices[0].tweets.length == 0) { - slices.splice(0,1); - } - while (slices[slices.length - 1].tweets.length == 0) { - slices.pop(); - } - return slices; -} - -function addToList(tweet) { - if (tweet_ids.indexOf(tweet.id) != -1) { - console.log("Error: Tweet already in list"); - return; - } - tweet.pos = tweets.length; - tweets.push(tweet); - tweet_ids.push(tweet.id); - var creadate = new Date(tweet.created_at); - if (!date_struct.length) { - date_struct = [ populateDateStruct(0, date_levels[0] * parseInt(creadate / date_levels[0])) ] - } - while (creadate > date_struct[date_struct.length - 1].end) { - date_struct.push( populateDateStruct(0, date_struct[date_struct.length - 1].end.valueOf()) ); - } - insertIntoDateStruct(date_struct, tweet); - if (tweet.in_reply_to_status_id) { - var ref = tweet_ids.indexOf(tweet.in_reply_to_status_id) - if (ref != -1) { - arcs.push({ - "from" : tweet.pos, - "to" : ref, - "type" : "reply" - }); - } - } - if (tweet.retweeted_status) { - var ref = tweet_ids.indexOf(tweet.retweeted_status.id_str) - if (ref != -1) { - arcs.push({ - "from" : tweet.pos, - "to" : ref, - "type" : "retweet" - }); - } - } -} - -function textids(object) { - for (var key in object) { - if (key.substr(-2) == 'id') { - object[key] = object[key + '_str']; - delete object[key + '_str']; - } - } -} - -function readTweetsFromFile() { - var dir = fs.readdirSync(TWEET_FILE_DIR), - fileName = null; - for (var i in dir) { - if ( dir[i].indexOf( TWEET_FILE_START + encodeURIComponent(TRACKING_KEYWORD) ) == 0 ) { - var oldtweets = fs.readFileSync(dir[i], 'utf8').split('\n'), - tweetscopied = 0; - for (var j in oldtweets) { - if (oldtweets[j].length > 1) { - addToList(JSON.parse(oldtweets[j])); - tweetscopied++; - } - } - console.log(tweetscopied, "tweets copied from", dir[i]); - console.log(arcs); - } - } -} - -function callBackNewTweets(chunk) { - var newdata = chunk.split('\r\n'); - for (var i in newdata) { - if (newdata[i].length > 0) { - var tweet = JSON.parse(newdata[i]), - annotations = []; - for (var a in annkw) { - if (tweet.text.indexOf(annkw[a]) != -1) { - annotations.push(a); - } - } - tweet.annotations = annotations; - textids(tweet); - textids(tweet.user); - - addToList(tweet); - var txt = JSON.stringify(tweet)+'\n'; - writestream.write(txt); - } - } - console.log("New tweets received. We now have", tweets.length, "tweets in memory"); - io.sockets.emit('tweetSummary', { tweetcount : tweets.length }); -} - -/* MAIN CODE */ - -if (READ_OLD_TWEETS) { - readTweetsFromFile(); -} - -if (RECORD_NEW_TWEETS) { - var now = new Date(), - fileTimestamp = (1900+now.getYear())+"-"+("0"+(1+now.getMonth())).slice(-2)+"-"+("0"+now.getDate()).slice(-2)+"."+("0"+now.getHours()).slice(-2)+"-"+("0"+now.getMinutes()).slice(-2), - fileName = TWEET_FILE_DIR + TWEET_FILE_START + encodeURIComponent(TRACKING_KEYWORD) + '-' + fileTimestamp + TWEET_FILE_EXT, - writestream = fs.createWriteStream(fileName, { flags: 'w', encoding: 'utf-8' }); - console.log('Writing to',fileName); - var req = https.request({ - host: "stream.twitter.com", - path: "/1/statuses/filter.json", - method: "POST", - headers: { - 'Authorization': 'Basic cmFwaHY6N3czMzdMZkMyM2dF', - 'Content-Type': 'application/x-www-form-urlencoded' - } - }, function(res) { - console.log('STATUS: ' + res.statusCode); - console.log('HEADERS: ' + JSON.stringify(res.headers)); - res.setEncoding('utf8'); - res.on('data', callBackNewTweets); - }); - - req.write('track=' + encodeURIComponent(TRACKING_KEYWORD)); - req.end(); -} - -io.set('log level', 0); -io.sockets.on('connection', function(socket) { - console.log("New connection from", socket.handshake.address.address,"id :", socket.id); - socket.emit('tweetSummary', { tweetcount : tweets.length }); - socket.on('getTweets', function(data) { - console.log('getTweets from',socket.id); - data.tweets = tweets.slice(Math.max(0, data.from), data.to); - socket.emit('tweets', data); - }); - socket.on('getTimeline', function(data) { - socket.emit('timeline', { "timeline" : trimFDS(flattenDateStruct(date_struct, data.level)), "arcs" : arcs }); - console.log('getTimeline from',socket.id); - }); -}); \ No newline at end of file