# HG changeset patch
# User Raphael Velt <raph.velt@gmail.com>
# Date 1326796558 -3600
# Node ID 65f3ce59d8b75a59bf8cbba7fed39123100f87f0
# Parent  1e091ac28dd9d75d8115feefe9819e197a54e0b6
New live PT client now uses client-side connection

diff -r 1e091ac28dd9 -r 65f3ce59d8b7 web/enmi2011/config.php
--- a/web/enmi2011/config.php	Mon Jan 09 13:06:23 2012 +0100
+++ b/web/enmi2011/config.php	Tue Jan 17 11:35:58 2012 +0100
@@ -107,65 +107,23 @@
 	
     // New Tweet live parameters
     'knowtex_url' => "http://labs.knowtex.com/enmi2011/?mapOnly#carto",
-    'use_tweetcast' => false,
+    'use_tweetcast' => true,
     'js_config' => "
-    var connect_type = 'gevent',
-    source_address = 'http://polemictweet.com/tweetcast-proxy.php',
-    tracking_keywords = [ \"#enmi\" ],
+    tracking_keywords = [ '#enmi' ],
     suggested_keywords = [
-        \"confiance\",
-        \"croyance\",
-        \"crédit\",
-        \"trace\",
-        \"foi\",
-        \"risque\",
-        \"assurance\",
-        \"démocratie\",
-        \"expertise\",
-        \"catastrophe\",
-        \"transparence\",
-        \"politique\"
-    ],
-    annotations = {
-        \"default\" : {
-            \"colors\" : {
-                \"h\" : 0,
-                \"s\" : 0
-            }
-        },
-        \"positive\" : {
-            \"display_name\" : \"++\",
-            \"keywords\" : [ /\\+\\+/ ],
-            \"colors\" : {
-                \"h\" : .3,
-                \"s\" : .65
-            }
-        },
-        \"negative\" : {
-            \"display_name\" : \"--\",
-            \"keywords\" : [ /\\-\\-/ ],
-            \"colors\" : {
-                \"h\" : 0,
-                \"s\" : .8
-            }
-        },
-        \"reference\" : {
-            \"display_name\" : \"==\",
-            \"keywords\" : [ /\\=\\=/ ],
-            \"colors\" : {
-                \"h\" : .16,
-                \"s\" : .8
-            }
-        },
-        \"question\" : {
-            \"display_name\" : \"??\",
-            \"keywords\" : [ /\\?\\?/ ],
-            \"colors\" : {
-                \"h\" : .6,
-                \"s\" : .8
-            }
-        }
-    }"
+        'confiance',
+        'croyance',
+        'crédit',
+        'trace',
+        'foi',
+        'risque',
+        'assurance',
+        'démocratie',
+        'expertise',
+        'catastrophe',
+        'transparence',
+        'politique'
+    ]"
     
     
 );
\ No newline at end of file
diff -r 1e091ac28dd9 -r 65f3ce59d8b7 web/res/js-tweetcast/live-polemic.js
--- a/web/res/js-tweetcast/live-polemic.js	Mon Jan 09 13:06:23 2012 +0100
+++ b/web/res/js-tweetcast/live-polemic.js	Tue Jan 17 11:35:58 2012 +0100
@@ -1,3 +1,50 @@
+if (typeof annotations == "undefined" || !annotations) {
+    var annotations = {
+        "default" : {
+            "colors" : {
+                "h" : 0,
+                "s" : 0
+            }
+        },
+        "positive" : {
+            "display_name" : "++",
+            "keywords" : [ /\+\+/ ],
+            "colors" : {
+                "h" : .3,
+                "s" : .65
+            }
+        },
+        "negative" : {
+            "display_name" : "--",
+            "keywords" : [ /\-\-/ ],
+            "colors" : {
+                "h" : 0,
+                "s" : .8
+            }
+        },
+        "reference" : {
+            "display_name" : "==",
+            "keywords" : [ /\=\=/ ],
+            "colors" : {
+                "h" : .16,
+                "s" : .8
+            }
+        },
+        "question" : {
+            "display_name" : "??",
+            "keywords" : [ /\?\?/ ],
+            "colors" : {
+                "h" : .6,
+                "s" : .8
+            }
+        }
+    }    
+}
+
+if (typeof max_pages == "undefined" || !max_pages) {
+    max_pages = 5;
+}
+
 var socket,
     tlPaper,
     twCx = {
@@ -28,8 +75,49 @@
     lastScrollPos = 0,
     rx_url = /https?:\/\/[0-9a-zA-Z\.%\/-_]+/g,
     rx_word = /[^ \.&;,'"!\?\d\(\)\+\[\]\\\…\-«»:\/]{3,}/g,
-    stop_list = [ 'and', 'avec', 'aux', 'car', 'comme', 'dans', 'donc', 'des', 'elle', 'est', 'être', 'eux', 'ils', 'les', 'leur', 'leurs', 'mes', 'mon', 'tes', 'ton', 'notre', 'nos', 'nous', 'ont', 'pas', 'que', 'qui', 'sont', 'the', 'une', 'votre', 'vos', 'vous' ],
-    l10n = { "rechercher" : "Rechercher" };
+    stop_list = [ 'and', 'avec', 'aux', 'car', 'comme', 'dans', 'donc', 'des', 'elle', 'est', 'être', 'eux', 'ils', 'les', 'leur', 'leurs', 'mes', 'mon', 'tes', 'ton', 'notre', 'nos', 'nous', 'ont', 'par', 'pas', 'que', 'qui', 'sont', 'sur', 'the', 'une', 'votre', 'vos', 'vous' ];
+
+function getTweets(options) {
+    
+    function getTweetUrl(url) {
+        $.getJSON(url, function(data) {
+            options.tweets = options.tweets.concat(data.results);
+            options.currentPage++;
+            if (options.cbData) {
+                options.cbData();
+            }
+            var _isLast = true;
+            if (data.results && data.results.length) {
+                var _oldestTweetId = data.results[data.results.length - 1].id_str,
+                    _maxId = _oldestTweetId;
+                if (options.currentPage < options.pages) {
+                    _isLast = false;
+                    getTweetUrl(baseurl + firstparams + '&max_id=' + _maxId + lastparams);
+                }
+            }
+            
+            if (_isLast) {
+                    options.tweets.sort(function(a,b) {
+                       return a.id - b.id; 
+                    });
+                    if (options.cbEnd) {
+                        options.cbEnd();
+                    }
+            }
+        });
+    }
+    
+    options.tweets = [];
+    options.pages || (options.pages = 1);
+    options.rpp || (options.rpp = 100);
+    options.currentPage = 0;
+    var baseurl = "http://search.twitter.com/search.json",
+        firstparams = "?q=" + encodeURIComponent(options.keyword)+ "&rpp=" + options.rpp
+            + (options.lang ? "&lang=" + options.lang : '' ),
+        lastparams = (options.since_id ? "&since_id=" + options.since_id : '' ) + "&callback=?",
+        jsonurl = baseurl + firstparams + lastparams;
+    getTweetUrl(jsonurl);
+}
 
 function getColor(annotation, lum) {
     return Raphael.hsl2rgb(annotations[annotation].colors.h, annotations[annotation].colors.s, lum);
@@ -56,6 +144,10 @@
 }
 
 function addTweet(tweet) {
+    if (!tweet) {
+        console.log(tweet);
+        return;
+    }
     function backRef(source_id, target_id, type) {
         var target = tweetById(target_id);
         if (target) {
@@ -71,24 +163,67 @@
         }
     }
     
+    _(['id', 'from_user_id', 'in_reply_to_status_id']).each(function(_i) {
+       tweet[_i] = tweet[_i + '_str'];
+       delete tweet[_i + '_str']; 
+    });
+    
     if (twCx.idIndex.indexOf(tweet.id) != -1) {
         return;
     }
     
-    var txt_date = tweet.created_at;
-    if (navigator.userAgent.search(/MSIE/) != -1) {
-        txt_date = txt_date.replace(/( \+)/, ' UTC$1');
-    }
-    tweet.date_value = Date.parse(txt_date);
+    tweet.entities = []
     
-    var tab = tweet.text.match(/\&\#[0-9]+\;/g);
-    for (var i in tab) {
-        var n = parseInt(tab[i].substr(2));
-        if (n != NaN) {
-            tweet.text = tweet.text.replace(tab[i], String.fromCharCode(n));
+    var _users = tweet.text.match(/@[\w_]+/gm),
+        _lastpos = 0;
+    if (_users) {
+        for (var _i = 0; _i < _users.length; _i++) {
+            var _m = _users[_i],
+                _start = _lastpos + tweet.text.substr(_lastpos).search(_m),
+                _lastpos = _start + _m.length;
+            tweet.entities.push({
+                "text" : _m,
+                "start" : _start,
+                "end" : _lastpos,
+                "link" :'<a href="http://twitter.com/' + _m.substr(1) + '" onclick="filtrerTexte(\'' + _m + '\'); return false;" target="_blank">',
+            });
         }
     }
     
+    var _hashes = tweet.text.match(/([^&]|^)#[^\s,.!?=#@&;()]+/gm),
+        _lastpos = 0;
+    if (_hashes) {
+        for (var _i = 0; _i < _hashes.length; _i++) {
+            var _m = _hashes[_i],
+                _h = ( _m[0] == '#' ? _m : _m.substr(1) ),
+                _start = _lastpos + tweet.text.substr(_lastpos).search(_h),
+                _lastpos = _start + _h.length;
+            tweet.entities.push({
+                "text" : _h,
+                "start" : _start,
+                "end" : _lastpos,
+                "link" :'<a href="http://twitter.com/search?q=' + encodeURIComponent(_h) + '" onclick="filtrerTexte(\'' + _.escape(_h) + '\'); return false;" target="_blank">',
+            });
+        }
+    }
+    
+    var _urls = tweet.text.match(/(www.|https?:\/\/)[\w./_\-]+/gim),
+        _lastpos = 0;
+    if (_urls) {
+        for (var _i = 0; _i < _urls.length; _i++) {
+            var _m = _urls[_i],
+                _start = _lastpos + tweet.text.substr(_lastpos).search(_m),
+                _lastpos = _start + _m.length;
+            tweet.entities.push({
+                "text" : _m,
+                "start" : _start,
+                "end" : _lastpos,
+                "link" :'<a href="' + _m + '" target="_blank">',
+            });
+        }
+    }
+    tweet.date_value = Date.parse(tweet.created_at.replace(/(\+|-)/,'UTC$1'));
+    
     var ann = [];
     for (var j in annotations) {
         if (j != "default") {
@@ -105,10 +240,19 @@
     if (tweet.in_reply_to_status_id) {
         backRef( tweet.id, tweet.in_reply_to_status_id, "reply" );
     }
-    if (tweet.retweeted_status) {
-        backRef( tweet.id,  tweet.retweeted_status.id, "retweet" );
+    var _retweet = tweet.text.match(/RT @[\w_]+:? /)
+    if (_retweet) {
+        var _user = _retweet[0].match(/@[\w_]+/)[0].substr(1).toLowerCase(),
+            _originalText = tweet.text.substr(tweet.text.search(_retweet[0]) + _retweet[0].length);
+        for (var i = 0; i < twCx.tweets.length; i++) {
+            if (twCx.tweets[i].from_user && twCx.tweets[i].from_user.toLowerCase() == _user && twCx.tweets[i].text == _originalText) {
+                tweet.retweeted_status_id = twCx.tweets[i].id;
+                backRef( tweet.id, twCx.tweets[i].id, "retweet" );
+                break;
+            }
+        }
     }
-    
+     
     var localWords = []
     
     var tab = tweet.text.replace(rx_url,'').match(rx_word);
@@ -258,13 +402,6 @@
     goToPos( delta + twCx.currentIdIndex.indexOf(twCx.position) );
 }
 
-function ktxHighlightTweet(tweet) {
-    var _gF = $("#graphFrame");
-    if ( _gF.is(":visible") && _gF.attr("src").indexOf("labs.knowtex.com") != -1 ) {
-        _gF.attr("src",'http://labs.knowtex.com/enmi2011/?mapOnly#carto::user=' + tweet.user.screen_name + '&time=' + ~~(tweet.date_value/1000));
-    }
-}
-
 function tweetToHtml(tweet, className, elName) {
     
     function highlight(texte) {
@@ -292,54 +429,24 @@
         html += '</div>';
     }
     html += '<div class="twmain">';
-    var a_user = '<a href="http://twitter.com/' + tweet.user.screen_name + '"  onclick="filtrerTexte(\'@' + tweet.user.screen_name + '\'); return false;" target="_blank">';
-    html += '<div class="around_img"><img class="profile_image" src="' + tweet.user.profile_image_url + '" />';
+    var a_user = '<a href="http://twitter.com/' + tweet.from_user + '"  onclick="filtrerTexte(\'@' + tweet.from_user + '\'); return false;" target="_blank">';
+    html += '<div class="around_img"><img class="profile_image" src="' + tweet.profile_image_url + '" />';
     if (className == 'full') {
         html += '<p class="created_at">' + new Date(tweet.date_value).toTimeString().substr(0,8) + '</a></p>';
     }
     html += '</div>';
     if (className != 'icons') {
         lastend = 0;
-        var txt = '',
-            entities = [];
-        for (var i in tweet.entities.hashtags) {
-            entities.push({
-                "start" : tweet.entities.hashtags[i].indices[0],
-                "end" : tweet.entities.hashtags[i].indices[1],
-                "link" : '<a href="http://twitter.com/search?q=%23' + tweet.entities.hashtags[i].text + '" onclick="filtrerTexte(\'#' + tweet.entities.hashtags[i].text + '\'); return false;" target="_blank">',
-                "text" : '#' + tweet.entities.hashtags[i].text
-            });
-        }
-        for (var i in tweet.entities.urls) {
-            var linkurl = ( tweet.entities.urls[i].expanded_url ? tweet.entities.urls[i].expanded_url : tweet.entities.urls[i].url ),
-                dispurl = linkurl.replace(/https?:\/\//,'');
-            if (linkurl.search(/https?:\/\//) == -1) {
-                linkurl = 'http://' + linkurl;
-            }
-            entities.push({
-                "start" : tweet.entities.urls[i].indices[0],
-                "end" : tweet.entities.urls[i].indices[1],
-                "link" : '<a href="' + linkurl  + '" target="_blank">',
-                "text" : dispurl
-            });
-        }
-        for (var i in tweet.entities.user_mentions) {
-            entities.push({
-                "start" : tweet.entities.user_mentions[i].indices[0],
-                "end" : tweet.entities.user_mentions[i].indices[1],
-                "link" : '<a href="http://twitter.com/' + tweet.entities.user_mentions[i].screen_name + '" onclick="filtrerTexte(\'@' + tweet.entities.user_mentions[i].screen_name + '\'); return false;" target="_blank">',
-                "text" : '@' + tweet.entities.user_mentions[i].screen_name
-            });
-        }
-        entities.sort(function(a, b) { return a.start - b.start });
-        for (var i in entities) {
-            txt += highlight( tweet.text.substring(lastend, entities[i].start) ) + entities[i].link + highlight( entities[i].text ) + '</a>';
-            lastend = entities[i].end;
-        }
+        var txt = '';
+        tweet.entities.sort(function(a, b) { return a.start - b.start });
+        _(tweet.entities).each(function(_e) {
+            txt += highlight( tweet.text.substring(lastend, _e.start) ) + _e.link + highlight( _e.text ) + '</a>';
+            lastend = _e.end;
+        });
         txt += highlight( tweet.text.substring(lastend) );
-        html += '<p class="tweet_text"><b>' + a_user + highlight('@' + tweet.user.screen_name) + '</a>' + ( className == 'full' ? ' (' + tweet.user.name + ')</b><br />' : '</b> : ') + txt + '</p>';
+        html += '<p class="tweet_text"><b>' + a_user + highlight('@' + tweet.from_user) + '</a>' + ( className == 'full' ? ' (' + tweet.from_user_name + ')</b><br />' : '</b> : ') + txt + '</p>';
         if (className == 'full' && el == 'li') {
-            html += '<div class="tweet_actions"><a href="http://twitter.com/' + tweet.user.screen_name + '/status/' + tweet.id + '" onclick="tweetPopup(this.href); return false;" target="_blank">afficher tweet</a> - ';
+            html += '<div class="tweet_actions"><a href="http://twitter.com/' + tweet.from_user + '/status/' + tweet.id + '" onclick="tweetPopup(this.href); return false;" target="_blank">afficher tweet</a> - ';
             html += '<a href="http://twitter.com/intent/tweet?in_reply_to=' + tweet.id + '" onclick="tweetPopup(this.href); return false;" target="_blank">répondre</a> · ';
             html += '<a href="http://twitter.com/intent/retweet?tweet_id=' + tweet.id + '" onclick="tweetPopup(this.href); return false;" target="_blank">retweeter</a> · ';
             html += '<a href="http://twitter.com/intent/favorite?tweet_id=' + tweet.id + '" onclick="tweetPopup(this.href); return false;" target="_blank">favori</a></div>';
@@ -408,7 +515,6 @@
     if (!t) {
         return;
     }
-    ktxHighlightTweet(t);
     var p = tlPosTweet(t, annotation);
     if (!p) {
         return;
@@ -449,8 +555,8 @@
         }
     }
     
-    if (tweet.retweeted_status) {
-        var t = tweetById(tweet.retweeted_status.id);
+    if (tweet.retweeted_status_id) {
+        var t = tweetById(tweet.retweeted_status_id);
         if (t) {
             tweetAndArc(pos, tlPosTweet(t));
         }
@@ -538,7 +644,7 @@
     }
     if (twCx.filtre) {
         var tweets = twCx.tweets.filter(function(tweet) {
-            var mention = '@' + tweet.user.screen_name;
+            var mention = '@' + tweet.from_user;
             return ( tweet.text.search(twCx.filtre) != -1 ) || ( mention.search(twCx.filtre) != -1 );
         });
         $("#inp_q").val(twCx.filtreTexte + ' (' + tweets.length + ' tweets)');
@@ -889,7 +995,7 @@
         if (twCx.filtre) {
             inpq.attr("class", "rechercheCourante").val(twCx.filtreTexte);
         } else {
-            inpq.attr("class", "greyed").val(l10n.rechercher);
+            inpq.attr("class", "greyed").val("Rechercher");
         }
     }
 }
@@ -912,6 +1018,39 @@
     $("#time_zoomin").attr("class",(twCx.timeLevel == twCx.date_levels.length - 1 ? "inactive" : ""));
 }
 
+function saveJSON() {
+    var _txt = JSON.stringify(twCx.tweets),
+        _buf = '';
+    for (var i = 0; i < _txt.length; i++) {
+        var _n = _txt.charCodeAt(i);
+        if (_n > 127) {
+            var _h = _n.toString(16);
+            while (_h.length < 4) {
+                _h = '0' + _h;
+            }
+            _buf += '\\u' + _h;
+        } else {
+            _buf += _txt.charAt(i);
+        }
+    }
+    document.location.href = "data:text/json;base64," + btoa(_buf);
+}
+
+function saveCSV() {
+    function csvEncode(tableau) {
+      return tableau.map(function(el) {
+          return '"' + unescape(encodeURIComponent(el)).replace(/"/gm, '""') + '"';
+      }).join(",")
+    };
+    var _csvfields = [ "id", "from_user", "from_user_name", "created_at", "text" ],
+        _csvtxt = csvEncode(_csvfields) + "\n" + twCx.tweets.map(function(tw) {
+            return csvEncode(_csvfields.map(function(field) {
+                return tw[field];
+            }));
+        }).join("\n");
+    document.location.href = "data:text/csv;base64," + btoa(_csvtxt);
+}
+
 $(document).ready(function() {
     tlPaper = Raphael("timeline", twCx.tlWidth, twCx.tlHeight);
     
@@ -1034,51 +1173,51 @@
     
     setInterval(function() {
         var sc = $("#scrollcont");
-        if (sc.scrollTop() != lastScrollPos && twCx.tweets) {
+        if (sc.scrollTop() != lastScrollPos && twCx.tweets && twCx.currentIdIndex) {
             var p = Math.floor( twCx.currentIdIndex.length * ( 1 - sc.scrollTop() / scrollExtent ) );
             goToPos(p);
         }
-        
     }, 100)
 });
 
-if (connect_type == "socketio") {
-    document.write('<script type="text/javascript" src="' + source_address + '/socket.io/socket.io.js"><' + '/script>');
-}
-
 function connectTweets() {
-    switch (connect_type) {
-        case "socketio" :
-            socket = io.connect( source_address );
-            socket.on("initial_data", function(data) {
-                loadTweets(data.tweets)
-            });
-            socket.on("update", function(data) {
-                if (!twCx.tweets) {
-                    return;
-                }
-                loadTweets(data.new_tweets, true);
-            });
-            break;
-            
-        case "gevent" :
-            $.getJSON( source_address + '?callback=?', function(data) {
-                loadTweets(data.tweets)
-            } );
-            
+    tlPaper.clear();
+    var _sq = tlPaper.rect(0, twCx.tlHeight, twCx.tlWidth, 0)
+        .attr({
+            "stroke" : "none",
+            "fill" : "#8080cc"
+        });
+    var _lb = tlPaper.text(twCx.tlWidth / 2, twCx.tlHeight / 2, "0 tweet")
+        .attr({
+            "font-size" : "20px",
+            "text-anchor" : "middle"
+        });
+    
+    getTweets({
+        "keyword" : tracking_keywords.join(" OR "),
+        "pages" : max_pages,
+        "rpp" : 100,
+        "cbData" : function() {
+            _lb.attr("text", (this.tweets.length - this.currentPage + 1) + " tweets");
+            var _h = twCx.tlHeight * this.currentPage / this.pages;
+            _sq.animate({
+                "y" : twCx.tlHeight - _h,
+                "height" : _h
+            })
+        },
+        "cbEnd" : function() {
+            loadTweets(this.tweets);
             setInterval(function() {
-                if (!twCx.tweets) {
-                    return;
-                }
-                $.getJSON( source_address + '/?from=' + twCx.tweets.length + '&callback=?', function(data) {
-                    loadTweets(data.tweets, true);
+                getTweets({
+                    "keyword" : tracking_keywords.join(" OR "),
+                    "pages" : 1,
+                    "since_id" : twCx.idIndex[twCx.idIndex.length - 1],
+                    "rpp" : 100,
+                    "cbEnd" : function() {
+                        loadTweets(this.tweets, true);
+                    }
                 });
-            }, 5000);
-            break;
-        
-        case "standalone" :
-            $.getScript( source_address );
-    }
-}
-
-
+            }, 20000)
+        }
+    });
+}
\ No newline at end of file