# HG changeset patch
# User Raphael Velt <raph.velt@gmail.com>
# Date 1320419552 -3600
# Node ID 19ed2a92d3c12e8e17d261a105f5683d72e67913
# Parent  9cabcd2709be09529830700067fc65ddd6ddab60# Parent  ef3ff87f98fde316a1d7e7fa01b07ee76ec398cb
Merge

diff -r ef3ff87f98fd -r 19ed2a92d3c1 tweetcast/nodejs/client/js/script.js
--- a/tweetcast/nodejs/client/js/script.js	Fri Nov 04 11:04:21 2011 +0100
+++ b/tweetcast/nodejs/client/js/script.js	Fri Nov 04 16:12:32 2011 +0100
@@ -63,7 +63,7 @@
     if (!tweet) {
         return placeHolder(className);
     }
-	html = '<li class="tweet ' + className + '" id="tweet_' + tweet.pos + '" onclick="return clicTweet(' + tweet.pos + ')"';
+	var html = '<li class="tweet ' + className + '" id="tweet_' + tweet.pos + '" onclick="return clicTweet(' + tweet.pos + ')"';
 	if (tweetData.followLast && tweet.pos == tweetData.position) {
 	    html += ' style="display: none"';
 	}
@@ -76,15 +76,19 @@
     	html += '</div>';
 	}
 	html += '<div class="twmain">';
-	a_user = '<a href="http://twitter.com/' + tweet.user.screen_name + '" target="_blank" title="' + tweet.user.name + '">';
+	a_user = '<a href="http://twitter.com/' + tweet.user.screen_name + '" var target="_blank" title="' + tweet.user.name + '">';
 	if (tweet.user.profile_image_url) {
 		html += a_user + '<img class="profile_image" src="' + tweet.user.profile_image_url + '" /></a>';
 	}
 	html += '<h4>' + a_user + '@' + tweet.user.screen_name + '</a></h4><p class="created_at">' + new Date(tweet.created_at).toLocaleTimeString() + '</p><p class="tweet_text">';
 	lastend = 0;
-	txt = tweet.text.replace(/(https?:[0-9a-zA-Z\/\-_\.~&?=]+)/g,'<a href="$1" target="_blank">$1</a>').replace(/@([0-9a-zA-Z_]+)/g,'<a href="http://twitter.com/$1" target="_blank">@$1</a>');
-/*	txt = '';
-	entities = [];
+	var tab = tweet.text.split(/\&\#|\;/);
+	var txta = '';
+	for (i = 0; i < tab.length; i++) {
+	    txta += (i % 2 && parseInt(tab[i]) != NaN) ? String.fromCharCode(tab[i]) : tab[i];
+	}
+	var txt = '',
+	    entities = [];
 	for (var i in tweet.entities.hashtags) {
 	    entities.push({
 	        "start" : tweet.entities.hashtags[i].indices[0],
@@ -96,7 +100,7 @@
 	    entities.push({
 	        "start" : tweet.entities.urls[i].indices[0],
 	        "end" : tweet.entities.urls[i].indices[1],
-	        "html" : '<a href="' + tweet.entities.urls[i].expanded_url + '" target="_blank">' + tweet.entities.urls[i].display_url + '</a>'
+	        "html" : '<a href="' + tweet.entities.urls[i].expanded_url + '" target="_blank">' + tweet.entities.urls[i].expanded_url + '</a>'
 	    });
 	}
 	for (var i in tweet.entities.user_mentions) {
@@ -108,10 +112,10 @@
 	}
  	entities.sort(function(a, b) { return a.start - b.start });
     for (var i in entities) {
- 		txt += tweet.text.substring(lastend, entities[i].start) + entities[i].html;
+ 		txt += txta.substring(lastend, entities[i].start) + entities[i].html;
         lastend = entities[i].end;
     }
-	txt += tweet.text.substring(lastend); */
+	txt += txta.substring(lastend);
 	html += txt + '</p></li></div>';
 	return html;
 }
diff -r ef3ff87f98fd -r 19ed2a92d3c1 tweetcast/nodejs/node-direct.js
--- a/tweetcast/nodejs/node-direct.js	Fri Nov 04 11:04:21 2011 +0100
+++ b/tweetcast/nodejs/node-direct.js	Fri Nov 04 16:12:32 2011 +0100
@@ -54,18 +54,19 @@
 
 function createTables() {
 
-    var requete = "CREATE TABLE IF NOT EXISTS tweets ( pos INTEGER PRIMARY KEY, tweet_id TEXT UNIQUE, created_at INTEGER, json TEXT" + annotationMap(function(a) { return ', a_' + a + ' INTEGER' }).join("") + " )";
-    db.execute(requete, function(err) {
+    var requete = "CREATE TABLE IF NOT EXISTS tweets ( pos INTEGER PRIMARY KEY, tweet_id TEXT UNIQUE, created_at INTEGER, json TEXT"
+        + annotationMap(function(a) { return ', a_' + a + ' INTEGER' }).join("")
+        + " );\n"
+        + "CREATE TABLE IF NOT EXISTS tweet_refs ( id INTEGER PRIMARY KEY, from_id TEXT, to_id TEXT, ref_type TEXT );\n"
+        + "CREATE INDEX IF NOT EXISTS idx_created_at ON tweets ( created_at );";
+    db.executeScript(requete, function(err) {
         if (err) { myLog("SQLITE error",err.stack); }
-        db.execute("CREATE INDEX IF NOT EXISTS idx_created_at ON tweets ( created_at )", function(err) { if (err) { myLog("SQLITE error",err.stack); } });
         getSendLastPos();
     });
-    
-    db.execute("CREATE TABLE IF NOT EXISTS tweet_refs ( id INTEGER PRIMARY KEY, from_id TEXT, to_id TEXT, ref_type TEXT )", function(err) { if (err) { myLog("SQLITE error",err.stack); } });
 }
 
 function commitReference(from_id, to_id, ref_type) {
-    commit_script += 'INSERT INTO tweet_refs ( from_id, to_id, ref_type ) VALUES ( "' + from_id + '", "' + to_id + '", "' + ref_type + '" );\n';
+    commit_script += 'INSERT OR IGNORE INTO tweet_refs ( from_id, to_id, ref_type ) VALUES ( "' + from_id + '", "' + to_id + '", "' + ref_type + '" );\n';
 }
 
 function commitTweet(data) {
@@ -136,7 +137,7 @@
             commitTweet(newdata[i]);
         }
     }
-    myLog("Data received - length :",chunk.length);
+//    myLog("Data received - length :",chunk.length);
 }
 
 function requestTwitter() {
@@ -191,7 +192,7 @@
         commit_script = '';
       //  console.log(requete);
       //  var reqd = new Date();
-        db.execute(requete, function (err) {
+        db.executeScript(requete, function (err) {
             if (err) { myLog("SQLITE error",err.stack); }
         //    myLog("Commit took",(new Date() - reqd),"ms");
             getSendLastPos();
@@ -427,6 +428,7 @@
 });
 
 setInterval(commitTweets,500);
+setInterval(function(){myLog("Still alive, tweet count",lastpos)}, 60000);
 
 if (RECORD_NEW_TWEETS) {
     requestTwitter();
diff -r ef3ff87f98fd -r 19ed2a92d3c1 tweetcast/nodejs/sqlite2txt.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tweetcast/nodejs/sqlite2txt.py	Fri Nov 04 16:12:32 2011 +0100
@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import sqlite3, sys, os.path, codecs, re
+
+if len(sys.argv) > 1:
+    filename = sys.argv[1]
+    if os.path.exists(filename):
+        print "Chargement du fichier %s" % filename
+        conn = sqlite3.connect(filename)
+        c = conn.cursor()
+        c.execute("SELECT json FROM tweets")
+        f = codecs.open("%s.txt" % filename, mode='w', encoding='utf-8')
+        for r in c:
+            txt = ''
+            t = re.split("&#([0-9]+);",r[0])
+            for i in range(len(t)):
+                txt += ( ( '\\u' + hex(int(t[i])).replace('0x','').rjust(4,'0') ) if i % 2 else t[i] )
+            f.write(u'%s\n' % txt)
+        f.close()
+        print "Le fichier %s.txt a ete cree" % filename
+    else:
+        print "Le fichier n'existe pas"
+else:
+    print "Merci de preciser le nom du fichier"
\ No newline at end of file