Optimized sqlite access
authorRaphael Velt <raph.velt@gmail.com>
Fri, 28 Oct 2011 18:48:14 +0200
changeset 341 cab5c9e10f90
parent 340 a99a04556e3b
child 342 60b8281442b8
child 344 84cf2be7e8bd
Optimized sqlite access
tweetcast/nodejs/client/js/script.js
tweetcast/nodejs/node-direct.js
--- a/tweetcast/nodejs/client/js/script.js	Fri Oct 28 16:46:13 2011 +0200
+++ b/tweetcast/nodejs/client/js/script.js	Fri Oct 28 18:48:14 2011 +0200
@@ -82,7 +82,8 @@
 	}
 	html += '<h4>' + a_user + '@' + tweet.user.screen_name + '</a></h4><p class="created_at">' + new Date(tweet.created_at).toLocaleTimeString() + '</p><p class="tweet_text">';
 	lastend = 0;
-	txt = '';
+	txt = tweet.text.replace(/(https?:[0-9a-zA-Z\/\-_\.~&?=]+)/g,'<a href="$1" target="_blank">$1</a>').replace(/@([0-9a-zA-Z_]+)/g,'<a href="http://twitter.com/$1" target="_blank">@$1</a>');
+/*	txt = '';
 	entities = [];
 	for (var i in tweet.entities.hashtags) {
 	    entities.push({
@@ -110,7 +111,7 @@
  		txt += tweet.text.substring(lastend, entities[i].start) + entities[i].html;
         lastend = entities[i].end;
     }
-	txt += tweet.text.substring(lastend);
+	txt += tweet.text.substring(lastend); */
 	html += txt + '</p></li></div>';
 	return html;
 }
@@ -216,7 +217,7 @@
         tweetData.waitUpdate = false;
         getUpdate();
         tweetData.blockUpdate = true;
-        setTimeout(delayedUpdate, 300);
+        setTimeout(delayedUpdate, 100);
     }
 }
 
--- a/tweetcast/nodejs/node-direct.js	Fri Oct 28 16:46:13 2011 +0200
+++ b/tweetcast/nodejs/node-direct.js	Fri Oct 28 18:48:14 2011 +0200
@@ -1,8 +1,11 @@
 /* DEFAULT CONFIGURATION */
 
-var TWITTER_USER = 'materiauxnum',
-    TWITTER_PASS = 'm473r14ux7w337',
-    RECORD_NEW_TWEETS = true;
+//var TWITTER_USER = 'materiauxnum',
+//    TWITTER_PASS = 'm473r14ux7w337',
+var TWITTER_USER = 'raphv',
+    TWITTER_PASS = '7w337LfC23gE',
+    RECORD_NEW_TWEETS = true,
+    commit_script = '';
 
 /* CALLING COMMON CONFIGURATION FILE */
 
@@ -55,17 +58,14 @@
     db.execute(requete, function(err) {
         if (err) { myLog("SQLITE error",err.stack); }
         db.execute("CREATE INDEX IF NOT EXISTS idx_created_at ON tweets ( created_at )", function(err) { if (err) { myLog("SQLITE error",err.stack); } });
+        getSendLastPos();
     });
     
     db.execute("CREATE TABLE IF NOT EXISTS tweet_refs ( id INTEGER PRIMARY KEY, from_id TEXT, to_id TEXT, ref_type TEXT )", function(err) { if (err) { myLog("SQLITE error",err.stack); } });
 }
 
 function commitReference(from_id, to_id, ref_type) {
-    db.execute(
-        "INSERT INTO tweet_refs ( from_id, to_id, ref_type ) VALUES ( ?, ?, ? )",
-        [ from_id, to_id, ref_type ],
-        function(err) { if (err) { myLog("SQLITE error",err.stack); } }
-    );
+    commit_script += 'INSERT INTO tweet_refs ( from_id, to_id, ref_type ) VALUES ( "' + from_id + '", "' + to_id + '", "' + ref_type + '" );\n';
 }
 
 function commitTweet(data) {
@@ -114,17 +114,19 @@
     if (tweet.retweeted_status) {
         commitReference( tweet.id, tweet.retweeted_status.id, "retweet" );
     }
-    db.execute(
-        "INSERT INTO tweets ( tweet_id, created_at, json "
+    commit_script += 'INSERT INTO tweets ( tweet_id, created_at, json '
         + annotationMap(function(a) { return ', a_' + a }).join("")
-        + " ) VALUES ( ?, ?, ? "
-        + annotationMap(function(a) { return ', ?' }).join("")
-        + " )",
-        [ tweet.id, tweet.created_at.valueOf(), JSON.stringify(tweet) ].concat(annotationMap(function(a) { return ann.indexOf(a) == -1 ? 0 : 1 })),
-        function(err) {
-            if (err) { myLog("SQLITE error",err.stack); }
-        }
-    );
+        + ' ) VALUES ( "'
+        + tweet.id
+        + '", '
+        + tweet.created_at.valueOf()
+        + ', "'
+        + JSON.stringify(tweet).replace(/\"/g, '""')
+        + '"'
+        + annotationMap(function(a) {
+            return ann.indexOf(a) == -1 ? ', 0' : ', 1'
+        }).join("")
+        + ' );\n';
 }
 
 function callBackNewTweets(chunk) {
@@ -172,7 +174,6 @@
         if (err) { myLog("SQLITE error",err.stack); }
         if (results[0].lastpos != lastpos) {
             lastpos = results[0].lastpos ? results[0].lastpos : 0;
-//            myLog("Broadcasting last pos = ",lastpos);
             try {
                 io.sockets.emit('tweetSummary', {
                     tweetcount : lastpos
@@ -184,6 +185,20 @@
     });
 }
 
+function commitTweets() {
+    if (commit_script != '') {
+        var requete = commit_script;
+        commit_script = '';
+      //  console.log(requete);
+      //  var reqd = new Date();
+        db.execute(requete, function (err) {
+            if (err) { myLog("SQLITE error",err.stack); }
+        //    myLog("Commit took",(new Date() - reqd),"ms");
+            getSendLastPos();
+        });
+    }
+}
+
 function getSendTweetPosByDate(date, socket) {
     db.execute("SELECT pos, created_at, ABS(created_at-" + date + ") AS dist FROM tweets ORDER BY dist ASC LIMIT 0,9", function (err, results) {
         if (err) { myLog("SQLITE error",err.stack); }
@@ -411,7 +426,7 @@
     createTables();
 });
 
-setInterval(getSendLastPos,300);
+setInterval(commitTweets,500);
 
 if (RECORD_NEW_TWEETS) {
     requestTwitter();