# HG changeset patch
# User veltr
# Date 1362765917 -3600
# Node ID 14dd1980b0b9a16b3bf4be511ce1ded1f12ef8c3
# Parent  490e4d1b6fee5b3e55eeca6df4bdd15aa215f47f
MMSO Caching

diff -r 490e4d1b6fee -r 14dd1980b0b9 css/playscreen.css
--- a/css/playscreen.css	Wed Mar 06 18:09:40 2013 +0100
+++ b/css/playscreen.css	Fri Mar 08 19:05:17 2013 +0100
@@ -60,7 +60,7 @@
 }
 
 .play-dataviz {
-    float: left; height: 100%; width: 100%; position: relative; margin-left: 8px;
+    float: left; height: 672px; width: 100%; position: relative; margin-left: 8px; overflow: hidden;
 }
 
 .play-svg {
@@ -84,8 +84,16 @@
     color: #333333; background: #ffff00;
 }
 
+.play-images {
+	left: 300px; position: absolute;
+}
+
+.play-images img {
+	position: absolute; left: 0; width: 80px; height: 45px;
+}
+
 .play-localtweets {
-    left: 310px;
+    left: 390px;
     position: absolute;
     z-index: 2;
 }
diff -r 490e4d1b6fee -r 14dd1980b0b9 js/playscreen.js
--- a/js/playscreen.js	Wed Mar 06 18:09:40 2013 +0100
+++ b/js/playscreen.js	Fri Mar 08 19:05:17 2013 +0100
@@ -152,11 +152,11 @@
         
         var deltaY = $(".play-dataviz").offset().top;
         
-        $(".play-localtweets .tweet").each(function() {
+        $(".play-localtweets .tweet:visible").each(function() {
         	var el = $(this),
         		liY = + el.offset().top + el.outerHeight() / 2 - deltaY,
         		tY = localyscale * (+el.attr("data-timestamp") - localpos + localduration / 2),
-        		p = "M" + localL + "," + tY + "L" + localR + "," + tY + "L320," + liY,
+        		p = "M" + localL + "," + tY + "L" + localR + "," + tY + "L400," + liY,
         		path = paper.path(p);
     		$(this).css("background",colors[el.attr("data-topic-id")] || "");
     		path.attr({
@@ -164,7 +164,6 @@
     		});
     		tweetLines.push(path);
         });
-        throttledGetTweets();
         
     }
     
@@ -285,7 +284,7 @@
 	        var rect = paper.rect( localL, y, localW, h );
 	        rect.attr({stroke: "none", title: s.id});
 	        if (parseInt(s.id.replace("MMSO_","")) % 2) {
-	        	var altrect = paper.rect( localR, y, (totalR - localR), h);
+	        	var altrect = paper.rect( localR, y, 60, h);
 	        	altrect.attr({stroke: "none", fill: "#222"});
 	        	mmsoAlt.push(altrect);
 	        }
@@ -335,134 +334,240 @@
 	        "stroke-width": 1,
 	        "stroke": "#ffffff"
 	    });
+	    
+	    var imghtm = '', imgrate = localduration / 12, imgstart = imgrate * Math.floor(localstart / imgrate), imgend = imgrate * Math.ceil(localend / imgrate);
+	    for (var i = imgstart; i <= imgend; i+= imgrate) {
+	    	var imgsrc = i + '.png';
+	    	while (imgsrc.length < 9) {
+	    		imgsrc = '0' + imgsrc;
+	    	}
+	    	var imgy = Math.floor( localyscale * (i - localstart) - 22.5 )
+	    	imghtm += '<img src="thumbnails/' + imgsrc + '" style="top: ' + imgy +'px;" />';
+	    }
+	    $(".play-images").html(imghtm);
+	    
+	    $(".play-localtweets .tweet").each(function() {
+	    	var el = $(this),
+	    		t = parseInt(el.attr("data-timestamp"));
+    		if (t > localstart && t < localend) {
+    			el.show();
+    		} else {
+    			el.hide();
+    		}
+	    });
+	    
+		throttledGetTweets();
+	    
 		showTopicViz();
     }
     
     
     var lastPos, lastDuration, lastTopics;
     
-    var tweetTemplate = _.template('<li class="tweet" data-timestamp="<%= _timestamp %>" data-topic-id="<%= _topic %>"><img src="<%- profile_image_url %>" /><p>@<%- from_user_name %>: <%- text %></p></li>'),
-    	callnum = 0;
+    var tweetTemplate = _.template('<li class="tweet" data-timestamp="<%= timestamp %>" data-topic-id="<%= topic.topic %>"><img src="<%- data.profile_image_url %>" /><p>@<%- data.from_user_name %>: <%- data.text %></p></li>'),
+    	callnum = 0,
+    	tweetstructure = {},
+    	requestedtweets = {},
+    	_NTWEETS = 10;
     
-    function getTweets() {
-    	var topicIds = Array.prototype.join.call($(".topic.selected").map(function(){return $(this).attr("data-topic-id")}));
-    	if (localduration === lastDuration && lastTopics === topicIds && Math.abs(localpos - lastPos) < (localduration / 10)) {
-    		console.log("We already have these tweets, delta =", localpos - lastPos );
-    		return;
-    	}
-    	if (!topicIds) {
-    		console.log("No topics selected");
-    		return;
+    function showTweets() {
+    	    	
+    	var toshow = [];
+    	var topics = Array.prototype.join.call($(".topic.selected").map(function(){return $(this).attr("data-topic-id")})).split(",");
+    	
+    	for (var i = 0; i < localMmsos.length; i++) {
+    		var mmso = data.segments[localMmsoDelta + i];
+    		var mmsostruct = tweetstructure[mmso.id];
+    		if (mmsostruct) {
+	    		for (var j = 0; j < topics.length; j++) {
+	    			var topicid = topics[j];
+					if (mmsostruct[topicid]) {
+						var tweetids = mmsostruct[topicid].tweetids;
+						for (var k = 0; k < tweetids.length; k++) {
+							toshow.push(tweetids[k]);
+						}
+					}
+	    		}
+    		}
     	}
-    	var localcall = ++callnum;
-    	console.log("getTweets was called",localcall,(function(d){return d.toLocaleTimeString()+"."+d.getMilliseconds()})(new Date()));
-    	var tweetids = {},
-    		rqtodo = 0,
-    		rqdone = 0,
-    		topics = topicIds.split(",");
+    	toshow = _(toshow).uniq();
+    	    	
+    	var tweetstoshow = toshow.map(function(tid) {
+    		return requestedtweets[tid];
+    	}).filter(function(tw) {
+    		return ((tw.status == 2) && (tw.timestamp > (localpos - localduration / 2)) && (tw.timestamp < (localpos + localduration / 2)));
+		});
+		
+		tweetstoshow.forEach(function(tw) {
+			tw.topic = tw.topics.filter(function(t) {
+				return topics.indexOf(t.topic) !== -1;
+			}).sort(function(a,b) {
+				return b.weight - a.weight
+			})[0];
+		});
+				
+		tweetstoshow.sort(function(a, b) {
+			return b.topic.weight - a.topic.weight; 
+		});
+		
+		tweetstoshow = tweetstoshow.slice(0,10);
+		
+		tweetstoshow.sort(function(a, b) {
+			return a.timestamp - b.timestamp;
+		});
+		
+		console.log(tweetstoshow);
+		
+		var html = tweetstoshow.map(tweetTemplate).join("");
+		
+		$(".play-localtweets").html(html);
+		var h = 0;
+		$(".play-localtweets .tweet").each(function() {
+			h += $(this).outerHeight();
+		});
+		$(".play-localtweets .tweet").css("margin-top",Math.max(0,($(".play-bottom").height() - h)/(tweetstoshow.length+1)));
+		showTopicViz();
+    	
+    }
+    
+    function getMmsoTweetIds(mmstruct) {
+    	    	
+    	TopicsBean.bestSocialInteractionsIdsMatching(mmstruct.mmso, mmstruct.topic, 0, _NTWEETS, {
+			callback: function(tw) {
+				mmstruct.status = 2;
+				mmstruct.tweetids = tw;
+				for (var k = 0; k < tw.length; k++) {
+					var tweetid = tw[k],
+						relevance = mmstruct.weight * (_NTWEETS - k) / _NTWEETS;
+					if (!requestedtweets[tweetid]) {
+						requestedtweets[tweetid] = {
+							id: tweetid,
+							status: 0,
+							topics: []
+						}
+					}
+					requestedtweets[tweetid].topics.push({
+						topic: mmstruct.topic,
+						weight: relevance
+					});
+				}
+				debouncedGetTweetData();
+			},
+			errorHandler: function(err) {
+				mmstruct.status = 0;
+				console.log(err);
+				throttledGetTweetIds();
+			}
+		});
+    }
+    
+    var _MAX_BATCH = 20;
+    
+    function getTweetIds() {
+    	
+    	console.log("getTweetIds");
+    	
+    	var toload = [];
     	
-    	function getMmsoTopicTweets(mmsoid, topic, ntweets) {
-    		rqtodo++;
-			TopicsBean.bestSocialInteractionsIdsMatching(mmsoid, topic, 0, ntweets, {
-				callback: function(tw) {
-					for (var k = 0; k < tw.length; k++) {
-						tweetids[tw[k]] = topic;
-					}
-					rqdone++;
-					if (rqdone === rqtodo) {
-						loadTweets();
-					}
+    	_(tweetstructure).each(function(v) {
+    		_(v).each(function(w) {
+    			if (!w.status) {
+    				w.status = 1;
+    				toload.push(w);
+    			}
+    		});
+    	});
+    	
+    	console.log("Loading tweet ids");
+    	
+    	if (toload.length) {
+    		
+    		if (toload.length > _MAX_BATCH) {
+    			toload = _(toload).shuffle().slice(0,_MAX_BATCH);
+    			window.setInterval(throttledGetTweetIds,0);
+    		}
+    		
+    		dwr.engine.beginBatch();
+    		toload.forEach(getMmsoTweetIds);
+    		dwr.engine.endBatch();
+    	}
+    }
+    
+    function getTweetData() {
+    	
+    	console.log("getTweetData");
+    	
+    	var toload = [];
+    	
+    	_(requestedtweets).each(function(v) {
+			if (!v.status) {
+				toload.push(v.id);
+			}
+		});
+    	
+    	console.log("Loading tweet data");
+		
+		$.getJSON(
+			solrUrl(
+				"twitter",
+				{
+					q:"id:(" + toload.join(" OR ") + ")",
+					fl: "id_str,created_at,from_user_name,text,profile_image_url",
+					rows: toload.length
 				}
-			});
+			),
+			function(t) {
+				var tweets = t.response.docs;
+				tweets.forEach(function(tweet) {
+					var timestamp = new Date(tweet.created_at).valueOf() / 1000 - deltaT;
+					requestedtweets[tweet.id_str].data = tweet;
+					requestedtweets[tweet.id_str].status = 2;
+					requestedtweets[tweet.id_str].timestamp = timestamp;
+				});
+				throttledShowTweets();
+			}
+		);
+		
+    }
+    
+    debouncedGetTweetData = _(getTweetData).debounce(125);
+    
+    throttledGetTweetIds = _(getTweetIds).throttle(10000);
+    
+    throttledShowTweets = _(showTweets).throttle(200);
+    
+    function getLocalTweets() {
+    	
+    	console.log("getLocalTweets");
+    	
+    	var topics = Array.prototype.join.call($(".topic.selected").map(function(){return $(this).attr("data-topic-id")})).split(",");
+    	
+    	for (var i = 0; i < localMmsos.length; i++) {
+    		var mmso = data.segments[localMmsoDelta + i];
+    		if (!tweetstructure[mmso.id]) {
+    			tweetstructure[mmso.id] = {}
+    		}
+    		var mmsostruct = tweetstructure[mmso.id];
+    		for (var j = 0; j < topics.length; j++) {
+				var topicid = topics[j],
+					weight = data.topics[topics[j]].weights[localMmsoDelta + i];
+				if (weight > .1 && !mmsostruct[topicid]) {
+					mmsostruct[topicid] = {
+						topic: topicid,
+						mmso: mmso.id,
+						status: 0,
+						weight: weight,
+						tweetids: []
+					};
+				}
+    		}
     	}
     	
-    	function loadTweets() {
-    		console.log("Tweet IDs retrieved",localcall,(function(d){return d.toLocaleTimeString()+"."+d.getMilliseconds()})(new Date()));
-    		var u = _.keys(tweetids);
-			$.getJSON(
-				solrUrl(
-					"twitter",
-					{
-						q:"id:(" + u.join(" OR ") + ")",
-						rows: u.length
-					}
-				),
-				function(t) {
-    				console.log("Full tweets retrieved",localcall,(function(d){return d.toLocaleTimeString()+"."+d.getMilliseconds()})(new Date()));
-					var tweets = t.response.docs;
-					var s = 600 / (1+tweets.length);
-					tweets.forEach(function(tweet, i) {
-						tweet._date = new Date(tweet.created_at);
-						tweet._timestamp = tweet._date.valueOf() / 1000 - deltaT;
-						tweet._topic = tweetids[tweet.id_str]
-					});
-					tweets.sort(function(a,b) {
-						return a._date - b._date;
-					});
-					var html = tweets.reduce(function(mem, tweet) {
-						return mem + tweetTemplate(tweet);
-					},'');
-					$(".play-localtweets").html(html);
-					var h = 0;
-					$(".play-localtweets .tweet").each(function() {
-						h += $(this).outerHeight();
-					});
-					$(".play-localtweets .tweet").css("margin-top",Math.max(0,($(".play-bottom").height() - h)/(tweets.length+1)));
-					showTopicViz();
-				}
-			);
-			$(".play-localtweets").html("");
-			showTopicViz();
-    	}
-    	
-    	function getMmsoTweets(nmmso) {
-    		var mmso = data.segments[nmmso],
-    			mmsopixels = localyscale * (Math.min(localpos + localduration / 2, mmso.end) - Math.max(localpos - localduration / 2, mmso.start));
-    		var ntweets = Math.floor(( 50 + mmsopixels ) / 80),
-    			mmsotopics = [];
-			if (!ntweets) {
-				return;
-			}
-			for (var j = 0; j < topics.length; j++) {
-				var weight = data.topics[topics[j]].weights[nmmso];
-				if (weight > .05) {
-					mmsotopics.push({topic:parseInt(topics[j]),weight:weight,ntweets:0});
-				}
-			}
-			mmsotopics.sort(function(a,b){return b.weight - a.weight});
-			if (mmsotopics.length) {
-				for (var j = 0; j < ntweets; j++) {
-					mmsotopics[j % mmsotopics.length].ntweets++;
-				}
-				mmsotopics = mmsotopics.filter(function(t) { return !!t.ntweets });
-				for (var j = 0; j < mmsotopics.length; j++) {
-					getMmsoTopicTweets(mmso.id, mmsotopics[j].topic, mmsotopics[j].ntweets);
-				}
-			} else {
-				var t = [], m = {};
-				while (t.length < ntweets) {
-					t = t.concat(topics);
-				}
-				for (var j = 0; j < ntweets; j++) {
-					m[t[j]] = 1 + (m[t[j]]||0);
-				}
-				_(m).each(function(v,k) {
-					getMmsoTopicTweets(mmso.id, k, v);
-				});
-			}
-    	}
-    	
-    	dwr.engine.beginBatch();
-    	for (var i = 0; i < localMmsos.length; i++) {
-    		getMmsoTweets(localMmsoDelta + i);
-    	}
-    	dwr.engine.endBatch();
-    	lastPos = localpos;
-    	lastDuration = localduration;
-    	lastTopics = topicIds;
+    	throttledGetTweetIds();
+    	throttledShowTweets();
     }
     
-    var throttledGetTweets = _.throttle(_.debounce(getTweets, 500), 10000),
+    var throttledGetTweets = _.throttle(getLocalTweets, 500),
     	throttledShowLocal = _.throttle(showLocal, 100);
     
     showTopics(sortedTopics);
@@ -491,24 +596,11 @@
     
     var mouseIsDown, isDragging, startY, startT, startPos, scrollGlobal, speedscale, slowiterations;
     
-    function inertia() {
-    	startPos = localpos;
-    	window.setTimeout(function() {
-    		speedscale = .75 * speedscale;
-    		localstart += 100*speedscale;
-    		throttledShowLocal();
-    		if (slowiterations < 5) {
-    			inertia();
-    		}
-    		slowiterations++;
-    	}, 100);
-    }
-    
-    $("body").mouseup(function() { mouseIsDown = false; });
+    $(document).mouseup(function() { mouseIsDown = false; isDragging = false; });
     
     $(".play-dataviz").mousedown(function(e) {
     	var l = $(this).offset().left,
-    		scrollLimit = l + 280;
+    		scrollLimit = l + 380;
 		if (e.pageX < scrollLimit) {
 			mouseIsDown = true;
 	    	startY = e.pageY;
@@ -536,15 +628,6 @@
     		localpos = Math.max(localduration / 2, Math.min(data.duration - localduration / 2, Math.floor(posY / yscale)));
 			throttledShowLocal();
     	}
-/*    	if (isDragging) {
-			var limit = $(this).offset().left + 140,
-				deltaY = e.pageY - startY,
-				deltaT = new Date() - startT,
-				delta = Math.floor(deltaY / (scrollGlobal ? yscale : - localyscale));
-			speedscale = delta / deltaT;
-			slowiterations = 0;
-			inertia();
-    } */
     });
     
     var totalScroll = 0, zoomlevels = [ 1800, 900, 600, 300, 120, 60 ], currentlevel = 2;
diff -r 490e4d1b6fee -r 14dd1980b0b9 js/startscreen.js
--- a/js/startscreen.js	Wed Mar 06 18:09:40 2013 +0100
+++ b/js/startscreen.js	Fri Mar 08 19:05:17 2013 +0100
@@ -86,7 +86,6 @@
             */
             TopicsBean.topicsForKeywords(selectedWords.join(","),{
                 callback: function(topicweights) {
-                    console.log(topicweights);
                     var topiclist = data.topics.filter(function(topic) {
                         return topicweights[topic.index] > .01 && topic.index !== topicPoubelle;
                     });
@@ -147,7 +146,7 @@
         
         showTopicViz();
     }
-// /*        
+/*        
         var topwords = [], globwords = {};
         
         data.topics.forEach(function(topic) {
@@ -169,37 +168,51 @@
         $(".keyword-search").autocomplete({
             source: topwords.map(function(w) { return w.word })
         }).on("keyup change paste", wordFilter);
-// */
- /*
-    var globwords = {},
-        allwords = [];
+ */
+// /*
+    var globwords = {};
     
     data.topics.forEach(function(topic) {
-        topic.words.forEach(function(word) {
-            globwords[word.word] = 1
+        topic.words.forEach(function(topicword) {
+        	topicword.word.split(" ").filter(function(w) {
+        		return w.length > 2;
+        	}).forEach(function(w) {
+        		globwords[w] = 2 * topicword.weight + (globwords[w] || 0)
+        	});
         });
     });
     
-    var ntw = data.topwords.length,
-        topwords = data.topwords.map(function(v, k) {
-        globwords[v] = 1;
-        return {
-            word: v,
-            weight: ntw - k
-        }
+    var ntw = data.topwords.length;
+    
+    data.topwords.forEach(function(v, k) {
+    	var weight = (ntw - k)/ntw;
+    	v.split(" ").filter(function(w) {
+    		return w.length > 2;
+    	}).forEach(function(w) {
+    		globwords[w] = weight + (globwords[w] || 0)
+    	});
     });
     
-    for (var w in globwords) {
-        if (globwords.hasOwnProperty(w)) {
-            allwords.push(w);
-        }
-    }
+    delete globwords.des;
+    
+    var allwords = _(globwords).keys(),
+    	topwords = _(globwords).map(function(v, k) {
+    		return {
+    			word: k,
+    			weight: v
+    		}
+    	});
+    
+    topwords.sort(function(a,b) {
+    	return b.weight - a.weight
+    });
+    
     allwords.sort();
     
     $(".keyword-search").autocomplete({
         source: allwords
     }).on("keyup change paste", wordFilter);
- */
+// */
 
     var wordsToShow = topwords.slice(0,80),
         max = wordsToShow[0].weight,
@@ -372,7 +385,6 @@
     function checkIfLoaded() {
         loadedSteps++;
         if (loadedSteps >= stepsToFullyLoaded) {
-            console.log("Showing data");
             setTimeout(showData,0);
         }
     }
@@ -445,8 +457,7 @@
                     start: start,
                     end: end,
                     duration: end - start,
-                    topics: topics,
-                    tweet_count: (end - start) * (35 + 3 * Math.random()) //TODO: dissociate tweet counts from segments
+                    topics: topics
                 }
             }).sort(function(a,b) {
                 return a.start - b.start;
@@ -479,6 +490,6 @@
         
     });
     
-    //loadFromTopicsBean("getTopWords","topwords",[400]);
+    loadFromTopicsBean("getTopWords","topwords",[200]);
     
 });
diff -r 490e4d1b6fee -r 14dd1980b0b9 playscreen.html
--- a/playscreen.html	Wed Mar 06 18:09:40 2013 +0100
+++ b/playscreen.html	Fri Mar 08 19:05:17 2013 +0100
@@ -44,6 +44,7 @@
                 
                 <div class="play-bottom">
                     <div class="play-dataviz">
+                        <div class="play-images"></div>
                         <div class="play-svg"></div>
                     	<ul class="play-localtweets"></ul>
                     </div>
diff -r 490e4d1b6fee -r 14dd1980b0b9 startscreen.html
--- a/startscreen.html	Wed Mar 06 18:09:40 2013 +0100
+++ b/startscreen.html	Fri Mar 08 19:05:17 2013 +0100
@@ -12,6 +12,7 @@
         <script>
             var pathToDwrServlet = "http://159.217.144.101:8050/sia-solr/dwr";
         </script>
+        <script src="lib/underscore-min.js"></script>
         <script src="lib/jquery.min.js"></script>
         <script src="lib/jquery-ui.min.js"></script>
         <script src="lib/raphael-min.js"></script>