js/playscreen.js
author veltr
Tue, 12 Mar 2013 18:29:44 +0100
branchold-fetch-method
changeset 8 ba3652cdd44b
parent 7 a2d5b669f663
permissions -rw-r--r--
Branch to keep old tweet fetch method

var topicPoubelle = 13;

var adjust = 54;

var deltaT = new Date("Wed, 02 May 2012 19:00:00 +0000") / 1000 + adjust;

function solrUrl(table, params) {
    return "http://159.217.144.101:8050/sia-solr/" + table + "/select?" + $.param(params) + "&wt=json&json.wrf=?";
}

function showData() {
    
    var topicHash = document.location.hash || "#selectedtopics=5,15&visibletopics=5,10,15";
    
    var pageParams = {};
    
    topicHash
    	.replace(/^#/,'')
    	.split('&')
    	.forEach(function(p) {
    		var s = p.split('=');
    		pageParams[s[0]] = s[1].split(",").map(function(t) { return decodeURIComponent(t)});
    	})
    
    var ordertag = 0;

    function secsToString(seconds) {
        var hours = Math.floor(seconds/3600),
            minutes = Math.floor(seconds/60) % 60,
            secs = Math.floor(seconds % 60);
        function pad(n) {
            var r = n.toString();
            while (r.length < 2) {
                r = "0" + r;
            }
            return r;
        }
        return (hours ? (hours + ":") : "") + pad(minutes) + ":" + pad(secs);
    }
    
    $(".duration").text(secsToString(data.duration));
        
    var nmmso = data.segments.length;
    
    data.topics.forEach(function(topic) {
        topic.score = 0;
        topic.weights = [];
        for (var i = 0; i < nmmso; i++) {
            topic.weights.push(0);
        }
    });
    
    data.segments.forEach(function(mmso, i) {
        mmso.topics.forEach(function(t) {
            data.topics[t.topic].weights[i] = t.weight;
            data.topics[t.topic].score += t.weight;
        });
    });
    
    var sortedTopics = data.topics.filter(function(t) {
        return pageParams.visibletopics.indexOf(t.index.toString()) !==-1 && t.index !== topicPoubelle;
    });
    
    
    function showTopics(topiclist) {
        var tbhtml = topiclist.reduce(function(mem, topic) {
            var wordsToShow = topic.words.slice(0,3),
                max = wordsToShow[0].weight,
                min = Math.min(wordsToShow[wordsToShow.length - 1].weight, max - .01),
                scale = 8 / (max - min);
            var li = '<li class="shadow-block topic" data-topic-id="'
                + topic.index
                + '" data-timestamp="999999"><ul class="topic-words">'
                + wordsToShow.reduce(function(memwords, word) {
                    return memwords
                        + '<li style="font-size: '
                        + ( 8 + scale * (word.weight - min) )
                        + 'px;">'
                        + word.word
                        + '</li>';
                },"")
                + '</ul></li>';
            return mem + li;
        },'');
        var tb = $(".topics-block");
        tb.html(tbhtml);
        tb.css("top",0);
        
        showTopicViz();
    }
    
    var tweetLines = [];
    
    function showTopicViz() {
        var selectedBlocks = $(".topic.selected, .topic.hover"),
            sbl = selectedBlocks.length,
            topicBlocks = $(".topic");
        if (!sbl && topicBlocks.length < sortedTopics.length) {
            selectedBlocks = topicBlocks;
            sbl = selectedBlocks.length;
        }
        var topicsAndColors = [],
        	colors = {};
        selectedBlocks.each(function() {
            var el = $(this),
                topicid = parseInt(el.attr("data-topic-id"));
            topicsAndColors.push({
                "$": el,
                timestamp: parseInt(el.attr("data-timestamp")),
                hovered: el.hasClass("hover"),
                id: topicid,
                topic: data.topics[topicid]
            });
        });
        topicsAndColors.sort(function(a,b) {
           return ( a.timestamp - b.timestamp ) || ( a.id - b.id );
        });
        topicBlocks.css("background","");
        topicsAndColors.forEach(function(topic, i) {
            topic.color = topic.hovered ? "#ffff00" : colorset[i % colorset.length];
            colors[topic.id] = topic.color;
            topic.$.css("background", topic.color);
        });
        
        for (var i = 0; i < nmmso; i++) {
            var opacity = 0,
                rgb = [0,0,0];
            topicsAndColors.forEach(function(topic) {
                var c = Raphael.getRGB(topic.color),
                    o = topic.topic.weights[i];
                rgb[0] += c.r * o;
                rgb[1] += c.g * o;
                rgb[2] += c.b * o;
                opacity += o;
            });
            if (opacity) {
                color = Raphael.rgb.apply(Raphael, rgb.map(function(c) {
                    return c/opacity;
                }));
                var attr = {
                    fill: color,
                    opacity: .5 + .5 * opacity
                };
                segmentrects[i].show();
                segmentrects[i].attr(attr);
                if (i >= localMmsoDelta && i < localMmsoDelta + localMmsos.length) {
                	localMmsos[i - localMmsoDelta].show();
                	localMmsos[i - localMmsoDelta].attr(attr);
                }
            } else {
                segmentrects[i].hide();
                if (i >= localMmsoDelta && i < localMmsoDelta + localMmsos.length) {
                	localMmsos[i - localMmsoDelta].hide();
                }
            }
        }
        
        tweetLines.forEach(function(tl) { tl.remove(); });
        tweetLines = [];
        
        var deltaY = $(".play-dataviz").offset().top;
        
        $(".play-localtweets .tweet:visible").each(function() {
        	var el = $(this),
        		liY = + el.offset().top + el.outerHeight() / 2 - deltaY,
        		tY = localyscale * (+el.attr("data-timestamp") - localpos + localduration / 2),
        		p = "M" + localL + "," + tY + "L" + localR + "," + tY + "L400," + liY,
        		path = paper.path(p);
    		$(this).css("background",colors[el.attr("data-topic-id")] || "");
    		path.attr({
    			stroke: "#ccc"
    		});
    		tweetLines.push(path);
        });
        
    }
    
    var jqsvg = $(".play-svg"),
        paper = new Raphael(jqsvg[0]),
        totalR = jqsvg.width(),
        ph = jqsvg.height(),
        globW = 85,
        globL = 40,
        entonnoirR = 155,
        localL = 235,
        localW = 85,
        localR = (localL + localW),
        localTimeR = (localL + localW) + globL,
        globR = (globW + globL),
        yscale = ph / data.duration,
        mx = Math.max.apply(Math, data.minutes.map(function(s) { return s.count})),
        xscale = globW/mx;
    
    var segmentrects = data.segments.map(function(mmso) {
        var rect = paper.rect(globL, yscale * mmso.start, globW, yscale * mmso.duration);
        rect.attr({stroke: "none"});
        return rect;
    });
    
    var d = "M" + data.minutes.map(function(s) {
            var x = globL + xscale * s.count;
            return x
                + ","
                + yscale * (s.from + 20)
                + "L"
                + x
                + ","
                + yscale * (s.from + 40);
        }).join("L");
        
    paper.path(d).attr({
        "stroke-width": 4,
        "stroke": "#000000",
        opacity: .5
    });
    paper.path(d).attr({
        "stroke-width": 1,
        "stroke": "#ffffff"
    });
    
    for (var i=0; i < data.duration; i += 1800) {
        var y = yscale * i;
        paper.path("M0" + "," + y + "l" + globR + ",0").attr({stroke: "#666"});
        paper.text(0, y + 6, secsToString(i)).attr({
        	"text-anchor": "start",
            "fill": "#ffffff"
        });
    }
    paper.text(0, ph-8, secsToString(data.duration)).attr({
        "text-anchor": "start",
        "fill": "#ffffff"
    });
    paper.path("M0" + globR + ",0L" + localTimeR + ",0" ).attr({stroke: "#666"});
    paper.path("M0," + (ph-1) + "l" + localTimeR + ",0" ).attr({stroke: "#666"});
    
    paper.path("M" + globR + ",0l0," + ph).attr({stroke: "#666"});
    paper.path("M" + localL + ",0l0," + ph).attr({stroke: "#666"});
    
    var entonnoir = paper.path("").attr("fill","#333"),
    	localStartText = paper.text(localTimeR,6,"").attr({
    		"text-anchor": "end",
    		"fill": "#ffffff"
    	}),
    	localEndText = paper.text(localTimeR,ph - 8, "").attr({
    		"text-anchor": "end",
    		"fill": "#ffffff"
    	}),
    	localTimes = [],
    	localMmsos = [],
    	localMmsoDelta,
    	mmsoAlt = [],
    	lowerFiveSecs,
    	higherFiveSecs,
    	localyscale;
	
	entonnoir.toBack();
    
    var cloudTemplate = _.template('<li style="font-size: <%- size %>px;"><%- word %></li>');
    
    function showLocal() {
		localyscale = ph / localduration;
    	var localstart = localpos - localduration/2;
    		localend = localpos + localduration/2;
    		globtop = yscale * localstart,
    		globbottom = yscale * localend,
    		betweenx = (globR + entonnoirR) / 2,
    		betweenyt = (globtop) / 2,
    		betweenyb = (globbottom + ph) / 2,
    		curve = (entonnoirR - globR) / 2,
    		entonnoird = "M0," + globtop + "l" + globR + ",0Q" + betweenx + "," + globtop + "," + betweenx + ","
    			+ Math.max(globtop - curve, betweenyt) + "L" + betweenx + "," + Math.min(curve, betweenyt) + "Q"
    			+ betweenx + ",0," + entonnoirR + ",0"
    			+ "L" + localR + ",0L" + localR + "," + ph + "L" + entonnoirR + "," + ph + "Q" + betweenx + "," + ph + ","
    			+ betweenx + "," + Math.max(ph - curve, betweenyb) +"L" + betweenx + "," + Math.min(globbottom + curve, betweenyb)
    			+ "Q" + betweenx + "," + globbottom + "," + globR + "," + globbottom + "L0," + globbottom,
			localkeywords = {};
			
		entonnoir.attr("path",entonnoird);
		localTimes.forEach(function(t) {
			t.text.remove();
			t.line.remove();
		});
		localMmsos.forEach(function(t) {
			t.remove();
		});
		mmsoAlt.forEach(function(t) {
			t.remove();
		});
		var filteredSegments = data.segments.filter(function(s) {
			return s.start < localend && s.end > localstart;
		});
		localMmsoDelta = parseInt(filteredSegments[0].id.split("_")[1]);
		localMmsos = filteredSegments.map(function(s) {
			var y = localyscale * (s.start - localstart),
				h = localyscale * s.duration,
				rect = paper.rect( localL, y, localW, h ),
				visibled = (
					s.start < localstart
					? s.duration - localstart + s.start
					: (
						s.end > localend
						? s.duration - s.end + localend
						: s.duration
					)
				);
			_(s.keywords).each(function(v,k) {
				localkeywords[k] = (v * visibled) + (localkeywords[k] || 0);
			});
	        rect.attr({stroke: "none", title: s.id});
	        if (parseInt(s.id.replace("MMSO_","")) % 2) {
	        	var altrect = paper.rect( localR, y, 60, h);
	        	altrect.attr({stroke: "none", fill: "#222"});
	        	mmsoAlt.push(altrect);
	        }
	        return rect;
		});
		localStartText.attr("text", secsToString(localstart)).toFront();
		localEndText.attr("text", secsToString(localend)).toFront();
		for ( var i = (1 + Math.floor(localstart / 120)) * 120; i < localend; i += 120 ) {
			var y = localyscale*(i - localstart)
			localTimes.push({
				text: paper.text(localTimeR,6+y,secsToString(i)).attr({
		    		"text-anchor": "end",
		    		"fill": "#ffffff"
		    	}),
		    	line: paper.path("M0" + localL + "," + y + "L" + localTimeR + "," + y).attr({stroke: "#666"})
			});
		}
		if (lowerFiveSecs) {
			lowerFiveSecs.remove();
		}
		if (higherFiveSecs) {
			higherFiveSecs.remove();
		}
		var filteredFiveSecs = data.fiveseconds.slice(Math.floor(localstart / 5), Math.ceil(localend / 5));
        var counts = filteredFiveSecs.map(function(s) { return s.count}),
        	lmx = Math.max.apply(Math, counts),
        	lmi = Math.min.apply(Math, counts.concat([lmx - 1]))
        	lxscale = localW/(lmx-lmi);
       	
	    var d = "M" + filteredFiveSecs.map(function(s) {
	            var x = localL + lxscale * (s.count - lmi);
	            return x
	                + ","
	                + localyscale * (s.from + 1 - localstart)
	                + "L"
	                + x
	                + ","
	                + localyscale * (s.from + 4 - localstart);
	        }).join("L");
	        
	    lowerFiveSecs = paper.path(d).attr({
	        "stroke-width": 4,
	        "stroke": "#000000",
	        opacity: .5
	    });
	    higherFiveSecs = paper.path(d).attr({
	        "stroke-width": 1,
	        "stroke": "#ffffff"
	    });
	    
	    var imghtm = '', imgrate = localduration / 12, imgstart = imgrate * Math.floor(localstart / imgrate), imgend = imgrate * Math.ceil(localend / imgrate);
	    for (var i = imgstart; i <= imgend; i+= imgrate) {
	    	var imgsrc = i + '.png';
	    	while (imgsrc.length < 9) {
	    		imgsrc = '0' + imgsrc;
	    	}
	    	var imgy = Math.floor( localyscale * (i - localstart) - 22.5 )
	    	imghtm += '<img src="thumbnails/' + imgsrc + '" style="top: ' + imgy +'px;" />';
	    }
	    $(".play-images").html(imghtm);
	    
	    $(".play-localtweets .tweet").each(function() {
	    	var el = $(this),
	    		t = parseInt(el.attr("data-timestamp"));
    		if (t > localstart && t < localend) {
    			el.show();
    		} else {
    			el.hide();
    		}
	    });
	    
	    localkeywords = _(localkeywords)
	    	.chain()
	    	.map(function(v,k) {
	    		return { word: k, score: v }
	    	})
	    	.sortBy(function(w) {
	    		return -w.score;
	    	})
	    	.first(30)
	    	.value();
    	
    	var values = _(localkeywords).pluck('score'),
    		max = Math.max.apply(Math, values),
    		min = Math.min.apply(Math, values),
    		scale = 10 / (max - Math.min(max - .1, min));
		
		localkeywords.forEach(function(w) {
			w.size = 10 + (w.score - min) * scale;
		});
    	
    	$(".play-tagcloud").html(localkeywords.map(cloudTemplate).join(""));
	    
		throttledGetTweets();
	    
		showTopicViz();
    }
    
    
    var lastPos, lastDuration, lastTopics;
    
    var tweetTemplate = _.template('<li class="tweet" data-timestamp="<%= timestamp %>" data-topic-id="<%= topic.topic %>"><img src="<%- data.profile_image_url %>" /><p>@<%- data.from_user_name %>: <%- data.text %></p></li>'),
    	callnum = 0,
    	tweetstructure = {},
    	requestedtweets = {},
    	_NTWEETS = 10;
    
    function showTweets() {
    	    	
    	var toshow = [];
    	var topics = Array.prototype.join.call($(".topic.selected").map(function(){return $(this).attr("data-topic-id")})).split(",");
    	
    	for (var i = 0; i < localMmsos.length; i++) {
    		var mmso = data.segments[localMmsoDelta + i];
    		var mmsostruct = tweetstructure[mmso.id];
    		if (mmsostruct) {
	    		for (var j = 0; j < topics.length; j++) {
	    			var topicid = topics[j];
					if (mmsostruct[topicid]) {
						var tweetids = mmsostruct[topicid].tweetids;
						for (var k = 0; k < tweetids.length; k++) {
							toshow.push(tweetids[k]);
						}
					}
	    		}
    		}
    	}
    	toshow = _(toshow).uniq();
    	    	
    	var tweetstoshow = toshow.map(function(tid) {
    		return requestedtweets[tid];
    	}).filter(function(tw) {
    		return ((tw.status == 2) && (tw.timestamp > (localpos - localduration / 2)) && (tw.timestamp < (localpos + localduration / 2)));
		});
		
		tweetstoshow.forEach(function(tw) {
			tw.topic = tw.topics.filter(function(t) {
				return topics.indexOf(t.topic) !== -1;
			}).sort(function(a,b) {
				return b.weight - a.weight
			})[0];
		});
		
		tweetstoshow = tweetstoshow.filter(function(t) {
			return !!t.topic;
		})
				
		tweetstoshow.sort(function(a, b) {
			return b.topic.weight - a.topic.weight; 
		});
		
		tweetstoshow = tweetstoshow.slice(0,10);
		
		tweetstoshow.sort(function(a, b) {
			return a.timestamp - b.timestamp;
		});
		
		var html = tweetstoshow.map(tweetTemplate).join("");
		
		$(".play-localtweets").html(html);
		var h = 0;
		$(".play-localtweets .tweet").each(function() {
			h += $(this).outerHeight();
		});
		$(".play-localtweets .tweet").css("margin-top",Math.max(0,($(".play-bottom").height() - h)/(tweetstoshow.length+1)));
		showTopicViz();
    	
    }
    
    function getMmsoTweetIds(mmstruct) {
    	console.log("TopicsBean.bestSocialInteractionsIdsMatching('" + mmstruct.mmso + "', " + mmstruct.topic + ", 0, " + _NTWEETS + ")");
    	TopicsBean.bestSocialInteractionsIdsMatching(mmstruct.mmso, mmstruct.topic, 0, _NTWEETS, {
			callback: function(tw) {
				mmstruct.status = 2;
				mmstruct.tweetids = tw;
				for (var k = 0; k < tw.length; k++) {
					var tweetid = tw[k],
						relevance = mmstruct.weight * (_NTWEETS - k) / _NTWEETS;
					if (!requestedtweets[tweetid]) {
						requestedtweets[tweetid] = {
							id: tweetid,
							status: 0,
							topics: []
						}
						requestedtweets[tweetid].topics.push({
							topic: j,
							weight: relevance
						});
					}
					requestedtweets[tweetid].topics.push({
						topic: mmstruct.topic,
						weight: relevance
					});
				}
				debouncedGetTweetData();
			},
			errorHandler: function(err,info) {
				mmstruct.status = 0;
				console.error(err,info);
				debouncedGetTweetIds();
			}
		});
    }
    
    var _MAX_BATCH = 20;
    
    function getTweetIds() {
    	
    	console.log("getTweetIds");
    	
    	var toload = [];
    	
    	_(tweetstructure).each(function(v) {
    		_(v).each(function(w) {
    			if (!w.status) {
    				w.status = 1;
    				toload.push(w);
    			}
    		});
    	});
    	
    	if (toload.length) {
    		
    		if (toload.length > _MAX_BATCH) {
    			toload = _(toload).shuffle().slice(0,_MAX_BATCH);
    			window.setInterval(throttledGetTweetIds,0);
    		}
    		console.log("Begin Batch");
    		dwr.engine.beginBatch();
    		toload.forEach(getMmsoTweetIds);
    		dwr.engine.endBatch();
    		console.log("End Batch");
    	}
    }
    
    function getTweetData() {
    	
    	console.log("getTweetData");
    	
    	var toload = [];
    	
    	_(requestedtweets).each(function(v) {
			if (!v.status) {
				toload.push(v.id);
			}
		});
    	
    	console.log("Loading tweet data");
		
		$.getJSON(
			solrUrl(
				"twitter",
				{
					q:"id:(" + toload.join(" OR ") + ")",
					fl: "id_str,created_at,from_user_name,text,profile_image_url",
					rows: toload.length
				}
			),
			function(t) {
				var tweets = t.response.docs;
				tweets.forEach(function(tweet) {
					var timestamp = new Date(tweet.created_at).valueOf() / 1000 - deltaT;
					requestedtweets[tweet.id_str].data = tweet;
					requestedtweets[tweet.id_str].status = 2;
					requestedtweets[tweet.id_str].timestamp = timestamp;
				});
				throttledShowTweets();
			}
		);
		
    }
    
    debouncedGetTweetData = _(getTweetData).debounce(125);
    
    throttledGetTweetIds = _(getTweetIds).throttle(10000);
    
    debouncedGetTweetIds = _(throttledGetTweetIds).debounce(125);
    
    throttledShowTweets = _(showTweets).throttle(200);
    
    function getLocalTweets() {
    	
    	console.log("getLocalTweets");
    	
    	var topics = Array.prototype.join.call($(".topic.selected").map(function(){return $(this).attr("data-topic-id")})).split(",");
    	
    	for (var i = 0; i < localMmsos.length; i++) {
    		var mmso = data.segments[localMmsoDelta + i];
    		if (!tweetstructure[mmso.id]) {
    			tweetstructure[mmso.id] = {}
    		}
    		var mmsostruct = tweetstructure[mmso.id];
    		for (var j = 0; j < topics.length; j++) {
				var topicid = topics[j],
					weight = data.topics[topics[j]].weights[localMmsoDelta + i];
				if (weight > .1 && !mmsostruct[topicid]) {
					mmsostruct[topicid] = {
						topic: topicid,
						mmso: mmso.id,
						status: 0,
						weight: weight,
						tweetids: []
					};
				}
    		}
    	}
    	
    	throttledGetTweetIds();
    	throttledShowTweets();
    }
    
    var throttledGetTweets = _.throttle(getLocalTweets, 500),
    	throttledShowLocal = _.throttle(showLocal, 100);
    
    showTopics(sortedTopics);
    
    pageParams.selectedtopics.forEach(function(id) {
    	$(".topic[data-topic-id=" + id + "]").addClass("selected").attr("data-timestamp",++ordertag);
    });
    
    var localpos = 300,
    	localduration = 600;
    	
    showLocal();
    
    $(".topics-block").on("mouseenter", ".topic", function() {
        var el = $(this);
        el.addClass("hover");
        showTopicViz();
    }).on("mouseleave", ".topic", function() {
        $(this).removeClass("hover");
        showTopicViz();
    }).on("click", ".topic", function() {
        var el = $(this);
        $(this).toggleClass("selected");
        el.attr("data-timestamp", el.hasClass("selected") ? ++ordertag : 999999);
        showTopicViz();
    });
    
    var mouseIsDown, isDragging, startY, startT, startPos, scrollGlobal, speedscale, slowiterations;
    
    $(document).mouseup(function() { mouseIsDown = false; isDragging = false; });
    
    $(".play-dataviz").mousedown(function(e) {
    	var l = $(this).offset().left,
    		scrollLimit = l + 380;
		if (e.pageX < scrollLimit) {
			mouseIsDown = true;
	    	startY = e.pageY;
	    	startT = new Date();
	    	startPos = localpos;
	    	scrollGlobal = e.pageX < (l + 140);
	    	e.preventDefault();
		}
    }).mousemove(function(e) {
    	if (mouseIsDown) {
    		if (isDragging) {
    			var limit = $(this).offset().left + 140,
    				deltaY = e.pageY - startY,
    				delta = Math.floor(deltaY / (scrollGlobal ? yscale : - localyscale));
				localpos = Math.max(localduration / 2, Math.min(data.duration - localduration / 2, startPos + delta));
				throttledShowLocal();
    		} else {
    			isDragging = true;
    		}
    	}
    }).mouseup(function(e) {
    	if (scrollGlobal && !isDragging) {
    		
    		var posY = e.pageY - $(this).offset().top;
    		localpos = Math.max(localduration / 2, Math.min(data.duration - localduration / 2, Math.floor(posY / yscale)));
			throttledShowLocal();
    	}
    });
    
    var totalScroll = 0, zoomlevels = [ 1800, 900, 600, 300, 120, 60 ], currentlevel = 2;
    
    $(".play-dataviz").mousewheel(function(_event, _scrolldelta) {
    	totalScroll += _scrolldelta;
	    if (Math.abs(totalScroll) >= 1) {
	    	var d = (totalScroll > 0 ? 1 : -1),
	    		newlevel = Math.max(0, Math.min(zoomlevels.length - 1, currentlevel + d));
    		if (newlevel !== currentlevel) {
    			currentlevel = newlevel;
    			localduration = zoomlevels[currentlevel];
    			localpos = Math.max(localduration / 2, Math.min(data.duration - localduration / 2, localpos));
    			throttledShowLocal();
    		}
	        totalScroll = 0;
	    }
    });
    
    if (pageParams.keywords && pageParams.keywords.length) {
    	$(".keyword-search a").removeClass("placeholder").text(pageParams.keywords.join(", "));
    }
    
}

var data = { duration: 10200, topics: [] },
    colorset = ["#E41A1C", "#377EB8", "#4DAF4A", "#984EA3", "#FF7F00", "#A65628", "#F781BF"];

$(function() {
	dwr.engine.setErrorHandler(function(a, b) { console.error("DWR", b); });
    
    $(".topics-block").draggable({axis:"x"});
    
    var loadedSteps = 0,
        stepsToFullyLoaded = 0;
    
    function checkIfLoaded() {
        loadedSteps++;
        if (loadedSteps >= stepsToFullyLoaded) {
            setTimeout(showData,0);
        }
    }
    
    function loadJson(url, propname, callback) {
        stepsToFullyLoaded++;
        $.getJSON(url, function(d) {
            if (callback) {
                var res = callback(d);
            } else {
                var res = d;
            }
            if (propname) {
                data[propname] = res;
            }
            checkIfLoaded();
        });
    }
    
    function loadFromTopicsBean(method, propname, args, callback) {
        stepsToFullyLoaded++;
        var arg = args || [],
            cb = function(d) {
                if (callback) {
                    var res = callback(d);
                } else {
                    var res = d;
                }
                if (propname) {
                    data[propname] = res;
                }
                checkIfLoaded();
            }
        arg.push({callback: cb});
        TopicsBean[method].apply(TopicsBean,arg);
    }
    
    loadJson("data/minutes.json", "minutes");
    loadJson("data/5secs.json", "fiveseconds");
    
    loadJson(
        solrUrl("MMSO", {q: "*:*", fl: "topic*,MMSO_id,multimediaSegment,keywordsFromSocial,keywordsFromAudio", rows: 250 }),
        "segments",
        function(d) {
            return d.response.docs.map(function(mmso) {
                var tc = mmso.multimediaSegment.match(/\d+/g),
                    start = parseInt(tc[0]),
                    end = parseInt(tc[1]),
                    topics = [],
                    keywords = {};
                
                function getKeywords(field) {
                	var keywordtexts = mmso[field].replace(/[{}]/g,'').split(", ");
	                keywordtexts.forEach(function(k) {
	                	var t = k.split('='),
	                		s = parseFloat(t[1]),
	                		w = t[0].split(" ");
	            		w.forEach(function(m) {
	            			if (m.length > 2) {
	            				keywords[m] = s + (keywords[m] || 0);
	            			}
	            		});
	                });
                }
                
                getKeywords("keywordsFromAudio");
                getKeywords("keywordsFromSocial");
                
                for (var k in mmso) {
                    if (k.substr(0,5) === "topic" && mmso[k] > .01) {
                        topics.push({
                            topic: parseInt(k.substr(5)),
                            weight: mmso[k]
                        })
                    }
                }
                topics.sort(function(a,b) {
                    return b.weight - a.weight;
                });
/*                topics = topics.filter(function(t) {
                    return t.topic !== topicPoubelle
                }).slice(0,1);
                topics[0].weight = 1; */
                return {
                    id: mmso.MMSO_id,
                    start: start,
                    end: end,
                    duration: end - start,
                    topics: topics,
                    keywords: keywords
                }
            }).sort(function(a,b) {
                return a.start - b.start;
            });
        })
    
    dwr.engine.setTimeout(60000);
    TopicsBean._path = "http://159.217.144.101:8050/sia-solr/dwr";
    
    loadFromTopicsBean("getTopicsNumber",false,false,function(topic_count) {
        for (var i = 0; i < topic_count; i++) {
            data.topics.push(null);
        }
        dwr.engine.beginBatch();
        data.topics.forEach(function(v, k) {
            loadFromTopicsBean("getTopicDistribution",false,[k, 50, false],function(topic) {
                var words = topic.match(/[^=,{]+=0.\d{0,8}/g);
                data.topics[k] = {
                    index: k,
                    words: words.map(function(w) {
                        var parts = w.split("=");
                        return {
                            word: parts[0].trim(),
                            weight: parseFloat(parts[1])
                        }
                    })
                }
            });
            
        });
        dwr.engine.endBatch();
        
    });
    
    
});