TWEET_FILE_DIR = './';

TWEET_FILE_START = 'tweets-';

TWEET_FILE_EXT = '.txt';

TRACKING_KEYWORD = '#p2';

    arcs = [],

    };

    if (typeof start == "object") {

        start = start.valueOf();

    }

        struct.annotations = {};

                if (tweet.annotations.length) {

                    var ann = tweet.annotations[0];

                    if (slices[i].annotations[ann]) {

                        slices[i].annotations[ann].push(tweet.pos);

                        slices[i].annotations[ann] = [ tweet.pos ];

function getSliceContent(slice) {

    if (slice.slices) {

        var twz = [],

            annotations = {};

        for (var i in slice.slices) {

            var data = getSliceContent(slice.slices[i]);

            twz = twz.concat(data.tweets);

            for (var j in data.annotations) {

                if (annotations[j]) {

                    annotations[j] = annotations[j].concat(data.annotations[j]);

                } else {

                    annotations[j] = data.annotations[j];

                }

            }

        }

    } else {

        twz = slice.tweets;

        annotations = slice.annotations;

    }

    return { "tweets" : twz, "annotations" : annotations }

};

function flattenDateStruct(slices, target_level) {

    var current_level = slices[0].level,

        result = [];

    if (current_level < target_level) {

        if (slices[0].slices) {

            for (var i in slices) {

                result = result.concat(flattenDateStruct(slices[i].slices, target_level));

            }

        }

    }

    else {

        for (var i in slices) {

            var data = getSliceContent(slices[i]);

            result.push({

                "start" : slices[i].start,

                "end" : slices[i].end,

                "tweets" : data.tweets,

                "annotations" : data.annotations

            });

        }

    }

    return result;

}

function trimFDS(slices) {

    while (slices[0].tweets.length == 0) {

        slices.splice(0,1);

    }

    while (slices[slices.length - 1].tweets.length == 0) {

        slices.pop();

    }

    return slices;

}

        date_struct.push( populateDateStruct(0, date_struct[date_struct.length - 1].end.valueOf()) );

    if (tweet.in_reply_to_status_id) {

        var ref = tweet_ids.indexOf(tweet.in_reply_to_status_id)

        if (ref != -1) {

            arcs.push({

                "from" : tweet.pos,

                "to" : ref,

                "type" : "reply"

            });

        }

    }

    if (tweet.retweeted_status) {

        var ref = tweet_ids.indexOf(tweet.retweeted_status.id_str)

        if (ref != -1) {

            arcs.push({

                "from" : tweet.pos,

                "to" : ref,

                "type" : "retweet"

            });

        }

    }

function readTweetsFromFile() {

    var dir = fs.readdirSync(TWEET_FILE_DIR),

        fileName = null;

    for (var i in dir) {

        if ( dir[i].indexOf( TWEET_FILE_START + encodeURIComponent(TRACKING_KEYWORD) ) == 0 ) {

            var oldtweets = fs.readFileSync(dir[i], 'utf8').split('\n'),

                tweetscopied = 0;

            for (var j in oldtweets) {

                if (oldtweets[j].length > 1) {

                    addToList(JSON.parse(oldtweets[j]));

            console.log(tweetscopied, "tweets copied from", dir[i]);

            console.log(arcs);

        }

function callBackNewTweets(chunk) {

    var newdata = chunk.split('\r\n');

    for (var i in newdata) {

        if (newdata[i].length > 0) {

            var tweet = JSON.parse(newdata[i]),

                annotations = [];

            for (var a in annkw) {

                if (tweet.text.indexOf(annkw[a]) != -1) {

                    annotations.push(a);

                }

            }

            tweet.annotations = annotations;

            textids(tweet);

            textids(tweet.user);

            addToList(tweet);

            var txt = JSON.stringify(tweet)+'\n';

            writestream.write(txt);

        }

    }

    console.log("New tweets received. We now have", tweets.length, "tweets in memory");

    io.sockets.emit('tweetSummary', { tweetcount : tweets.length });

}

/* MAIN CODE */

if (READ_OLD_TWEETS) {

    readTweetsFromFile();

}

    var now = new Date(),

        fileTimestamp = (1900+now.getYear())+"-"+("0"+(1+now.getMonth())).slice(-2)+"-"+("0"+now.getDate()).slice(-2)+"."+("0"+now.getHours()).slice(-2)+"-"+("0"+now.getMinutes()).slice(-2),

        fileName = TWEET_FILE_DIR + TWEET_FILE_START + encodeURIComponent(TRACKING_KEYWORD) + '-' + fileTimestamp + TWEET_FILE_EXT,

        writestream = fs.createWriteStream(fileName, { flags: 'w', encoding: 'utf-8' });

        console.log('Writing to',fileName);

        res.on('data', callBackNewTweets);

    console.log("New connection from", socket.handshake.address.address,"id :", socket.id);

        console.log('getTweets from',socket.id);

        data.tweets = tweets.slice(Math.max(0, data.from), data.to);

        socket.emit('tweets', data);

    });

    socket.on('getTimeline', function(data) {

        socket.emit('timeline', { "timeline" : trimFDS(flattenDateStruct(date_struct, data.level)), "arcs" : arcs });

        console.log('getTimeline from',socket.id);