diff -r 0a2505c3b547 -r 8276f3ff7a3f tweetcast/server-nodejs/tweetcast.js --- a/tweetcast/server-nodejs/tweetcast.js Mon Mar 05 18:41:11 2012 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,235 +0,0 @@ -READ_OLD_TWEETS = true; -RECORD_NEW_TWEETS = true; - -tweet_file = flagOption('-f', 'tweets.txt'); -app_port = flagOption('-p', 8000); -tracking_keyword = flagOption('-t', null); -user_pass = flagOption('-u', null); - -if (!user_pass) { - console.log("You must provide user credentials on the command-line, e.g. node tweetcast.js -u user:pass") - process.exit(); -} -if (!tracking_keyword) { - console.log("You must provide keyword(s) on the command-line, e.g. node tweetcast.js -t Bieber") - process.exit(); -} - -console.log("Tracking keyword "+tracking_keyword); - -var fs = require('fs'), - http = require('http'), - https = require('https'), - socketio = require('socket.io'), - app = http.createServer(httpHandler), - io = socketio.listen(app) - tweets = [], - tweet_ids = [], - keys_to_delete = [ - 'in_reply_to_screen_name', - 'in_reply_to_user_id', - 'retweeted', - 'place', - 'geo', - 'source', - 'contributors', - 'coordinates', - 'retweet_count', - 'favorited', - 'truncated', - 'possibly_sensitive' - ], - user_keys_to_delete = [ - 'default_profile_image', - 'show_all_inline_media', - 'contributors_enabled', - 'profile_sidebar_fill_color', - 'created_at', - 'lang', - 'time_zone', - 'profile_sidebar_border_color', - 'follow_request_sent', - 'profile_background_image_url', - 'profile_background_image_url_https', - 'followers_count', - 'description', - 'url', - 'geo_enabled', - 'profile_use_background_image', - 'default_profile', - 'following', - 'profile_text_color', - 'is_translator', - 'favourites_count', - 'listed_count', - 'friends_count', - 'profile_link_color', - 'protected', - 'location', - 'notifications', - 'profile_image_url_https', - 'statuses_count', - 'verified', - 'profile_background_color', - 'profile_background_tile', - 'utc_offset' - ], - content_types = { - css : "text/css; charset=utf-8", - html : "text/html; charset=utf-8", - js : "text/javascript; charset=utf-8", - png : "image/png" - }; - -function wrapContentType(ext) { - return { - "Content-Type" : ( content_types[ext] ? content_types[ext] : 'text/plain' ) - } -} - -function httpHandler(req, res) { - console.log("HTTP Request for URL "+req.url); - var url = req.url.split('?')[0]; - url = __dirname + "/client" + url + ( url[url.length - 1] == "/" ? "index.html" : "" ); - var ext = url.split('.').slice(-1)[0].toLowerCase(); - fs.readFile( url, function(err, data) { - if (err) { - console.log("Error 404"); - res.writeHead(404); - return res.end('File not found'); - } - res.writeHead(200, wrapContentType(ext)); - res.end(data); - }); -} - -function flagOption(flag, defaultValue) { - var flagPos = process.argv.indexOf(flag); - return ( flagPos != -1 && flagPos < process.argv.length - 1) ? process.argv[flagPos + 1] : defaultValue; -} - -function addToList(tweet) { - if (tweet_ids.indexOf(tweet.id) != -1) { - console.log("Error: Tweet already in list"); - return false; - } - tweets.push(tweet); - tweet_ids.push(tweet.id); - return true; -} - -function textids(object) { - for (var key in object) { - // Workaround for Unicode bug in socket.io. - - if (typeof object[key] == "string") { - var tmp = ''; - for (var i = 0; i < object[key].length; i++) { - tmp += ( object[key].charCodeAt(i) < 128 ? object[key].charAt(i) : "&#" + object[key].charCodeAt(i) + ";" ); - } - object[key] = tmp; - } - if (key.substr(-2) == 'id') { - object[key] = object[key + '_str']; - delete object[key + '_str']; - } - } -} - -function readTweetsFromFile(file_name) { - console.log("Trying to read tweets from " + file_name); - try { - var oldtweets = fs.readFileSync(file_name, 'utf8').split('\n'); - var tweetscopied = 0; - for (var i in oldtweets) { - if (oldtweets[i].length > 0) { - addToList(JSON.parse(oldtweets[i])); - tweetscopied++; - } - } - console.log(tweetscopied + " tweets copied"); - } - catch (err) { - console.log("Error opening "+file_name); - } -} - -function requestTweets() { - console.log("Fetching tweets from https://stream.twitter.com/1/statuses/filter.json") - var writestream = null; - var req = https.request({ - host: "stream.twitter.com", - path: "/1/statuses/filter.json", - method: "POST", - headers: { - 'Authorization': 'Basic ' + new Buffer( user_pass ).toString('base64'), - 'Content-Type': 'application/x-www-form-urlencoded' - } - }, function(res) { - writestream = fs.createWriteStream( tweet_file, { flags: 'a+', encoding: 'utf-8' } ); - console.log('Response received, status : ' + res.statusCode); - res.setEncoding('utf8'); - res.on('data', function(chunk) { - var newdata = chunk.split('\r\n'), - tweetpos = tweets.length; - try { - for (var i in newdata) { - if (newdata[i].length > 0) { - var tweet = JSON.parse(newdata[i]); - textids(tweet); - for (var j in keys_to_delete) { - delete tweet[keys_to_delete[j]]; - } - textids(tweet.user); - for (var j in user_keys_to_delete) { - delete tweet.user[user_keys_to_delete[j]]; - } - if (tweet.retweeted_status) { - textids(tweet.retweeted_status); - for (var j in keys_to_delete) { - delete tweet.retweeted_status[keys_to_delete[j]]; - } - } - /* tweet.date_value = Date.parse(tweet.created_at); */ - if (addToList(tweet)) { - writestream.write(JSON.stringify(tweet)+'\n'); - } - } - } - io.sockets.emit('update', { - "new_tweets" : tweets.slice(tweetpos) - }); - console.log("New tweets received. We now have", tweets.length, "tweets in memory"); - } - catch(err) { - console.log(err.message); - } - }); - }); - - req.write('track=' + encodeURIComponent( ( tracking_keyword ) ) ); - req.end(); -} - -app.listen(app_port); -console.log("Listening on port: "+app_port); - -if (READ_OLD_TWEETS) { - readTweetsFromFile(tweet_file); -} else { - console.log("Reading old tweets disabled !"); -} - -if (RECORD_NEW_TWEETS) { - requestTweets(); -} else { - console.log("Recording new tweets disabled !"); -} - -io.set('log level', 0); -io.sockets.on('connection', function(socket) { - console.log("New connection from "+socket.handshake.address.address); - socket.emit('initial_data', { - "tweets" : tweets - }); -}); \ No newline at end of file