1 /* CONFIGURATION */ |
1 /* CALLING COMMON CONFIGURATION FILE */ |
2 |
2 |
3 RECORD_NEW_TWEETS = true; |
3 console.log('Reading Configuration from conf.js'); |
4 DEFAULT_SIO_PORT = 8000; |
4 |
5 /* Overriden par the "-p" parameter, e.g. node tweetcast.js -p 8080 */ |
5 var fs = require('fs'); |
6 SQLITE_FILE_DIR = __dirname + '/'; |
6 eval(fs.readFileSync(__dirname + '/conf.js','utf8')); |
7 SQLITE_FILE_START = 'tweets-'; |
7 |
8 SQLITE_FILE_EXT = '.sqlite'; |
8 /* SERVER-SIDE ONLY CONFIGURATION */ |
9 DEFAULT_TRACKING_KEYWORD = 'Bieber'; |
9 |
10 /* Overriden par the "-T" parameter, e.g. node tweetcast.js -T "Bieber" */ |
10 sqlfile = __dirname + '/tweets-' + encodeURIComponent(tracking_keyword) + '.sqlite'; |
11 TWITTER_USER = 'materiauxnum'; |
11 TWITTER_USER = 'materiauxnum'; |
12 TWITTER_PASS = 'm473r14ux7w337'; |
12 TWITTER_PASS = 'm473r14ux7w337'; |
|
13 RECORD_NEW_TWEETS = true; |
13 |
14 |
14 /* FUNCTIONS */ |
15 /* FUNCTIONS */ |
15 |
16 |
|
17 function annotationMap(callback, options) { |
|
18 var includeDefault = ( options && options.includeDefault ? options.includeDefault : false ); |
|
19 var returnObject = ( options && options.returnObject ? options.returnObject : false ); |
|
20 res = (returnObject ? {} : []); |
|
21 for (var i in annotations) { |
|
22 if (i != "default" || includeDefault) { |
|
23 var el = callback(i, annotations[i]) |
|
24 if (returnObject) { |
|
25 res[i] = el; |
|
26 } else { |
|
27 res.push(el); |
|
28 } |
|
29 } |
|
30 } |
|
31 return res; |
|
32 } |
|
33 |
16 function createTables() { |
34 function createTables() { |
17 |
35 |
18 var requete = "CREATE TABLE IF NOT EXISTS tweets ( pos INTEGER PRIMARY KEY, tweet_id TEXT UNIQUE, created_at INTEGER, json TEXT" + annotations.map(function(a) { return ', a_' + a + ' INTEGER' }).join("") + " )"; |
36 var requete = "CREATE TABLE IF NOT EXISTS tweets ( pos INTEGER PRIMARY KEY, tweet_id TEXT UNIQUE, created_at INTEGER, json TEXT" + annotationMap(function(a) { return ', a_' + a + ' INTEGER' }).join("") + " )"; |
19 db.execute(requete, function(err) { |
37 db.execute(requete, function(err) { |
20 if (err) throw err; |
38 if (err) throw err; |
21 db.execute("CREATE INDEX IF NOT EXISTS idx_created_at ON tweets ( created_at )", function(err) { if (err) throw err; }); |
39 db.execute("CREATE INDEX IF NOT EXISTS idx_created_at ON tweets ( created_at )", function(err) { if (err) throw err; }); |
22 getSendLastPos(); |
40 getSendLastPos(); |
23 }); |
41 }); |
50 textids(tweet.retweeted_status); |
68 textids(tweet.retweeted_status); |
51 for (var j in keys_to_delete) { |
69 for (var j in keys_to_delete) { |
52 delete tweet.retweeted_status[keys_to_delete[j]]; |
70 delete tweet.retweeted_status[keys_to_delete[j]]; |
53 } |
71 } |
54 } |
72 } |
55 for (var i in annotations_keywords) { |
73 annotationMap(function(i, annotation) { |
56 for (var j in annotations_keywords[i]) { |
74 for (var j in annotation.keywords) { |
57 if (tweet.text.indexOf(annotations_keywords[i][j]) != -1) { |
75 if (tweet.text.search(annotation.keywords[j]) != -1) { |
58 ann.push(annotations[i]); |
76 ann.push(i); |
59 break; |
77 break; |
60 } |
78 } |
61 } |
79 } |
62 } |
80 }); |
63 tweet.annotations = ann; |
81 tweet.annotations = ann; |
64 tweet.created_at = new Date(tweet.created_at); |
82 tweet.created_at = new Date(tweet.created_at); |
65 |
83 |
66 if (tweet.in_reply_to_status_id) { |
84 if (tweet.in_reply_to_status_id) { |
67 commitReference( tweet.id, tweet.in_reply_to_status_id, "reply" ); |
85 commitReference( tweet.id, tweet.in_reply_to_status_id, "reply" ); |
69 if (tweet.retweeted_status) { |
87 if (tweet.retweeted_status) { |
70 commitReference( tweet.id, tweet.retweeted_status.id, "retweet" ); |
88 commitReference( tweet.id, tweet.retweeted_status.id, "retweet" ); |
71 } |
89 } |
72 db.execute( |
90 db.execute( |
73 "INSERT INTO tweets ( tweet_id, created_at, json " |
91 "INSERT INTO tweets ( tweet_id, created_at, json " |
74 + annotations.map(function(a) { return ', a_' + a }).join("") |
92 + annotationMap(function(a) { return ', a_' + a }).join("") |
75 + " ) VALUES ( ?, ?, ? " |
93 + " ) VALUES ( ?, ?, ? " |
76 + annotations.map(function(a) { return ', ?' }).join("") |
94 + annotationMap(function(a) { return ', ?' }).join("") |
77 + " )", |
95 + " )", |
78 [ tweet.id, tweet.created_at.valueOf(), JSON.stringify(tweet) ].concat(annotations.map(function(a) { return ann.indexOf(a) == -1 ? 0 : 1 })), |
96 [ tweet.id, tweet.created_at.valueOf(), JSON.stringify(tweet) ].concat(annotationMap(function(a) { return ann.indexOf(a) == -1 ? 0 : 1 })), |
79 function(err) { |
97 function(err) { |
80 if (err) throw err; |
98 if (err) throw err; |
81 getSendLastPos(); |
99 getSendLastPos(); |
82 } |
100 } |
83 ); |
101 ); |
125 requete = "SELECT COUNT(*) AS nb, " |
143 requete = "SELECT COUNT(*) AS nb, " |
126 + lvl |
144 + lvl |
127 + "*ROUND(created_at/" |
145 + "*ROUND(created_at/" |
128 + lvl |
146 + lvl |
129 + ") AS tranche" |
147 + ") AS tranche" |
130 + annotations.map(function (a) { return " , SUM(a_" + a + ") AS s_" + a }).join("") |
148 + annotationMap(function (a) { return " , SUM(a_" + a + ") AS s_" + a }).join("") |
131 + " FROM tweets GROUP BY tranche ORDER BY tranche DESC LIMIT 0,50"; |
149 + " FROM tweets GROUP BY tranche ORDER BY tranche DESC LIMIT 0,50"; |
132 db.execute(requete, function (err, results) { |
150 db.execute(requete, function (err, results) { |
133 if (err) throw err; |
151 if (err) throw err; |
134 var tbl = [], |
152 var tbl = [], |
135 lastend = parseInt(results[results.length - 1].tranche); |
153 lastend = parseInt(results[results.length - 1].tranche); |
144 lastend += lvl; |
162 lastend += lvl; |
145 var struct = { |
163 var struct = { |
146 "start" : start, |
164 "start" : start, |
147 "end" : lastend, |
165 "end" : lastend, |
148 "tweets" : results[i].nb, |
166 "tweets" : results[i].nb, |
149 "annotations" : {} |
167 "annotations" : annotationMap(function (a) { |
150 } |
168 return results[i]['s_'+a]; |
151 for (var j in annotations) { |
169 },{returnObject: true}) |
152 struct.annotations[annotations[j]] = results[i]['s_' + annotations[j]]; |
|
153 } |
170 } |
154 tbl.push(struct); |
171 tbl.push(struct); |
155 } |
172 } |
156 socket.emit('timeline', tbl); |
173 socket.emit('timeline', tbl); |
157 }); |
174 }); |
166 } |
183 } |
167 } |
184 } |
168 |
185 |
169 function httpHandler(req, res) { |
186 function httpHandler(req, res) { |
170 console.log("HTTP Request for URL "+req.url); |
187 console.log("HTTP Request for URL "+req.url); |
171 var url = req.url + ( req.url[req.url.length - 1] == "/" ? "index.html" : "" ); |
188 var url = __dirname + ( req.url == "/conf.js" ? "" : "/client" ) + req.url + ( req.url[req.url.length - 1] == "/" ? "index.html" : "" ); |
172 fs.readFile(__dirname + "/client" + url, function(err, data) { |
189 fs.readFile( url, function(err, data) { |
173 if (err) { |
190 if (err) { |
174 console.log("Error 404"); |
191 console.log("Error 404"); |
175 res.writeHead(404); |
192 res.writeHead(404); |
176 return res.end('File not found'); |
193 return res.end('File not found'); |
177 } |
194 } |
180 }); |
197 }); |
181 } |
198 } |
182 |
199 |
183 /* Initialization */ |
200 /* Initialization */ |
184 |
201 |
185 var fs = require('fs'), |
202 var http = require('http'), |
186 http = require('http'), |
|
187 https = require('https'), |
203 https = require('https'), |
188 sqlite = require('sqlite'), |
204 sqlite = require('sqlite'), |
189 socketio = require('socket.io'), |
205 socketio = require('socket.io'), |
190 tweets = [], |
206 tweets = [], |
191 lastpos = 0, |
207 lastpos = 0, |
197 15 * 60 * 1000, |
213 15 * 60 * 1000, |
198 5 * 60 * 1000, |
214 5 * 60 * 1000, |
199 60 * 1000, |
215 60 * 1000, |
200 15 * 1000 |
216 15 * 1000 |
201 ], |
217 ], |
202 annotations = [ 'positive', 'negative', 'reference', 'question' ], |
|
203 annotations_keywords = [ [ '++' ], [ '--' ], [ '==' ], [ '??' ] ], |
|
204 annkw = { |
|
205 'positive' : '++', |
|
206 'negative' : '--', |
|
207 'reference' : '==', |
|
208 'question' : '??' |
|
209 }, |
|
210 keys_to_delete = [ |
218 keys_to_delete = [ |
211 'in_reply_to_screen_name', |
219 'in_reply_to_screen_name', |
212 'in_reply_to_user_id', |
220 'in_reply_to_user_id', |
213 'retweeted', |
221 'retweeted', |
214 'place', |
222 'place', |
255 'profile_background_color', |
263 'profile_background_color', |
256 'profile_background_tile', |
264 'profile_background_tile', |
257 'utc_offset' |
265 'utc_offset' |
258 ], |
266 ], |
259 app = http.createServer(httpHandler), |
267 app = http.createServer(httpHandler), |
260 port_flag = process.argv.indexOf("-p"), |
|
261 sio_port = ( port_flag != -1 && port_flag < process.argv.length - 1 && parseInt(process.argv[port_flag + 1]) ? parseInt(process.argv[port_flag + 1]) : DEFAULT_SIO_PORT ) |
|
262 io = socketio.listen(app), |
268 io = socketio.listen(app), |
263 track_flag = process.argv.indexOf("-T"), |
|
264 tracking_keyword = ( track_flag != -1 && track_flag < process.argv.length - 1 ? process.argv[track_flag + 1] : DEFAULT_TRACKING_KEYWORD ), |
|
265 sqlfile = SQLITE_FILE_DIR + SQLITE_FILE_START + encodeURIComponent(tracking_keyword) + SQLITE_FILE_EXT, |
|
266 db = new sqlite.Database(); |
269 db = new sqlite.Database(); |
267 |
270 |
268 /* MAIN CODE */ |
271 /* MAIN CODE */ |
269 |
272 |
270 app.listen(sio_port); |
273 app.listen(app_port); |
271 |
274 console.log("Listening on port: "+app_port); |
272 console.log("Listening on port: "+sio_port); |
|
273 console.log("Opening SQLITE file: "+sqlfile); |
275 console.log("Opening SQLITE file: "+sqlfile); |
274 db.open(sqlfile , function(err) { |
276 db.open(sqlfile , function(err) { |
275 if (err) throw err; |
277 if (err) throw err; |
276 createTables(); |
278 createTables(); |
277 }); |
279 }); |