1 from models import (Tweet, User, Hashtag, EntityHashtag, EntityUser, Url, |
1 from models import (Tweet, User, Hashtag, EntityHashtag, EntityUser, Url, |
2 EntityUrl, CONSUMER_KEY, CONSUMER_SECRET, APPLICATION_NAME, ACCESS_TOKEN_KEY, |
2 EntityUrl, CONSUMER_KEY, CONSUMER_SECRET, APPLICATION_NAME, ACCESS_TOKEN_KEY, |
3 ACCESS_TOKEN_SECRET, adapt_date, adapt_json, TweetSource, TweetLog, MediaType, |
3 ACCESS_TOKEN_SECRET, adapt_date, adapt_json, TweetSource, TweetLog, MediaType, |
4 Media, EntityMedia, Entity, EntityType) |
4 Media, EntityMedia, Entity, EntityType) |
5 from sqlalchemy.sql import select, or_ #@UnresolvedImport |
5 from sqlalchemy.sql import select, or_ #@UnresolvedImport |
|
6 import Queue #@UnresolvedImport |
6 import anyjson #@UnresolvedImport |
7 import anyjson #@UnresolvedImport |
7 import datetime |
8 import datetime |
8 import email.utils |
9 import email.utils |
9 import logging #@UnresolvedImport |
10 import logging |
10 import os.path |
11 import os.path |
11 import sys |
12 import sys |
12 import twitter.oauth #@UnresolvedImport |
13 import twitter.oauth #@UnresolvedImport |
13 import twitter.oauth_dance #@UnresolvedImport |
14 import twitter.oauth_dance #@UnresolvedImport |
14 import twitter_text #@UnresolvedImport |
15 import twitter_text #@UnresolvedImport |
15 |
|
16 |
16 |
17 |
17 |
18 CACHE_ACCESS_TOKEN = {} |
18 CACHE_ACCESS_TOKEN = {} |
19 |
19 |
20 def get_oauth_token(token_file_path=None, check_access_token=True, application_name=APPLICATION_NAME, consumer_key=CONSUMER_KEY, consumer_secret=CONSUMER_SECRET): |
20 def get_oauth_token(token_file_path=None, check_access_token=True, application_name=APPLICATION_NAME, consumer_key=CONSUMER_KEY, consumer_secret=CONSUMER_SECRET): |
446 self.obj_buffer.add_object(TweetLog, None, {'tweet_source_id':self.source_id, 'status':TweetLog.TWEET_STATUS['OK']}, True) |
446 self.obj_buffer.add_object(TweetLog, None, {'tweet_source_id':self.source_id, 'status':TweetLog.TWEET_STATUS['OK']}, True) |
447 |
447 |
448 self.obj_buffer.persists(self.session) |
448 self.obj_buffer.persists(self.session) |
449 |
449 |
450 |
450 |
451 def set_logging(options, plogger=None): |
451 def set_logging(options, plogger=None, queue=None): |
452 |
452 |
453 logging_config = { |
453 logging_config = { |
454 "format" : '%(asctime)s %(levelname)s:%(name)s:%(message)s', |
454 "format" : '%(asctime)s %(levelname)s:%(name)s:%(message)s', |
455 "level" : max(logging.NOTSET, min(logging.CRITICAL, logging.WARNING - 10 * options.verbose + 10 * options.quiet)), #@UndefinedVariable |
455 "level" : max(logging.NOTSET, min(logging.CRITICAL, logging.WARNING - 10 * options.verbose + 10 * options.quiet)), #@UndefinedVariable |
456 } |
456 } |
464 |
464 |
465 logger = plogger |
465 logger = plogger |
466 if logger is None: |
466 if logger is None: |
467 logger = get_logger() #@UndefinedVariable |
467 logger = get_logger() #@UndefinedVariable |
468 |
468 |
469 if len(logger.handlers) == 0: |
469 if len(logger.handlers) == 0: |
470 filename = logging_config.get("filename") |
470 filename = logging_config.get("filename") |
471 if filename: |
471 if queue is not None: |
|
472 hdlr = QueueHandler(queue, True) |
|
473 elif filename: |
472 mode = logging_config.get("filemode", 'a') |
474 mode = logging_config.get("filemode", 'a') |
473 hdlr = logging.FileHandler(filename, mode) #@UndefinedVariable |
475 hdlr = logging.FileHandler(filename, mode) #@UndefinedVariable |
474 else: |
476 else: |
475 stream = logging_config.get("stream") |
477 stream = logging_config.get("stream") |
476 hdlr = logging.StreamHandler(stream) #@UndefinedVariable |
478 hdlr = logging.StreamHandler(stream) #@UndefinedVariable |
|
479 |
477 fs = logging_config.get("format", logging.BASIC_FORMAT) #@UndefinedVariable |
480 fs = logging_config.get("format", logging.BASIC_FORMAT) #@UndefinedVariable |
478 dfs = logging_config.get("datefmt", None) |
481 dfs = logging_config.get("datefmt", None) |
479 fmt = logging.Formatter(fs, dfs) #@UndefinedVariable |
482 fmt = logging.Formatter(fs, dfs) #@UndefinedVariable |
480 hdlr.setFormatter(fmt) |
483 hdlr.setFormatter(fmt) |
481 logger.addHandler(hdlr) |
484 logger.addHandler(hdlr) |
482 level = logging_config.get("level") |
485 level = logging_config.get("level") |
483 if level is not None: |
486 if level is not None: |
484 logger.setLevel(level) |
487 logger.setLevel(level) |
485 |
488 |
486 options.debug = (options.verbose-options.quiet > 0) |
489 options.debug = (options.verbose-options.quiet > 0) |
|
490 return logger |
487 |
491 |
488 def set_logging_options(parser): |
492 def set_logging_options(parser): |
489 parser.add_option("-l", "--log", dest="logfile", |
493 parser.add_option("-l", "--log", dest="logfile", |
490 help="log to file", metavar="LOG", default="stderr") |
494 help="log to file", metavar="LOG", default="stderr") |
491 parser.add_option("-v", dest="verbose", action="count", |
495 parser.add_option("-v", dest="verbose", action="count", |
534 |
538 |
535 query = query.filter(or_(*map(lambda h: Hashtag.text.contains(h), htags))) #@UndefinedVariable |
539 query = query.filter(or_(*map(lambda h: Hashtag.text.contains(h), htags))) #@UndefinedVariable |
536 |
540 |
537 return query.distinct() |
541 return query.distinct() |
538 |
542 |
|
543 logger_name = "iri.tweet" |
|
544 |
539 def get_logger(): |
545 def get_logger(): |
540 return logging.getLogger("iri_tweet") #@UndefinedVariable |
546 global logger_name |
|
547 return logging.getLogger(logger_name) #@UndefinedVariable |
|
548 |
|
549 |
|
550 # Next two import lines for this demo only |
|
551 |
|
552 class QueueHandler(logging.Handler): #@UndefinedVariable |
|
553 """ |
|
554 This is a logging handler which sends events to a multiprocessing queue. |
|
555 """ |
|
556 |
|
557 def __init__(self, queue, ignore_full): |
|
558 """ |
|
559 Initialise an instance, using the passed queue. |
|
560 """ |
|
561 logging.Handler.__init__(self) #@UndefinedVariable |
|
562 self.queue = queue |
|
563 self.ignore_full = True |
|
564 |
|
565 def emit(self, record): |
|
566 """ |
|
567 Emit a record. |
|
568 |
|
569 Writes the LogRecord to the queue. |
|
570 """ |
|
571 try: |
|
572 ei = record.exc_info |
|
573 if ei: |
|
574 dummy = self.format(record) # just to get traceback text into record.exc_text |
|
575 record.exc_info = None # not needed any more |
|
576 if not self.ignore_full or not self.queue.full(): |
|
577 self.queue.put_nowait(record) |
|
578 except Queue.Full: |
|
579 if self.ignore_full: |
|
580 pass |
|
581 else: |
|
582 raise |
|
583 except (KeyboardInterrupt, SystemExit): |
|
584 raise |
|
585 except: |
|
586 self.handleError(record) |