tweet_live: comparison script/utils/export_twitter

equal deleted inserted replaced

-:883d0724ffd0
+:e4d0094f097b
 #!/usr/bin/env python
 # coding=utf-8
 from lxml import etree
 from iri_tweet.models import setup_database, Tweet, User
-from optparse import OptionParser #@UnresolvedImport
 from sqlalchemy import Table, Column, BigInteger, event, bindparam
 from sqlalchemy.sql import select, func
 from iri_tweet.utils import (set_logging_options, set_logging, get_filter_query,
 get_logger)
+import argparse
 import anyjson
 import datetime
 import requests
 import os.path
 import re
 return polemics.keys()
 else:
 return None
 def get_options():
-parser = OptionParser()
-parser.add_option("-f", "--file", dest="filename",
+usage = "usage: %(prog)s [options]"
+parser = argparse.ArgumentParser(usage)
+parser.add_argument("-f", "--file", dest="filename",
 help="write export to file", metavar="FILE", default="project.ldt")
-parser.add_option("-d", "--database", dest="database",
+parser.add_argument("-d", "--database", dest="database",
 help="Input database", metavar="DATABASE")
-parser.add_option("-s", "--start-date", dest="start_date",
+parser.add_argument("-s", "--start-date", dest="start_date",
 help="start date", metavar="START_DATE", default=None)
-parser.add_option("-e", "--end-date", dest="end_date",
+parser.add_argument("-e", "--end-date", dest="end_date",
 help="end date", metavar="END_DATE", default=None)
-parser.add_option("-I", "--content-file", dest="content_file",
+parser.add_argument("-I", "--content-file", dest="content_file",
 help="Content file", metavar="CONTENT_FILE")
-parser.add_option("-c", "--content", dest="content",
+parser.add_argument("-c", "--content", dest="content",
 help="Content url", metavar="CONTENT")
-parser.add_option("-V", "--video-url", dest="video",
+parser.add_argument("-V", "--video-url", dest="video",
 help="video url", metavar="VIDEO")
-parser.add_option("-i", "--content-id", dest="content_id",
+parser.add_argument("-i", "--content-id", dest="content_id",
 help="Content id", metavar="CONTENT_ID")
-parser.add_option("-x", "--exclude", dest="exclude",
+parser.add_argument("-x", "--exclude", dest="exclude",
 help="file containing the id to exclude", metavar="EXCLUDE")
-parser.add_option("-C", "--color", dest="color",
+parser.add_argument("-C", "--color", dest="color",
 help="Color code", metavar="COLOR", default="16763904")
-parser.add_option("-H", "--hashtag", dest="hashtag",
+parser.add_argument("-H", "--hashtag", dest="hashtag",
 help="Hashtag", metavar="HASHTAG", default=[], action="append")
-parser.add_option("-D", "--duration", dest="duration", type="int",
+parser.add_argument("-D", "--duration", dest="duration", type=int,
 help="Duration", metavar="DURATION", default=None)
-parser.add_option("-n", "--name", dest="name",
+parser.add_argument("-n", "--name", dest="name",
 help="Cutting name", metavar="NAME", default=u"Tweets")
-parser.add_option("-R", "--replace", dest="replace", action="store_true",
+parser.add_argument("-R", "--replace", dest="replace", action="store_true",
-help="Replace tweet ensemble", metavar="REPLACE", default=False)
+help="Replace tweet ensemble", default=False)
-parser.add_option("-m", "--merge", dest="merge", action="store_true",
+parser.add_argument("-m", "--merge", dest="merge", action="store_true",
-help="merge tweet ensemble, choose the first ensemble", metavar="MERGE", default=False)
+help="merge tweet ensemble, choose the first ensemble", default=False)
-parser.add_option("-L", "--list-conf", dest="listconf",
+parser.add_argument("-L", "--list-conf", dest="listconf",
 help="list of file to process", metavar="LIST_CONF", default=None)
-parser.add_option("-E", "--extended", dest="extended_mode", action="store_true",
+parser.add_argument("-E", "--extended", dest="extended_mode", action="store_true",
-help="Trigger polemic extended mode", metavar="EXTENDED", default=False)
+help="Trigger polemic extended mode", default=False)
-parser.add_option("-b", "--base-url", dest="base_url",
+parser.add_argument("-b", "--base-url", dest="base_url",
 help="base URL of the platform", metavar="BASE_URL", default="http://ldt.iri.centrepompidou.fr/ldtplatform/")
-parser.add_option("-p", "--project", dest="project_id",
+parser.add_argument("-p", "--project", dest="project_id",
 help="Project id", metavar="PROJECT_ID", default=None)
-parser.add_option("-P", "--post-param", dest="post_param",
+parser.add_argument("-P", "--post-param", dest="post_param",
 help="Post param", metavar="POST_PARAM", default=None)
-parser.add_option("--user-whitelist", dest="user_whitelist", action="store",
+parser.add_argument("--user-whitelist", dest="user_whitelist", action="store",
 help="A list of user screen name", metavar="USER_WHITELIST",default=None)
 set_logging_options(parser)
-return parser.parse_args() + (parser,)
+return (parser.parse_args(), parser)
 if __name__ == "__main__" :
-(options, args, parser) = get_options()
+(options, parser) = get_options()
 set_logging(options)
 get_logger().debug("OPTIONS : " + repr(options)) #@UndefinedVariable
 if content_file and content_file.find("http") == 0:
 get_logger().debug("url : " + content_file) #@UndefinedVariable
 r = requests.get(content_file, params=post_param)
-#get_logger().debug("url response " + repr(r) + " content " + repr(r.text)) #@UndefinedVariable
+get_logger().debug("url response " + repr(r) + " content " + repr(r.text)) #@UndefinedVariable
 project = r.json()
 text_match = re.match(r"\<\?\s*xml.*?\?\>(.*)", project['ldt'], re.I|re.S)
 root = etree.fromstring(text_match.group(1) if text_match else project['ldt'])
 elif content_file and os.path.exists(content_file):
 content_node = annotations_node.find(u"content")
 if content_node is None:
 content_node = etree.SubElement(annotations_node,u"content", id=media.get(u"id"))
 ensemble_parent = content_node
 content_id = content_node.get(u"id")
+display_nodes = root.xpath("//displays/display/content[@id='%s']" % content_id)
+if len(display_nodes) == 0:
+get_logger().info("No display node found. Will not update display")
+display_content_node = None
+else:
+display_content_node = display_nodes[0]
 elif file_type == "iri":
 body_node = root.find(u"body")
 if body_node is None:
 body_node = etree.SubElement(root, u"body")
 ensembles_node = body_node.find(u"ensembles")
 if ensembles_node is None:
 ensembles_node = etree.SubElement(body_node, u"ensembles")
 ensemble_parent = ensembles_node
 content_id = root.xpath("head/meta[@name='id']/@content")[0]
+display_content_node = None
 if ensemble_parent is None:
 get_logger().error("Can not process file") #@UndefinedVariable
 sys.exit()
 if options.replace:
 for ens in ensemble_parent.iterchildren(tag=u"ensemble"):
-if ens.get("id","").startswith("tweet_"):
+ens_id = ens.get("id","")
+if ens_id.startswith("tweet_"):
 ensemble_parent.remove(ens)
+# remove in display nodes
+if display_content_node is not None:
+for cut_display in display_content_node.iterchildren():
+if cut_display.get('idens','') == ens_id:
+display_content_node.remove(cut_display)
 ensemble = None
 elements = None
 if options.merge:
-ensemble = ensemble_parent.find(u"ensemble")
+for ens in ensemble_parent.findall(u"ensemble"):
-if ensemble is not None:
+if ens.get('id',"").startswith("tweet_"):
-elements = ensemble.find(u".//elements")
+ensemble = ens
+break
+if ensemble is not None:
+elements = ensemble.find(u".//elements")
+decoupage = ensemble.find(u"decoupage")
 if ensemble is None or elements is None:
 ensemble = etree.SubElement(ensemble_parent, u"ensemble", {u"id":u"tweet_" + unicode(uuid.uuid4()), u"title":u"Ensemble Twitter", u"author":u"IRI Web", u"abstract":u"Ensemble Twitter"})
 decoupage = etree.SubElement(ensemble, u"decoupage", {u"id": unicode(uuid.uuid4()), u"author": u"IRI Web"})
 etree.SubElement(decoupage, u"title").text = unicode(options.name)
 etree.SubElement(decoupage, u"abstract").text = unicode(options.name)
 elements = etree.SubElement(decoupage, u"elements")
+ensemble_id = ensemble.get('id', '')
+decoupage_id = decoupage.get('id', '') if decoupage is not None else None
 end_date = None
 if end_date_str:
 end_date = parse_date(end_date_str)
 elif start_date and duration:
 # remove all elements and put them in a array
 # sort them with tc
 #put them back
 elements[:] = sorted(elements,key=lambda n: int(n.get('begin')))
+#add to display node
+if display_content_node is not None:
+display_dec = None
+for dec in display_content_node.iterchildren(tag=u"decoupage"):
+if dec.get('idens','') == ensemble_id and dec.get('id', '') == decoupage_id:
+display_dec = dec
+break
+if display_dec is None and ensemble_id and decoupage_id:
+etree.SubElement(display_content_node, u"decoupage", attrib={'idens': ensemble_id, 'id': decoupage_id, 'tagsSelect':''})
 output_data = etree.tostring(root, encoding="utf-8", method="xml", pretty_print=False, xml_declaration=True)
 if content_file_write and content_file_write.find("http") == 0:
 project["ldt"] = output_data
+project['owner'] = project['owner'].replace('%7E','~')
+project['contents'] = [c_url.replace('%7E','~') for c_url in project['contents']]
 post_param = {}
 if options.post_param:
 post_param = anyjson.loads(options.post_param)
 get_logger().debug("write http " + content_file_write) #@UndefinedVariable

changeset 957	e4d0094f097b
parent 888	6fc6637d8403
child 1023	7d87ba8cc268