# HG changeset patch # User ymh # Date 1430261585 -7200 # Node ID dd3b4c9d5035116d967aaf53891f1d6378769f6f # Parent 0c03dba0c5156e133c9ba01c2acc5c41a0b5853c add command to create media and content diff -r 0c03dba0c515 -r dd3b4c9d5035 .settings/org.eclipse.core.resources.prefs --- a/.settings/org.eclipse.core.resources.prefs Thu Apr 09 00:06:43 2015 +0200 +++ b/.settings/org.eclipse.core.resources.prefs Wed Apr 29 00:53:05 2015 +0200 @@ -39,6 +39,7 @@ encoding//src/ldt/ldt/ldt_utils/migrations/0028_all_users_in_everyone.py=utf-8 encoding//src/ldt/ldt/ldt_utils/migrations/0029_tagging_to_taggit.py=utf-8 encoding//src/ldt/ldt/ldt_utils/views/json.py=utf-8 +encoding//src/ldt/ldt/management/commands/createmediacontent.py=utf-8 encoding//src/ldt/ldt/management/commands/synciri.py=utf-8 encoding//src/ldt/ldt/management/commands/updateiriurlinprojects.py=utf-8 encoding//src/ldt/ldt/management/utils.py=utf-8 diff -r 0c03dba0c515 -r dd3b4c9d5035 src/ldt/ldt/api/ldt/resources/project.py --- a/src/ldt/ldt/api/ldt/resources/project.py Thu Apr 09 00:06:43 2015 +0200 +++ b/src/ldt/ldt/api/ldt/resources/project.py Wed Apr 29 00:53:05 2015 +0200 @@ -5,7 +5,7 @@ from django.conf.urls import url from django.contrib.auth.models import Group from django.core.exceptions import ObjectDoesNotExist -from guardian.shortcuts import assign +from guardian.shortcuts import assign_perm from tastypie import fields, http from tastypie.authorization import Authorization from tastypie.exceptions import BadRequest @@ -72,11 +72,11 @@ unprotect_models() bundle = super(ProjectResource, self).obj_create(bundle, **kwargs) # Assign permission for the owner - assign('view_project', request.user, bundle.obj) - assign('change_project', request.user, bundle.obj) + assign_perm('view_project', request.user, bundle.obj) + assign_perm('change_project', request.user, bundle.obj) # Since the project is published by default, we assign permission for the everyone group everyone = Group.objects.get(name=settings.PUBLIC_GROUP_NAME) - assign('ldt_utils.view_project', everyone, bundle.obj) + assign_perm('ldt_utils.view_project', everyone, bundle.obj) protect_models() return bundle diff -r 0c03dba0c515 -r dd3b4c9d5035 src/ldt/ldt/indexation/search_indexes.py --- a/src/ldt/ldt/indexation/search_indexes.py Thu Apr 09 00:06:43 2015 +0200 +++ b/src/ldt/ldt/indexation/search_indexes.py Wed Apr 29 00:53:05 2015 +0200 @@ -74,4 +74,4 @@ def index_queryset(self, using=None): "Used when the entire index for model is updated." - return self.get_model().objects.prefetch_related("tags") \ No newline at end of file + return self.get_model().objects.prefetch_related("tags") diff -r 0c03dba0c515 -r dd3b4c9d5035 src/ldt/ldt/ldt_utils/models.py --- a/src/ldt/ldt/ldt_utils/models.py Thu Apr 09 00:06:43 2015 +0200 +++ b/src/ldt/ldt/ldt_utils/models.py Wed Apr 29 00:53:05 2015 +0200 @@ -1,7 +1,11 @@ import datetime -import lxml.etree #@UnresolvedImport +import logging +import lxml.etree # @UnresolvedImport +import mimetypes import os.path import re +from shutil import move +import uuid from django.conf import settings from django.contrib.auth import get_user_model @@ -9,7 +13,7 @@ from django.core.files.storage import default_storage from django.db import models from django.utils.translation import ugettext_lazy as _ -from guardian.shortcuts import assign, remove_perm, get_perms +from guardian.shortcuts import assign_perm, remove_perm, get_perms from sorl.thumbnail import ImageField from taggit.managers import TaggableManager @@ -20,17 +24,16 @@ from ldt.security.models import SafeModel from ldt.utils import generate_hash, url as url_utils from ldt.utils.web_url_management import get_web_url -import mimetypes -from shutil import move from utils import create_ldt, copy_ldt, create_empty_iri, update_iri, \ generate_uuid -import uuid from .events import post_project_save User = get_user_model() +logger = logging.getLogger(__name__); + class Author(SafeModel): handle = models.CharField(max_length=255, unique=True, blank=True, null=True) @@ -103,7 +106,7 @@ def fget(self): res_src = self.src.rstrip() if self.videopath and self.videopath.startswith("rtmp://") and "mp3:" not in res_src and "mp4:" not in res_src: - extension = res_src.split(".")[-1] + extension = res_src.split(".")[-1].lower() res_src = { 'flv': lambda s: s, 'mp3': lambda s: "%s:%s" % ("mp3", res_src[:-4]), @@ -130,7 +133,7 @@ if self.pk: everyone = Group.objects.get(name=settings.PUBLIC_GROUP_NAME) if value: - assign('ldt_utils.view_media', everyone, self) + assign_perm('ldt_utils.view_media', everyone, self) else: remove_perm('ldt_utils.view_media', everyone, self) @@ -277,6 +280,7 @@ created = False try: iri_file_path = self.iri_file_path() + logger.debug("IRI_FILE_PATH %s" % iri_file_path) if not os.path.exists(iri_file_path): iri_dir = os.path.dirname(iri_file_path) if not os.path.exists(iri_dir): @@ -287,7 +291,6 @@ else: created = False update_iri(iri_file_path, self, "IRI") - except Exception, e: if created: if os.path.exists(iri_file_path): @@ -301,7 +304,7 @@ create_front_project = False # update it - self.sync_iri_file() + self.sync_iri_file() if not self.pk: create_front_project = True @@ -312,11 +315,11 @@ # We need a primary key for self in create_project, so # save() has to be called first self.create_front_project() - assign('ldt_utils.change_content', get_current_user(), self) + assign_perm('ldt_utils.change_content', get_current_user(), self) def __unicode__(self): - return str(self.id) + ":" + self.iri_id + ":" + self.title.replace("\n", " ") + return str(self.id) + ":" + self.iri_id + ":" + self.title.replace("\n", " ") if self.title else "" def iri_url(self, web_url=None): if not web_url: @@ -442,7 +445,7 @@ if self.pk: everyone = Group.objects.get(name=settings.PUBLIC_GROUP_NAME) if value: - assign('ldt_utils.view_content', everyone, self) + assign_perm('ldt_utils.view_content', everyone, self) else: remove_perm('ldt_utils.view_content', everyone, self) @@ -480,11 +483,6 @@ return proj - # Tag management - def get_tags(self): - return Tag.objects.get_for_object(self) - - # add polemic attributes and polemic attribute rates to class Content def __add_polemic_attributes(self): for element in POL_INDICES.keys(): @@ -630,7 +628,7 @@ def get_xml_doc(self): #remove the xml header declaration - return lxml.etree.fromstring(re.sub(r"^<\?\s*xml .*\?>", "", self.ldt)) + return lxml.etree.fromstring(re.sub(r"^<\?\s*xml .*\?>", "", self.ldt)) # @UndefinedVariable def __unicode__(self): return unicode(self.id) + u"::" + unicode(self.ldt_id) + u"::" + unicode(self.title) @@ -693,8 +691,8 @@ project.changed_by = user.username project.state = 1 project.save() - assign('view_project', user, project) - assign('change_project', user, project) + assign_perm('view_project', user, project) + assign_perm('change_project', user, project) for content in contents: project.contents.add(content) @@ -709,10 +707,10 @@ def copy_project(self, user, title, description='', group=None): project = Project(title=title, owner=user, description=description) project = copy_ldt(self, project, user) - assign('view_project', user, project) - assign('change_project', user, project) + assign_perm('view_project', user, project) + assign_perm('change_project', user, project) if group: - assign('view_project', group, project) + assign_perm('view_project', group, project) for content in self.contents.all(): project.contents.add(content) return project @@ -722,9 +720,9 @@ self.save() self.state = 2 everyone = Group.objects.get(name=settings.PUBLIC_GROUP_NAME) - assign('ldt_utils.view_project', everyone, self) + assign_perm('ldt_utils.view_project', everyone, self) if allow_write: - assign('ldt_utils.change_project', everyone, self) + assign_perm('ldt_utils.change_project', everyone, self) self.save() def unpublish(self): diff -r 0c03dba0c515 -r dd3b4c9d5035 src/ldt/ldt/ldt_utils/views/content.py --- a/src/ldt/ldt/ldt_utils/views/content.py Thu Apr 09 00:06:43 2015 +0200 +++ b/src/ldt/ldt/ldt_utils/views/content.py Wed Apr 29 00:53:05 2015 +0200 @@ -53,16 +53,17 @@ # If the url is kind of rtmp://site/flv:path/to/file.flv or rtmp://site/mp4:path/to/file.mp4, we parse it. # If the url is kind of rtmp://site/path/to/file, we don't parse it because we can't get the right streamer. a = cleaned_data['src'].split(":") - if len(a)==3 and (a[1].endswith("flv") or a[1].endswith("mp4") or a[1].endswith("mp3")): + filepart = a[1].lower() if len(a) == 3 else "" + if len(a)==3 and (filepart.endswith("flv") or filepart.endswith("mp4") or filepart.endswith("mp3")): # We update with the good streamer cleaned_data['videopath'] = a[0] + ":" + a[1][:-3] # We remove the "flv:" from the url because it's useless in the real url - if a[1].endswith("flv"): + if filepart.endswith("flv"): cleaned_data['src'] = a[2] else: cleaned_data['src'] = a[1][-3:] + ":" + a[2] # We remove the ".mp3" at the the end of the src if necessary - if a[1].endswith("mp3") and cleaned_data['src'].endswith(".mp3"): + if filepart.endswith("mp3") and cleaned_data['src'].endswith(".mp3"): cleaned_data['src'] = cleaned_data['src'][:-4] # We get or create the media with the correct datas media, created = Media.objects.get_or_create(src=cleaned_data['src'], defaults=cleaned_data) #@UndefinedVariable diff -r 0c03dba0c515 -r dd3b4c9d5035 src/ldt/ldt/ldt_utils/views/lignesdetemps.py --- a/src/ldt/ldt/ldt_utils/views/lignesdetemps.py Thu Apr 09 00:06:43 2015 +0200 +++ b/src/ldt/ldt/ldt_utils/views/lignesdetemps.py Wed Apr 29 00:53:05 2015 +0200 @@ -262,7 +262,7 @@ elem.set('video', c.videopath) else : elem.set('video', '') - elem.set('title', c.title) + elem.set('title', c.title or '') elem.set('author', '') elem.set('category', '') elem.set('pict', '') diff -r 0c03dba0c515 -r dd3b4c9d5035 src/ldt/ldt/management/commands/createmediacontent.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/ldt/ldt/management/commands/createmediacontent.py Wed Apr 29 00:53:05 2015 +0200 @@ -0,0 +1,287 @@ +# -*- coding: utf-8 -*- +''' +Created on Apr 10, 2015 + +@author: ymh +''' + +import logging +from optparse import make_option +import time +import urlparse + +from django.conf import settings +from django.contrib.auth import get_user_model +from django.contrib.auth.models import Group +from django.core.management.base import BaseCommand, CommandError +from guardian.shortcuts import remove_perm + +from ldt.ldt_utils.models import Content, Media +from ldt.ldt_utils.stat import update_stat_content +from ldt.ldt_utils.utils import generate_uuid +from ldt.security.cache import cached_assign +from ldt.security.utils import assign_perm_to_obj + + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + + formats = ["%M:%S", "%H:%M:%S", "%Hh%M", "%Ss", "%Ssec", "%Mmn %Ss"] + + def get_duration(self, value): + dur = None + for f in self.formats: + try: + dur = time.strptime(value, f) + dur = (dur.tm_hour*3600 + dur.tm_min*60 + dur.tm_sec) * 1000 + break + except: + pass + return dur + + + help = 'Create a content (and Media)' + option_list = BaseCommand.option_list + ( + + #TODO: source is compulsory + make_option("-s", "--source", + dest="source", + action="store", + type="string", + help="The media source"), + #TODO: title is compulsory + make_option("-t", "--title", + dest="title", + action="store", + type="string", + help="The content title"), + #TODO: optional + make_option("-V", "--videopath", + dest="videopath", + action="store", + type="string", + default=settings.STREAM_URL, + help="The media videopath"), + #TODO: optional + make_option("--media-title", + dest="media_title", + action="store", + type="string", + help="The media title. Optional, same as content title"), + + make_option("--description", + dest="description", + action="store", + type="string", + default='', + help="The description. Optional, same as content title"), + + make_option("--media-description", + dest="media_description", + action="store", + type="string", + help="The media description. Optional, same as content description"), + + #TODO: optional + make_option("-d", "--duration", + dest="duration", + action="store", + type="string", + help="Media duration"), + + make_option("-p", "--public", + dest="public", + action="store_true", + default=False, + help="Is the media/content public ?" + ), + + make_option("-r", "--read", + dest="readers", + action="store", + type="string", + default=None, + help="comma separated list of user/group with read access" + ), + make_option("-w", "--write", + dest="writers", + action="store", + type="string", + default=None, + help="comma separated list of user/group with write access" + ), + + make_option("-u", "--update", + dest='update', + action="store_true", + default=False, + help="update objects when not created" + ), + + make_option("-T", "--tags", + dest='tags', + action="store", + type="string", + default='', + help="comma separated list of tags" + ), + make_option("-c", "--creator", + dest='creator', + action="store", + type="string", + default=None, + help="The handle for object creator." + ), + + make_option("--external-id", + dest='external_id', + action="store", + type="string", + default = None, + help="The media external id" + ), + ) + + + def handle(self, *args, **options): + #BaseCommand.handle(self, *args, **options) + + title = options.get('title', '') + if not title: + raise CommandError("Title s compulsory") + title = title.decode("utf-8") + + media_title = options.get('media_title', '') + media_title = title if not media_title else media_title.decode("utf-8") + + src = options.get('source', None) + if src: + src = src.decode("utf-8") + + if not src: + raise CommandError("Source option is compulsory") + videopath = options.get('videopath', settings.STREAM_URL) + url_parts = urlparse.urlparse(src) + if url_parts.scheme in ['http','rtmp']: + videopath = '' + #TODO : test url is relative path + duration_str = options.get('duration', '') + duration = self.get_duration(duration_str) + if duration is None: + raise CommandError("Duration %r not recognized" % duration_str) + + description = options.get('description', '') + if description: + description = description.decode("utf-8") + + media_description = options.get('media_description', '') + + if not media_description: + media_description = description + else: + media_description.decode("utf-8") + + public = options.get('public', False) + + tags_str = options.get('tags', '') + if tags_str: + tags_str = tags_str.decode("utf-8") + + tags = filter(lambda t: t, [t.strip() for t in tags_str.split(",")]) if tags_str else [] + + creator_str = options.get("creator", None) + User = get_user_model() + + creator = None + + if creator_str: + try: + creator = User.objects.get(username=creator_str) + except User.DoesNotExist: + raise CommandError("Creator user %s not found" % creator_str) + else: + #TODO: replace by first() + try: + creator = User.objects.filter(is_superuser=True)[0] + except IndexError: + creator = None + + if creator is None: + raise CommandError("Creator not found") + + readers_str = options.get("readers", "") + writers_str = options.get("writers", "") + + readers = filter(lambda s: s, [r.strip() for r in readers_str.split(',')]) if readers_str else [] + writers = filter(lambda s: s, [r.strip() for r in writers_str.split(',')]) if writers_str else [] + + readers_list = set(readers+writers) + + + if readers_list: + read_list = list(User.objects.filter(username__in=readers_list)) + list(Group.objects.filter(name__in=readers_list)) + else: + read_list = [] + + if writers: + write_list = list(User.objects.filter(username__in = writers)) + list(Group.objects.filter(name__in = writers)) + else: + write_list = [] + + external_id = options.get("external_id", None) + + update_objects = options.get('update', False) + + #TODO: wrap everything in transaction + defaults_values = { + 'videopath': videopath, + 'title': media_title, + 'duration': duration, + 'description': media_description, + 'creator': creator, + 'external_id': external_id + } + + media, created = Media.objects.get_or_create(src=src, defaults= defaults_values) + + if not created and update_objects: + for k,v in defaults_values.items(): + setattr(media, k, v) + media.save() + + if created or update_objects: + media.is_public = public + assign_perm_to_obj(media, read_list, write_list, creator) + + content_uuid = generate_uuid() + content_iriurl = "%s/%s.iri" % (content_uuid, content_uuid); + content = Content.objects.create( + iri_id = content_uuid, + iriurl = content_iriurl, + media_obj=media, + title=title, + description=description, + duration=duration + ) + + content.tags.add(*tags) + + everyone = Group.objects.get(name=settings.PUBLIC_GROUP_NAME) + if public: + cached_assign('view_content', everyone, content) + cached_assign('view_media', everyone, media) + else: + remove_perm('ldt_utils.view_media', everyone, media) + remove_perm('ldt_utils.view_content', everyone, content) + assign_perm_to_obj(content, read_list, write_list, creator) + assign_perm_to_obj(media, read_list, write_list, creator) + + content.is_public = public + content.save() + + assign_perm_to_obj(content, read_list, write_list, creator) + update_stat_content(content) + + return content.iri_id diff -r 0c03dba0c515 -r dd3b4c9d5035 src/ldt/ldt/security/cache.py --- a/src/ldt/ldt/security/cache.py Thu Apr 09 00:06:43 2015 +0200 +++ b/src/ldt/ldt/security/cache.py Wed Apr 29 00:53:05 2015 +0200 @@ -3,7 +3,7 @@ from guardian.core import ObjectPermissionChecker from django.contrib.auth import get_user_model from django.db.models import Q -from guardian.shortcuts import assign +from guardian.shortcuts import assign_perm from django.utils.encoding import smart_str import re import hashlib @@ -23,7 +23,7 @@ def cached_assign(perm, user_or_group, object_inst): - assign(perm, user_or_group, object_inst) + assign_perm(perm, user_or_group, object_inst) if hasattr(user_or_group, 'username'): key = get_key(user_or_group.username) diff -r 0c03dba0c515 -r dd3b4c9d5035 src/ldt/ldt/security/command.py --- a/src/ldt/ldt/security/command.py Thu Apr 09 00:06:43 2015 +0200 +++ b/src/ldt/ldt/security/command.py Wed Apr 29 00:53:05 2015 +0200 @@ -1,7 +1,7 @@ from django.conf import settings from django.contrib.auth import get_user_model from django.contrib.auth.models import Group -from guardian.shortcuts import assign, remove_perm +from guardian.shortcuts import assign_perm, remove_perm from ldt.ldt_utils.models import Project, Content from ldt.management.utils import show_progress @@ -32,18 +32,18 @@ for i, proj in enumerate(list_model['Project'].objects.all()): if proj.state == 2: - assign('ldt_utils.view_project', everyone, proj) + assign_perm('ldt_utils.view_project', everyone, proj) else: remove_perm('ldt_utils.view_project', everyone, proj) - assign('ldt_utils.view_project', proj.owner, proj) - assign('ldt_utils.change_project', proj.owner, proj) + assign_perm('ldt_utils.view_project', proj.owner, proj) + assign_perm('ldt_utils.change_project', proj.owner, proj) if verbose: writer = show_progress(i, total, "Assign permissions to projects", 40, writer) for published_proj in list_model['Project'].objects.filter(state=2): - assign('ldt_utils.view_project', everyone, published_proj) + assign_perm('ldt_utils.view_project', everyone, published_proj) if verbose: writer = None @@ -56,11 +56,11 @@ for group in user.groups.exclude(name=settings.PUBLIC_GROUP_NAME): for proj in list_model['Project'].objects.filter(owner=user): - assign('ldt_utils.view_project', group, proj) + assign_perm('ldt_utils.view_project', group, proj) for content in list_model['Content'].objects.all(): if user.has_perm('view_content', content): - assign('ldt_utils.view_media', user, content.media_obj) + assign_perm('ldt_utils.view_media', user, content.media_obj) for admin in list_model['User'].objects.filter(is_superuser=True): diff -r 0c03dba0c515 -r dd3b4c9d5035 src/ldt/ldt/security/utils.py --- a/src/ldt/ldt/security/utils.py Thu Apr 09 00:06:43 2015 +0200 +++ b/src/ldt/ldt/security/utils.py Wed Apr 29 00:53:05 2015 +0200 @@ -101,7 +101,7 @@ remove_perms(read_list, old_users, obj, name) remove_perms(read_list, old_groups, obj, name) - cache.delete('userlist') + cache.delete('userlist') def get_userlist(user, filter=None): # @ReservedAssignment