# HG changeset patch # User Nicolas DURAND # Date 1418833266 -3600 # Node ID 8ca7be41e3ca9cfb499c9f2b6814a163d03c5c5d # Parent f2d54d2841f3fcb32680ae19d39961ca4a6248ab Pylint + pep8 + adapted code to Python 3 + added support for authentication when persistence is set to PersistenceToFile + cleaning up settings.py/config.py.tmpl and updated readme diff -r f2d54d2841f3 -r 8ca7be41e3ca .hgignore --- a/.hgignore Tue Dec 16 11:14:55 2014 +0100 +++ b/.hgignore Wed Dec 17 17:21:06 2014 +0100 @@ -4,3 +4,5 @@ *.pyc src/catedit/config.py src/catedit/log/ +run/log/ +run/files/ \ No newline at end of file diff -r f2d54d2841f3 -r 8ca7be41e3ca Readme.md --- a/Readme.md Tue Dec 16 11:14:55 2014 +0100 +++ b/Readme.md Wed Dec 17 17:21:06 2014 +0100 @@ -28,7 +28,11 @@ * HOST : The host on which the app will run, default is "0.0.0.0" for development purpose as "localhost" won't work with vagrant and windows. * LOGGING : Wether or not the app write log files +* LOGGING_LEVEL : if LOGGING is True, indicates what logging level will be used * DEBUG : Wether or not the app is on debug mode. If True, then each modification of a given file will restart the server allowing for easy development. +* SECRET_KEY : Secret key to secure WTForms +* PERSISTENCE_METHOD : What Persistence method will be used. Currently, either "PersistenceToFile" or "PersistenceToGithub" +* FILE_SAVE_DIRECTORY : If using PersistenceToFile, directory where the turtle files will be saved locally * REPOSITORY_NAME : The name of the repository your app will use to store categories * REPOSITORY_OWNER : The name of the owner of the repository (typically a "admin" user) * CATEGORIES_PATH : Where on the repository categories will be stored. Default to /categories @@ -44,7 +48,7 @@ ## Additional/Advanced informations ## -** Changing the property list : ** If you want to change the property list available to users editing/creating categories, you have to put the following entry in config.py +** Changing the property list : ** If you want to change the property list available to users editing/creating categories, you have to edit the following entry in config.py * PROPERTY_LIST : My list of properties ... diff -r f2d54d2841f3 -r 8ca7be41e3ca dev/modules/sysconfig/manifests/packages.pp --- a/dev/modules/sysconfig/manifests/packages.pp Tue Dec 16 11:14:55 2014 +0100 +++ b/dev/modules/sysconfig/manifests/packages.pp Wed Dec 17 17:21:06 2014 +0100 @@ -7,13 +7,14 @@ 'libjpeg8-dev', 'libxslt-dev', 'libxml2', - 'mercurial' + 'mercurial', + 'pylint' ] - + package { $catedit_pkgs: ensure => "installed" } #upgrade setuptools exec { '/usr/bin/easy_install --upgrade setuptools': require => Package[$catedit_pkgs]} - + } diff -r f2d54d2841f3 -r 8ca7be41e3ca dev/modules/sysconfig/manifests/virtualenv.pp --- a/dev/modules/sysconfig/manifests/virtualenv.pp Tue Dec 16 11:14:55 2014 +0100 +++ b/dev/modules/sysconfig/manifests/virtualenv.pp Wed Dec 17 17:21:06 2014 +0100 @@ -15,7 +15,7 @@ provider => 'shell', require => Exec['easy_install_pip']; 'create_virtualenv': - command => "virtualenv ${path_to_virtualenv}", + command => "virtualenv -p `which python3.4` ${path_to_virtualenv}", timeout => 2400, returns => [ 0, 100 ], provider => 'shell', diff -r f2d54d2841f3 -r 8ca7be41e3ca src/catedit/__init__.py --- a/src/catedit/__init__.py Tue Dec 16 11:14:55 2014 +0100 +++ b/src/catedit/__init__.py Wed Dec 17 17:21:06 2014 +0100 @@ -1,15 +1,19 @@ +""" +__init__.py: +module main file used to configure the Flask app +""" from flask import Flask, session from flask.ext.github import GitHub from flask.ext.cache import Cache -from settings import * -from config import * -from logging import * +from settings import AppSettings +from config import AppConfig +from logging import FileHandler, Formatter # set up app and database app = Flask(__name__) -app.config.from_object(appSettings) +app.config.from_object(AppSettings) cache = Cache(app, config={"CACHE_TYPE": "simple"}) -app.config.from_object(appConfig) +app.config.from_object(AppConfig) github = GitHub(app) diff -r f2d54d2841f3 -r 8ca7be41e3ca src/catedit/api.py --- a/src/catedit/api.py Tue Dec 16 11:14:55 2014 +0100 +++ b/src/catedit/api.py Wed Dec 17 17:21:06 2014 +0100 @@ -1,82 +1,101 @@ -from flask.ext.restful import Resource, fields, Api, reqparse +""" +api.py: +contains the api that links the views (views.py) to the model (models.py) and +its persistence method (persistence.py). As it only trades rdf graphs +serializations strings, it isn't bound to one specific view +""" + +from flask.ext.restful import Resource, Api, reqparse from flask import request from catedit import app, cache -from models import Category, CategoryManager -from rdflib import Graph, RDF -from utils import * -from settings import * -from config import * -import os +from catedit.models import Category, CategoryManager api = Api(app) -logger = app.logger + +LOGGER = app.logger -cat_parser = reqparse.RequestParser() -cat_parser.add_argument('label', type=str) -cat_parser.add_argument('description', type=str) -cat_parser.add_argument('commit_message', type=str) -cat_parser.add_argument('property_predicate', type=str, action="append") -cat_parser.add_argument('property_object', type=str, action="append") -# cat_parser.add_argument('delete_message', type=str) +CAT_PARSER = reqparse.RequestParser() +CAT_PARSER.add_argument('label', type=str) +CAT_PARSER.add_argument('description', type=str) +CAT_PARSER.add_argument('commit_message', type=str) +CAT_PARSER.add_argument('property_predicate', type=str, action="append") +CAT_PARSER.add_argument('property_object', type=str, action="append") +CAT_PARSER.add_argument('delete_message', type=str) class CategoryAPI(Resource): - # returns category cat_id or all if cat_id is None + """ + The API to create and edit categories, returns rdf graph serializations + when successful + """ @classmethod @cache.memoize(timeout=3600) - def get(self, cat_id=None): + def get(cls, cat_id=None): + """ + API to get the category of id cat_id, or if cat_id is None, + get the list of category + """ cat_manager_instance = CategoryManager() if cat_id is not None: - c = cat_manager_instance.load_cat(cat_id) - return c.cat_graph.serialize(format='turtle') + cat = cat_manager_instance.load_cat(cat_id) + return cat.cat_graph.serialize(format='turtle').decode("utf-8") else: response = [] - for c in cat_manager_instance.list_cat(): - response.append(c.cat_graph.serialize(format='turtle')) + for cat in cat_manager_instance.list_cat(): + response.append(cat.cat_graph.serialize(format='turtle') + .decode("utf-8")) return response # update category cat_id @classmethod - def put(self, cat_id): - args = cat_parser.parse_args() + def put(cls, cat_id): + """ + API to edit the category of id cat_id + """ + args = CAT_PARSER.parse_args() cat_manager_instance = CategoryManager() new_property_list = [] - logger.debug(args["property_predicate"]) - logger.debug(args["property_object"]) - for property_predicate, property_object in zip( - args["property_predicate"], - args["property_object"]): - if property_object: - property_object_to_append = property_object - # if URIRef category, we must prefix id with namespace - if (app.config["PROPERTY_LIST"] - [property_predicate] - ["object_type"]) == "uriref-category": - property_object_to_append = \ - app.config["ONTOLOGY_NAMESPACE"] + property_object - logger.debug(property_object_to_append) - new_property_list.append((property_predicate, - property_object_to_append)) - logger.debug(new_property_list) - c = cat_manager_instance.load_cat(cat_id) - c.edit_category(new_description=args["description"], - new_label=args["label"], - new_other_properties=new_property_list) - cat_manager_instance.save_cat(c, message=args["commit_message"]) - logger.debug("put id: "+c.cat_id) + LOGGER.debug(args["property_predicate"]) + LOGGER.debug(args["property_object"]) + if args["property_predicate"] is not None and \ + args["property_object"] is not None: + for property_predicate, property_object in zip( + args["property_predicate"], + args["property_object"]): + if property_object: + property_object_to_append = property_object + # if URIRef category, we must prefix id with namespace + if (app.config["PROPERTY_LIST"] + [property_predicate] + ["object_type"]) == "uriref-category": + property_object_to_append = \ + app.config["CATEGORY_NAMESPACE"] + property_object + LOGGER.debug(property_object_to_append) + new_property_list.append((property_predicate, + property_object_to_append)) + LOGGER.debug(new_property_list) + cat = cat_manager_instance.load_cat(cat_id) + cat.edit_category(new_description=args["description"], + new_label=args["label"], + new_other_properties=new_property_list) + cat_manager_instance.save_cat(cat, message=args["commit_message"]) + LOGGER.debug("put id: "+cat.cat_id) cache.clear() - return c.cat_graph.serialize(format='turtle'), 200 + return cat.cat_graph.serialize(format='turtle').decode("utf-8"), 200 # Maybe not send the whole cat back, see if it's worth it @classmethod - def post(self): - args = cat_parser.parse_args() + def post(cls): + """ + API to create a new category + """ + args = CAT_PARSER.parse_args() property_list = [] - logger.debug(args["property_predicate"]) - logger.debug(args["property_object"]) + LOGGER.debug(args["property_predicate"]) + LOGGER.debug(args["property_object"]) for property_predicate, property_object in zip( request.form.getlist('property_predicate'), request.form.getlist('property_object')): @@ -85,30 +104,34 @@ [property_predicate] ["object_type"]) == "uriref-category": property_list.append((property_predicate, - app.config["ONTOLOGY_NAMESPACE"] + app.config["CATEGORY_NAMESPACE"] + property_object)) else: property_list.append((property_predicate, property_object)) - logger.debug(property_list) - c = Category(label=args["label"], - description=args["description"], - other_properties=property_list) + LOGGER.debug(property_list) + cat = Category(label=args["label"], + description=args["description"], + other_properties=property_list) cat_manager_instance = CategoryManager() - cat_manager_instance.save_cat(c, message=args["commit_message"]) - logger.debug("post id: "+c.cat_id) + cat_manager_instance.save_cat(cat, message=args["commit_message"]) + LOGGER.debug("post id: "+cat.cat_id) cache.clear() - return c.cat_graph.serialize(format='turtle'), 201 + return cat.cat_graph.serialize(format='turtle').decode("utf-8"), 201 @classmethod - def delete(self, cat_id): + def delete(cls, cat_id): + """ + API to delete the category of id cat_id + """ + args = CAT_PARSER.parse_args() cat_manager_instance = CategoryManager() - if not request.form["delete_message"]: + if args["delete_message"] is None: message = "Deleting category "+cat_id else: - message = request.form["delete_message"] + message = args["delete_message"] cat_manager_instance.delete_cat(cat_id, message=message) - logger.debug("delete id: "+cat_id) + LOGGER.debug("delete id: "+cat_id) cache.clear() return 204 diff -r f2d54d2841f3 -r 8ca7be41e3ca src/catedit/config.py.tmpl --- a/src/catedit/config.py.tmpl Tue Dec 16 11:14:55 2014 +0100 +++ b/src/catedit/config.py.tmpl Wed Dec 17 17:21:06 2014 +0100 @@ -1,17 +1,83 @@ -# Overwriting settings -class appConfig(object): +""" +config.py: +Contains all settings that can change depending on the deployment environment +""" + +from rdflib import URIRef, RDF, RDFS, Literal +from rdflib.namespace import SKOS + +class AppConfig(object): + + # Debug and running settings - DEBUG = False + HOST = "0.0.0.0" + DEBUG = True + LOGGING_LEVEL = "DEBUG" + + # WTForms settings + + SECRET_KEY = 'totally-secret-key' + + # Saving file for local persistence + FILE_SAVE_DIRECTORY = "../../run/files/" + + # Logging config + LOG_FILE_PATH = "../../run/log/log.txt" LOGGING = False - SERVER_NAME = "0.0.0.0" - # Github repository settings + # Github repository config - REPOSITORY_NAME = "habitabilite-prototype" + REPOSITORY_NAME = "catedit-dev-testing" REPOSITORY_OWNER = "catedit-system" CATEGORIES_PATH = "categories/" # Github parameters - GITHUB_CLIENT_ID = "3e31f3b000a4914f75ef" - GITHUB_CLIENT_SECRET = "bc17eb0ec11385628c2e75aacb5ff8ef5f29e490" + GITHUB_CLIENT_ID = "github-id-placeholder" + GITHUB_CLIENT_SECRET = "github-secret-placeholder" + + # Property List + + PROPERTY_LIST = { + "subClassOf": { + "descriptive_label_fr": "Sous-classe de", + "descriptive_label_en": "Subclass of", + "object_type": "uriref-category", + "rdflib_class": RDFS.subClassOf, + "object_rdflib_class": URIRef, + }, + "value": { + "descriptive_label_fr": "Valeur", + "descriptive_label_en": "Value", + "object_type": "literal", + "rdflib_class": RDF.value, + "object_rdflib_class": Literal, + }, + "type": { + "descriptive_label_fr": "Type", + "descriptive_label_en": "Type", + "object_type": "uriref-category", + "rdflib_class": RDF.type, + "object_rdflib_class": URIRef, + }, + "resource": { + "descriptive_label_fr": "Ressource", + "descriptive_label_en": "Resource", + "object_type": "uriref-link", + "rdflib_class": RDFS.Resource, + "object_rdflib_class": URIRef, + }, + "related": { + "descriptive_label_fr": "En relation avec", + "descriptive_label_en": "Related to", + "object_type": "uriref-category", + "rdflib_class": SKOS.related, + "object_rdflib_class": URIRef, + } + } + + # Category persistence parameters + # "PersistenceToFile" : will save categories to files on system + # "PersistenceToGithub" : will save categories to files on Github + + PERSISTENCE_METHOD = "PersistenceToGithub" diff -r f2d54d2841f3 -r 8ca7be41e3ca src/catedit/main.py --- a/src/catedit/main.py Tue Dec 16 11:14:55 2014 +0100 +++ b/src/catedit/main.py Wed Dec 17 17:21:06 2014 +0100 @@ -1,8 +1,12 @@ -from catedit import app +""" +main.py: +script that is used to boot up the application +""" -from api import api -from models import * -from views import * +from catedit import app +from catedit.api import api +from catedit.views import cat_editor, cat_recap, github_login, \ + github_callback, logout if __name__ == '__main__': app.run(host=app.config["HOST"]) diff -r f2d54d2841f3 -r 8ca7be41e3ca src/catedit/models.py --- a/src/catedit/models.py Tue Dec 16 11:14:55 2014 +0100 +++ b/src/catedit/models.py Wed Dec 17 17:21:06 2014 +0100 @@ -1,34 +1,44 @@ -from flask import Flask, request -from flask.ext.restful import fields -from rdflib import Graph, RDF, RDFS, BNode, Literal, URIRef -from uuid import uuid4 +""" +models.py: +contains the "principal" objects that will be manipulated by the application: +* categories +* helper classes to manage category life cycle +""" + +from rdflib import Graph, RDF, RDFS, Literal, URIRef +# from uuid import uuid4 from io import StringIO from slugify import slugify -import persistence from catedit import app - -logger = app.logger +import catedit.persistence -""" -Namespace: ld.iri-research.org/ontology/categorisation/# -Category URI: ld.iri-research.org/ontology/categorisation/#cat_id -Category Class: -label is the rdf label of the category -description is the description of the category -other_properties is a dictionnary containing every other supported property -as defined in the PROPERTY_LIST dict in settings.py -""" +LOGGER = app.logger class Category(object): + """ + Category Class: + + Init: + * label is the rdf label of the category, unique and non-empty + * description is the description of the category, unique and non-empty + * other_properties is a dictionnary containing every other supported + property as defined in the PROPERTY_LIST dict in settings.py + * graph is used if we want to create the Category from a turtle + rdf graph + + Additional info: + * cat_id is a generated, hidden id that uniquely identify + a given category + """ def __init__(self, label=None, description=None, other_properties=None, graph=None): - if not(graph): + if not graph: # cat_id = uuid4().hex - Alternate method of generating ids - cat_id = "category_id_"+slugify(bytes(label)) + cat_id = "category_id_"+slugify(label) self.cat_graph = Graph() - self.this_category = URIRef(app.config["ONTOLOGY_NAMESPACE"] + + self.this_category = URIRef(app.config["CATEGORY_NAMESPACE"] + cat_id) self.cat_graph.add((self.this_category, RDF.ID, Literal(cat_id))) @@ -53,12 +63,15 @@ else: self.cat_graph = graph - self.this_category = self.cat_graph \ - .subjects(predicate=RDF.ID) \ - .next() # Warning: not foolproof + # Warning: not foolproof + self.this_category = next(self.cat_graph + .subjects(predicate=RDF.ID)) @property def label(self): + """ + Returns category label + """ return_value = self.cat_graph.value(self.this_category, RDFS.label) if return_value is None: return None @@ -67,6 +80,9 @@ @property def description(self): + """ + Returns category description + """ return_value = \ self.cat_graph.value(self.this_category, RDF.Description) if return_value is None: @@ -76,10 +92,16 @@ @property def cat_id(self): + """ + Returns category id + """ return self.cat_graph.value(self.this_category, RDF.ID).toPython() @property def properties(self): + """ + Returns category property list + """ property_list = [] for key in app.config["PROPERTY_LIST"]: for obj in self.cat_graph \ @@ -94,10 +116,19 @@ new_description=False, new_other_properties=False): """ - Checks if there is a new label and if so apply changes + Edits category + * new_label is the new label of the category + * new_description is the new description of the category + * new_other_property is the new property list of the category + + Every argument is optional and defaults to False for consistency, + as giving a property list that is None means we want + to delete all properties """ - if new_label is not (False or None) and \ - new_label != self.label: + + if (new_label is not False) and \ + (new_label is not None) and \ + (new_label != self.label): self.cat_graph.remove((self.this_category, RDFS.label, self.cat_graph.label(self.this_category))) @@ -105,11 +136,9 @@ RDFS.label, Literal(new_label))) - """ - Checks if there is a new description and if so apply changes - """ - if new_description is not (False or None) and \ - new_description != self.description: + if (new_description is not False) and \ + (new_description is not None) and \ + (new_description != self.description): self.cat_graph.remove((self.this_category, RDF.Description, self.cat_graph.value(self.this_category, @@ -118,27 +147,23 @@ RDF.Description, Literal(new_description))) - """ - Checks if there is a new property list and if so apply changes - (can be [] so it deletes everything) - """ if new_other_properties is not False or \ (new_other_properties is None and self.properties is not None): - logger.debug("before suppressing properties: ") - logger.debug(self.properties) - logger.debug("will replace with: ") - logger.debug(new_other_properties) + LOGGER.debug("before suppressing properties: ") + LOGGER.debug(self.properties) + LOGGER.debug("will replace with: ") + LOGGER.debug(new_other_properties) for key in app.config["PROPERTY_LIST"]: self.cat_graph.remove((self.this_category, app.config["PROPERTY_LIST"] [key] ["rdflib_class"], None)) - logger.debug("now properties are: ") - logger.debug(self.properties) - logger.debug("making new properties: ") - logger.debug(new_other_properties) + LOGGER.debug("now properties are: ") + LOGGER.debug(self.properties) + LOGGER.debug("making new properties: ") + LOGGER.debug(new_other_properties) for (predicate, obj) in new_other_properties: self.cat_graph.add((self.this_category, app.config["PROPERTY_LIST"] @@ -149,40 +174,50 @@ ["object_rdflib_class"](obj))) -""" -CategoryManager class - -This class deals with creation and loading of Category objects - -Persistence method is set in config files -""" - - class CategoryManager(object): + """ + CategoryManager class + This class deals with creation and loading of Category objects + Persistence method is set in config.py and used to save + and load categories + """ def load_cat(self, cat_id): - p = getattr(persistence, app.config["PERSISTENCE_METHOD"])() - cat_serial = p.load(name=cat_id) + """ + Loads a category from its id + """ + persistence = getattr(catedit.persistence, + app.config["PERSISTENCE_METHOD"])() + cat_serial = persistence.load(name=cat_id) loaded_cat_graph = Graph() loaded_cat_graph.parse(source=StringIO(cat_serial), format='turtle') cat = Category(graph=loaded_cat_graph) return cat def save_cat(self, cat, message=None): - p = getattr(persistence, app.config["PERSISTENCE_METHOD"])() - p.save(content=cat.cat_graph.serialize(format='turtle'), - name=cat.cat_id, message=message) + """ + Saves a category, message serves as commit message + if github is used as a persistence method + """ + persistence = getattr(catedit.persistence, + app.config["PERSISTENCE_METHOD"])() + persistence.save(content=cat.cat_graph.serialize(format='turtle'), + name=cat.cat_id, message=message) def delete_cat(self, deleted_cat_id, message=None): + """ + Deletes a category from its id, message serves + as commit message if github is used as a persistence method + """ cat_list = self.list_cat() for cat in cat_list: - if cat.cat_id != app.config["ONTOLOGY_NAMESPACE"]+deleted_cat_id: + if cat.cat_id != app.config["CATEGORY_NAMESPACE"]+deleted_cat_id: new_property_list_for_cat = [] for (predicate, obj) in cat.properties: if not ( (app.config["PROPERTY_LIST"] [predicate] ["object_type"] == "uriref-category") and - (obj == (app.config["ONTOLOGY_NAMESPACE"] + + (obj == (app.config["CATEGORY_NAMESPACE"] + deleted_cat_id)) ): new_property_list_for_cat.append((predicate, obj)) @@ -193,13 +228,18 @@ cat, message=message+", cleaning up other properties" ) - p = getattr(persistence, app.config["PERSISTENCE_METHOD"])() - p.delete(name=deleted_cat_id, message=message) + persistence = getattr(catedit.persistence, + app.config["PERSISTENCE_METHOD"])() + persistence.delete(name=deleted_cat_id, message=message) def list_cat(self): - p = getattr(persistence, app.config["PERSISTENCE_METHOD"])() - cat_serial_list = p.list() - logger.debug(cat_serial_list) + """ + Lists all categories available + """ + persistence = getattr(catedit.persistence, + app.config["PERSISTENCE_METHOD"])() + cat_serial_list = persistence.list() + # LOGGER.debug(cat_serial_list) cat_list = [] for cat_serial in cat_serial_list: loaded_cat_graph = Graph() @@ -207,6 +247,6 @@ source=StringIO(cat_serial), format='turtle' ) - c = Category(graph=loaded_cat_graph) - cat_list.append(c) + cat = Category(graph=loaded_cat_graph) + cat_list.append(cat) return cat_list diff -r f2d54d2841f3 -r 8ca7be41e3ca src/catedit/persistence.py --- a/src/catedit/persistence.py Tue Dec 16 11:14:55 2014 +0100 +++ b/src/catedit/persistence.py Wed Dec 17 17:21:06 2014 +0100 @@ -1,76 +1,132 @@ +""" +persistence.py: +contains what we need to make a (category) objects (see models.py) persist +beyond application scope and load them from outside sources + +For now we can save, load and list +* to/from a file (using a turtle rdf graph serialization for categories) +* to/from a github repository (using a turtle rdf graph serialization +for categories) +""" from abc import ABCMeta, abstractmethod -from catedit import app +from catedit import app, github from base64 import b64encode, b64decode +from flask.ext.github import GitHubError import os import json +LOGGER = app.logger + class Persistence: + """ + Meta-class for all persistence classes + """ __metaclass__ = ABCMeta @abstractmethod def save(self, **kwargs): + """ + Abstract - Saves object + """ return @abstractmethod def delete(self, **kwargs): + """ + Abstract - Deletes object + """ return @abstractmethod def load(self, **kwargs): + """ + Abstract - Loads object + """ return @abstractmethod def list(self, **kwargs): + """ + Abstract - Lists objects + """ return -""" -kwargs for saving to a file should be - -pathDir = directory to save the file in -fileName = name of the file to write in -content = desired content of the file - -There is still adjustments to do on this -""" - class PersistenceToFile(Persistence): + """ + Persistence Class for saving to a local file (see config.py + for tweaking) + + Expected kwargs for saving to/loading from a file are: + * name : name of the file to write in/read from + * content : desired content of the file when writing + """ def save(self, **kwargs): + """ + Saves to a file + """ path_to_save = app.config["FILE_SAVE_DIRECTORY"]+kwargs["name"] - file = open(path_to_save, 'wb') - file.write(kwargs["content"]) - file.close() + file_to_save = open(path_to_save, 'wb') + file_to_save.write(kwargs["content"]) + file_to_save.close() def load(self, **kwargs): + """ + Loads from a file + """ path_to_load = app.config["FILE_SAVE_DIRECTORY"]+kwargs["name"] - file = open(path_to_load, 'rb') - file_content = file.read() - file.close() + file_to_load = open(path_to_load, 'rb') + file_content = file_to_load.read() + file_to_load.close() return file_content def delete(self, **kwargs): + """ + Deletes a file + """ path_to_delete = app.config["FILE_SAVE_DIRECTORY"]+kwargs["name"] os.remove(path_to_delete) # IDEA: return { file_name: file_content } type dict def list(self, **kwargs): + """ + Lists all files in file directory (as set in config.py) + """ file_content_list = [] for file_name in os.listdir(app.config["FILE_SAVE_DIRECTORY"]): if not file_name or file_name[0] == ".": continue - file = open(app.config["FILE_SAVE_DIRECTORY"]+file_name) - file_content = file.read() - file.close() + path_to_load = open(app.config["FILE_SAVE_DIRECTORY"]+file_name) + file_content = path_to_load.read() + path_to_load.close() file_content_list.append(file_content) - # logger.debug(file_content_list) + # LOGGER.debug(file_content_list) return file_content_list class PersistenceToGithub(Persistence): + """ + Persistence Class for saving to/loading from a Github repository (see + config.py for tweaks) + + Expected kwargs for saving to/loading from a Github repository are: + * name : name of the file to write in/read from + * content : desired content of the file when writing + * message : when saving or deleting, commit message to be saved + to Github + """ def save(self, **kwargs): - # logger.debug(kwargs["content"]) - request_data = {"content": b64encode(kwargs["content"]), + """ + Saves to a Github repository + + IMPORTANT: To save to a file Github, we use a PUT request, and + if the file already exists we must put its sha in the request data. + The first try-except block actually expects an error if creating a + new file because we need to check if the file already exists + """ + # LOGGER.debug(kwargs["content"]) + request_data = {"content": str(b64encode(kwargs["content"]), "ascii"), "message": kwargs["message"]} try: filedict = github.get("repos/" @@ -81,8 +137,14 @@ + kwargs["name"]) request_data["sha"] = filedict["sha"] except GitHubError: - pass - # logger.debug(json.dumps(request_data)) + LOGGER.debug("Github sent an error, either: \ + 1- You're trying to create a new file named : " + + kwargs["name"] + " OR \ + 2- You're trying to edit a file named : " + + kwargs["name"] + ", \ + in which case something went wrong \ + trying to get its sha") + # LOGGER.debug(json.dumps(request_data)) try: github.request('PUT', "repos/" @@ -93,9 +155,15 @@ + kwargs["name"], data=json.dumps(request_data)) except GitHubError: - pass + LOGGER.debug("Github Error trying to update file: "+kwargs["name"]) + LOGGER.debug("Github sent an error, if 404, either you may not \ + have access to the repository or it doesn't exist ") + LOGGER.debug(GitHubError.response.text) def load(self, **kwargs): + """ + Loads from a Github repository + """ try: filedict = github.get("repos/" + app.config["REPOSITORY_OWNER"]+"/" @@ -103,12 +171,18 @@ + "/contents/" + app.config["CATEGORIES_PATH"] + kwargs["name"]) - file_content = b64decode(filedict["content"]) + file_content = str(b64decode(filedict["content"]), "utf-8") except GitHubError: - pass + LOGGER.debug("Github Error trying to get file: "+kwargs["name"]) + LOGGER.debug("Github sent an error, if 404, either you may not \ + have access to the repository or it doesn't exist ") + LOGGER.debug(GitHubError.response.text) return file_content def delete(self, **kwargs): + """ + Deletes from a Github repository + """ request_data = {"message": kwargs["message"]} try: filedict = github.get("repos/" @@ -119,7 +193,9 @@ + kwargs["name"]) request_data["sha"] = filedict["sha"] except GitHubError: - pass + LOGGER.debug("Github Error trying to get sha for \ + file: "+kwargs["name"]) + LOGGER.debug(GitHubError.response.text) try: github.request('DELETE', @@ -129,9 +205,13 @@ + kwargs["name"], data=json.dumps(request_data)) except GitHubError: - pass + LOGGER.debug("Github Error trying to delete file: "+kwargs["name"]) + LOGGER.debug(GitHubError.response.text) def list(self, **kwargs): + """ + Lists all files in the github repository (as set in config.py) + """ filenames_list = [] try: files_in_repo = github.get("repos/" @@ -139,10 +219,17 @@ + app.config["REPOSITORY_NAME"] + "/contents/" + app.config["CATEGORIES_PATH"]) - filenames_list = [file["name"] for file in files_in_repo] - # logger.debug(filenames_list) + filenames_list = [github_file["name"] + for github_file in files_in_repo] + # LOGGER.debug(filenames_list) except GitHubError: - pass + LOGGER.debug("Github Error trying to get the file list in the \ + category repository") + LOGGER.debug("NOTE: Github sent an error, if 404 either you \ + may not have access to the repository or it doesn't \ + exist") + LOGGER.debug(GitHubError.response.text) + file_content_list = [] for filename in filenames_list: try: @@ -152,8 +239,10 @@ + "/contents/" + app.config["CATEGORIES_PATH"] + filename) - file_content_list.append(b64decode(filedict["content"])) + file_content_list.append(str(b64decode(filedict["content"]), + "utf-8")) except GitHubError: - pass - # logger.debug(file_content_list) + LOGGER.debug("Github Error trying to get file: "+filename) + LOGGER.debug(GitHubError.response.text) + LOGGER.debug(file_content_list) return file_content_list diff -r f2d54d2841f3 -r 8ca7be41e3ca src/catedit/settings.py --- a/src/catedit/settings.py Tue Dec 16 11:14:55 2014 +0100 +++ b/src/catedit/settings.py Wed Dec 17 17:21:06 2014 +0100 @@ -1,83 +1,15 @@ -from rdflib import URIRef, RDF, RDFS, Literal -from rdflib.namespace import SKOS - - -class appSettings(object): +""" +settings.py: +Contains all settings that are needed but are not to be changed +""" - # Debug and running settings - - HOST = "0.0.0.0" - DEBUG = True - LOGGING_LEVEL = "DEBUG" +class AppSettings(object): # WTForms settings WTF_CSRF_ENABLED = True - SECRET_KEY = 'totally-secret-key' # RDF Namespace and prefixes ONTOLOGY_NAMESPACE = "http://ld.iri-research.org/ontology/categorisation#" - CATEGORY_NAMESPACE = "http://ld.iri-research.org/categorisation/category/" - - # Saving file for local persistence - FILE_SAVE_DIRECTORY = "../../run/files/" - - # Logging config - LOG_FILE_PATH = "../../run/log/log.txt" - LOGGING = False - - # Github repository config - - REPOSITORY_NAME = "catedit-dev-testing" - REPOSITORY_OWNER = "catedit-system" - CATEGORIES_PATH = "categories/" - - # Github parameters - - GITHUB_CLIENT_ID = "3e31f3b000a4914f75ef" - GITHUB_CLIENT_SECRET = "bc17eb0ec11385628c2e75aacb5ff8ef5f29e490" - - # Property List - - PROPERTY_LIST = { - "subClassOf": { - "descriptive_label_fr": "Sous-classe de", - "descriptive_label_en": "Subclass of", - "object_type": "uriref-category", - "rdflib_class": RDFS.subClassOf, - "object_rdflib_class": URIRef, - }, - "value": { - "descriptive_label_fr": "Valeur", - "descriptive_label_en": "Value", - "object_type": "literal", - "rdflib_class": RDF.value, - "object_rdflib_class": Literal, - }, - "type": { - "descriptive_label_fr": "Type", - "descriptive_label_en": "Type", - "object_type": "uriref-category", - "rdflib_class": RDF.type, - "object_rdflib_class": URIRef, - }, - "resource": { - "descriptive_label_fr": "Ressource", - "descriptive_label_en": "Resource", - "object_type": "uriref-link", - "rdflib_class": RDFS.Resource, - "object_rdflib_class": URIRef, - }, - "related": { - "descriptive_label_fr": "En relation avec", - "descriptive_label_en": "Related to", - "object_type": "uriref-category", - "rdflib_class": SKOS.related, - "object_rdflib_class": URIRef, - } - } - - # Category persistence parameters - - PERSISTENCE_METHOD = "PersistenceToFile" + CATEGORY_NAMESPACE = "http://ld.iri-research.org/ontology/categorisation/category#" diff -r f2d54d2841f3 -r 8ca7be41e3ca src/catedit/templates/cateditor.html --- a/src/catedit/templates/cateditor.html Tue Dec 16 11:14:55 2014 +0100 +++ b/src/catedit/templates/cateditor.html Wed Dec 17 17:21:06 2014 +0100 @@ -31,7 +31,7 @@
  • Editeur de catégorie: {% if cat_id: %} Edition {% else %} Création {% endif %}
    • - {% if not session.get("user_code", None)%} Non authentifié - S'authentifier + {% if not session.get("user_logged", None)%} Non authentifié - S'authentifier {% else %} Authentifié: {{ session["user_login"] }} - Quitter{% endif %}
    @@ -106,7 +106,7 @@ {% if config["PROPERTY_LIST"][predicate]["object_type"]=="uriref-category" %} {% for cat in cat_list %} - {% if object == config["ONTOLOGY_NAMESPACE"]+cat.cat_id %} + {% if object == config["CATEGORY_NAMESPACE"]+cat.cat_id %} diff -r f2d54d2841f3 -r 8ca7be41e3ca src/catedit/templates/catrecap.html --- a/src/catedit/templates/catrecap.html Tue Dec 16 11:14:55 2014 +0100 +++ b/src/catedit/templates/catrecap.html Wed Dec 17 17:21:06 2014 +0100 @@ -48,7 +48,7 @@
  • Editeur de catégorie: Création
    • - {% if not session.get("user_code", None)%}Non authentifié - S'authentifier + {% if not session.get("user_logged", None)%}Non authentifié - S'authentifier {% else %} Authentifié: {{ session["user_login"] }} - Quitter{% endif %}
    @@ -97,7 +97,7 @@
    {% if config["PROPERTY_LIST"][predicate]["object_type"]=="uriref-category" %} {% for cat in cat_list %} - {% if object == config["ONTOLOGY_NAMESPACE"]+cat.cat_id %} + {% if object == config["CATEGORY_NAMESPACE"]+cat.cat_id %} {{ cat.cat_label }} {% endif %} {% endfor %} diff -r f2d54d2841f3 -r 8ca7be41e3ca src/catedit/views.py --- a/src/catedit/views.py Tue Dec 16 11:14:55 2014 +0100 +++ b/src/catedit/views.py Wed Dec 17 17:21:06 2014 +0100 @@ -1,17 +1,27 @@ +""" +views.py: +The views functions that handle the front-end of the application +""" + from catedit import app, github -from models import Category, CategoryManager +from catedit.models import Category from flask import render_template, request, redirect, url_for, session from flask.ext.github import GitHubError from flask_wtf import Form -from api import CategoryAPI +from catedit.api import CategoryAPI from wtforms import StringField, TextAreaField from wtforms.validators import DataRequired from rdflib import Graph from io import StringIO -logger = app.logger +LOGGER = app.logger + class NewCategoryMinimalForm(Form): + """ + Custom form class for creating a category with the absolute minimal + attributes (label and description) + """ label = StringField( "Nom de la categorie (obligatoire)", validators=[DataRequired()] @@ -26,31 +36,37 @@ ) -@app.route('/', methods=['GET']) -@app.route('/catrecap', methods=['GET']) -@app.route('/catrecap/delete/', methods=['POST']) -def cat_recap(delete_cat_id=None): +@app.route('/catrecap/delete-', methods=['POST']) +@app.route('/', defaults={'delete_cat_id': None}, methods=['GET']) +@app.route('/catrecap', defaults={'delete_cat_id': None}, methods=['GET']) +def cat_recap(delete_cat_id): + """ + View that has a list of all categories available. Template is + catrecap.html, located in src/templates/ + + Note: it also handles category deletion from the same page. + """ cat_api_instance = CategoryAPI() - # list categories if delete_cat_id is None: + LOGGER.debug("Category to delete is None") serialized_cat_list = cat_api_instance.get() - logger.debug(serialized_cat_list) + # LOGGER.debug(serialized_cat_list) cat_list = [] - logger.debug(cat_list) for serialized_cat in serialized_cat_list: cat_rdf_graph = Graph() cat_rdf_graph.parse(source=StringIO(serialized_cat), format='turtle') - c = Category(graph=cat_rdf_graph) + cat = Category(graph=cat_rdf_graph) - cat_list.append({"cat_label": c.label, - "cat_description": c.description, - "cat_id": c.cat_id, - "cat_properties": c.properties}) - logger.debug(c.properties) + cat_list.append({"cat_label": cat.label, + "cat_description": cat.description, + "cat_id": cat.cat_id, + "cat_properties": cat.properties}) + # LOGGER.debug(c.properties) return render_template('catrecap.html', cat_list=cat_list) else: + LOGGER.debug("Category "+delete_cat_id+" will be deleted.") cat_api_instance.delete(delete_cat_id) return redirect(url_for('cat_recap')) @@ -58,6 +74,10 @@ @app.route('/cateditor', methods=['GET', 'POST']) @app.route('/cateditor/', methods=['GET', 'POST']) def cat_editor(cat_id=None): + """ + View that handles creation and edition of categories. Template is + cateditor.html, located in src/templates + """ cat_api_instance = CategoryAPI() serialized_cat_list = cat_api_instance.get() @@ -66,40 +86,42 @@ cat_rdf_graph = Graph() cat_rdf_graph.parse(source=StringIO(serialized_cat), format='turtle') - c = Category(graph=cat_rdf_graph) + cat = Category(graph=cat_rdf_graph) - cat_list.append({"cat_label": c.label, - "cat_description": c.description, - "cat_id": c.cat_id, - "cat_properties": c.properties}) + cat_list.append({"cat_label": cat.label, + "cat_description": cat.description, + "cat_id": cat.cat_id, + "cat_properties": cat.properties}) if cat_id is not None: - catSerial = cat_api_instance.get(cat_id) + specific_serialized_cat = cat_api_instance.get(cat_id) cat_rdf_graph = Graph() - cat_rdf_graph.parse(source=StringIO(catSerial), + cat_rdf_graph.parse(source=StringIO(specific_serialized_cat), format='turtle') - c = Category(graph=cat_rdf_graph) + cat = Category(graph=cat_rdf_graph) setattr(NewCategoryMinimalForm, 'label', StringField("Nom de la categorie", validators=[DataRequired()], - default=c.label)) + default=cat.label)) setattr(NewCategoryMinimalForm, 'description', TextAreaField("Description de la categorie", validators=[DataRequired()], - default=c.description)) - logger.debug("CatForm fields preset to "+c.label+" and "+c.description) + default=cat.description)) + LOGGER.debug("CatForm fields preset to " + + cat.label + " and " + + cat.description) cat_form = NewCategoryMinimalForm(request.form) # GET + cat_id = Edit cat form if request.method == 'GET': return render_template('cateditor.html', - cat_id=c.cat_id, - cat_properties=c.properties, + cat_id=cat.cat_id, + cat_properties=cat.properties, form=cat_form, cat_list=cat_list) @@ -110,7 +132,7 @@ else: return render_template('cateditor.html', cat_id=cat_id, - cat_properties=c.properties, + cat_properties=cat.properties, form=cat_form, cat_list=cat_list) @@ -144,43 +166,69 @@ @app.route('/catedit-github-login') def github_login(): - return github.authorize(scope="repo") + """ + Function that manages authentication (Github), login + + Note: If Persistence is set to PersistenceToFile (categories stored + in local files, used for debugging), creates a mock user named + "FileEditUser" + """ + if app.config["PERSISTENCE_METHOD"] == "PersistenceToGithub": + return github.authorize(scope="repo") + elif app.config["PERSISTENCE_METHOD"] == "PersistenceToFile": + session["user_logged"] = True + session["user_can_edit"] = True + session["user_login"] = "FileEditUser" + return redirect(url_for('cat_recap')) @app.route('/catedit-github-callback') @github.authorized_handler def github_callback(oauth_code): + """ + Function that handles callback from Github after succesful login + """ session.permanent = False session["user_code"] = oauth_code session["user_logged"] = True session["user_login"] = github.get("user")["login"] try: - repoList = [] - repoList = github.get("user/repos") - for repo in repoList: - logger.debug(repo["name"]) + repo_list = [] + repo_list = github.get("user/repos") + for repo in repo_list: + LOGGER.debug(repo["name"]) session["user_can_edit"] = True if not any(repo["name"] == app.config["REPOSITORY_NAME"] - for repo in repoList): + for repo in repo_list): session["user_can_edit"] = False - logger.debug(session["user_can_edit"]) + LOGGER.debug(session["user_can_edit"]) except GitHubError: - logger.debug("error getting repos!") - pass + LOGGER.debug("error getting repos!") - logger.debug(session["user_login"]) + LOGGER.debug(session["user_login"]) return redirect(url_for('cat_recap')) @github.access_token_getter def token_getter(): + """ + Utility function for github-flask module to get user token when + making authenticated requests + """ if session.get("user_logged", None): - logger.debug("I made an authentified request") + # LOGGER.debug("I made an authentified request") return session["user_code"] @app.route('/catedit-logout') def logout(): + """ + Function that manages authentication (Github), logout + + Note: if you want to switch github users, you will have to logout of + Github, else when logging back in, github will send the app the + same oauth code + """ session["user_code"] = None session["user_logged"] = None session["user_login"] = None diff -r f2d54d2841f3 -r 8ca7be41e3ca virtualenv/requirements.txt --- a/virtualenv/requirements.txt Tue Dec 16 11:14:55 2014 +0100 +++ b/virtualenv/requirements.txt Wed Dec 17 17:21:06 2014 +0100 @@ -1,8 +1,20 @@ -rdflib -flask -flask-restful -flask-wtf -Github-Flask -flask-cache -python-slugify -pep8 +Flask==0.10.1 +Flask-Cache==0.13.1 +Flask-RESTful==0.3.1 +Flask-WTF==0.10.3 +GitHub-Flask==2.0.0 +Jinja2==2.7.3 +MarkupSafe==0.23 +Unidecode==0.04.16 +WTForms==2.0.1 +Werkzeug==0.9.6 +aniso8601==0.90 +isodate==0.5.1 +itsdangerous==0.24 +pep8==1.5.7 +pyparsing==2.0.3 +python-slugify==0.1.0 +pytz==2014.10 +rdflib==4.1.2 +requests==2.5.0 +six==1.8.0