# HG changeset patch # User Nicolas DURAND # Date 1421060430 -3600 # Node ID d1bc73ce855a35ce54bc4250043aabb2ce8c58e1 # Parent b7ec9ee75bbb35d360162d02028f6aae55f2cad4 Finished WTForms support for properties + Refactored API so it doesn't handle form parsing anymore + added version.py and edited setup.py, version is now displayed in the footer of every page + reworked config to condense every persistence parameter and logging parameter diff -r b7ec9ee75bbb -r d1bc73ce855a setup.py --- a/setup.py Wed Jan 07 12:38:57 2015 +0100 +++ b/setup.py Mon Jan 12 12:00:30 2015 +0100 @@ -1,8 +1,12 @@ from setuptools import setup, find_packages +# Will set CURRENT_VERSION to the current version string and VERSION to the +# current version tuple +exec(compile(open("src/catedit/version.py"), "version.py", "exec")) + setup( name='catedit', - version="0.1.2", + version=CURRENT_VERSION, url='http://www.iri.centrepompidou.fr', author='I.R.I.', author_email='dev@iri.centrepompidou.fr', diff -r b7ec9ee75bbb -r d1bc73ce855a src/catedit/__init__.py --- a/src/catedit/__init__.py Wed Jan 07 12:38:57 2015 +0100 +++ b/src/catedit/__init__.py Mon Jan 12 12:00:30 2015 +0100 @@ -5,15 +5,17 @@ from logging import FileHandler, Formatter import os +from catedit.version import CURRENT_VERSION from flask import Flask, session +from flask_wtf.csrf import CsrfProtect from flask.ext.github import GitHub from flask.ext.cache import Cache from flask.ext.restful import Api from catedit.settings import AppSettings -# set up app and database +# Set up app app = Flask(__name__) app.config.from_object(AppSettings) cache = Cache(app, config={"CACHE_TYPE": "simple"}) @@ -32,14 +34,24 @@ if not app_configured: raise Exception("Catedit not configured") -#github +# CSRF protection +CsrfProtect(app) + +# Github +app.config["GITHUB_CLIENT_ID"] = app.config["PERSISTENCE_CONFIG"] \ + ["GITHUB_CLIENT_ID"] +app.config["GITHUB_CLIENT_SECRET"] = app.config["PERSISTENCE_CONFIG"] \ + ["GITHUB_CLIENT_SECRET"] github = GitHub(app) -#api +# Api api = Api(app) +# Version -#views +app.config["CURRENT_VERSION"] = CURRENT_VERSION + +# Views and APIs from catedit.views import cat_editor, cat_recap, github_login,\ github_callback, logout @@ -56,17 +68,18 @@ endpoint='category_changes') -# set up logging -if app.config["LOGGING"]: - file_handler = FileHandler(filename=app.config["LOG_FILE_PATH"]) +# Set up logging +if app.config["LOGGING_CONFIG"]["IS_LOGGING"]: + file_handler = FileHandler(filename=app.config["LOGGING_CONFIG"] + ["LOG_FILE_PATH"]) file_handler.setFormatter(Formatter( '%(asctime)s %(levelname)s: %(message)s ' '[in %(pathname)s:%(lineno)d]', '%Y-%m-%d %H:%M:%S' )) app.logger.addHandler(file_handler) - app.logger.setLevel(app.config["LOGGING_LEVEL"]) + app.logger.setLevel(app.config["LOGGING_CONFIG"]["LOGGING_LEVEL"]) -# session management +# Session management app.secret_key = app.config["SECRET_KEY"] diff -r b7ec9ee75bbb -r d1bc73ce855a src/catedit/config.py.tmpl --- a/src/catedit/config.py.tmpl Wed Jan 07 12:38:57 2015 +0100 +++ b/src/catedit/config.py.tmpl Mon Jan 12 12:00:30 2015 +0100 @@ -12,34 +12,56 @@ HOST = "0.0.0.0" DEBUG = True - LOGGING_LEVEL = "DEBUG" # WTForms settings SECRET_KEY = 'totally-secret-key' - # Saving file for local persistence - FILE_SAVE_DIRECTORY = "../../run/files/" + """ + Logging parameters + * IS_LOGGING is a boolean to activate/deactivate logging + * LOGGING_LEVEL sets the minimum level of messages to log + * LOG_FILE_PATH sets where the log files will be saved + """ + LOGGING_CONFIG = { + "IS_LOGGING": False, + "LOGGING_LEVEL": "DEBUG", + "LOG_FILE_PATH": "../../run/log/log.txt", + } - # Logging config - LOG_FILE_PATH = "../../run/log/log.txt" - LOGGING = False + """ + Category persistence parameters + METHOD can be: + * "PersistenceToFile" : will save categories to files on system + * "PersistenceToGithub" : will save categories to files on Github - # Github repository config + You then need additional parameters related to the chosen METHOD. + + If "PersistenceToFile" (Note: currently not supported) + * FILE_SAVE_PATH is the path to save the category files - REPOSITORY_LIST = [ - "catedit-dev-testing", - "habitabilite-prototype" - ] - REPOSITORY_OWNER = "catedit-system" - CATEGORIES_PATH = "categories/" - - # Github parameters - - GITHUB_CLIENT_ID = "github-id-placeholder" - GITHUB_CLIENT_SECRET = "github-secret-placeholder" - - # Property List + If "PersistenceToGithub" + * REPOSITORY_LIST is the list of repository available for users (Note: + for now, each user must have pushing right to EVERY repository in + this list in order to use the app) + * REPOSITORY_OWNER is the owner of the category repositories + * CATEGORIES_PATH is the path where the category serializations will + be saved within the repositories + * GITHUB_CLIENT_ID is the client ID of the Github app + * GITHUB_CLIENT_SECRET is the secret ID of the Github app + """ + PERSISTENCE_CONFIG = { + "METHOD" : "PersistenceToGithub", + "REPOSITORY_LIST" : [ + "catedit-dev-testing", + "catedit-dev-testing-2", + "habitabilite-prototype" + ], + "REPOSITORY_OWNER" : "catedit-system", + "CATEGORIES_PATH" : "categories/", + "GITHUB_CLIENT_ID" : "github-client-placeholder", + "GITHUB_CLIENT_SECRET" : "github-secret-placeholder", + } PROPERTY_LIST = { "subClassOf": { @@ -49,20 +71,13 @@ "rdflib_class": RDFS.subClassOf, "object_rdflib_class": URIRef, }, - "value": { - "descriptive_label_fr": "Valeur", - "descriptive_label_en": "Value", + "comment": { + "descriptive_label_fr": "Commentaire", + "descriptive_label_en": "Comment", "object_type": "literal", - "rdflib_class": RDF.value, + "rdflib_class": RDFS.comment, "object_rdflib_class": Literal, }, - "type": { - "descriptive_label_fr": "Type", - "descriptive_label_en": "Type", - "object_type": "uriref-category", - "rdflib_class": RDF.type, - "object_rdflib_class": URIRef, - }, "resource": { "descriptive_label_fr": "Ressource", "descriptive_label_en": "Resource", @@ -78,9 +93,3 @@ "object_rdflib_class": URIRef, } } - - # Category persistence parameters - # "PersistenceToFile" : will save categories to files on system - # "PersistenceToGithub" : will save categories to files on Github - - PERSISTENCE_METHOD = "PersistenceToGithub" diff -r b7ec9ee75bbb -r d1bc73ce855a src/catedit/models.py --- a/src/catedit/models.py Wed Jan 07 12:38:57 2015 +0100 +++ b/src/catedit/models.py Mon Jan 12 12:00:30 2015 +0100 @@ -52,13 +52,21 @@ if other_properties: for (predicate, obj) in other_properties: + rdf_obj = "" + if (app.config["PROPERTY_LIST"] + [predicate] + ["object_type"] == "uriref-category"): + rdf_obj = app.config["CATEGORY_NAMESPACE"] + obj + else: + rdf_obj = obj self.cat_graph.add((self.this_category, app.config["PROPERTY_LIST"] [predicate] ["rdflib_class"], app.config["PROPERTY_LIST"] [predicate] - ["object_rdflib_class"](obj))) + ["object_rdflib_class"] + (rdf_obj))) else: self.cat_graph = graph @@ -157,13 +165,21 @@ ["rdflib_class"], None)) for (predicate, obj) in new_other_properties: + rdf_obj = "" + if (app.config["PROPERTY_LIST"] + [predicate] + ["object_type"] == "uriref-category"): + rdf_obj = app.config["CATEGORY_NAMESPACE"] + obj + else: + rdf_obj = obj self.cat_graph.add((self.this_category, - app.config["PROPERTY_LIST"] - [predicate] - ["rdflib_class"], - app.config["PROPERTY_LIST"] - [predicate] - ["object_rdflib_class"](obj))) + app.config["PROPERTY_LIST"] + [predicate] + ["rdflib_class"], + app.config["PROPERTY_LIST"] + [predicate] + ["object_rdflib_class"] + (rdf_obj))) class CategoryManager(object): diff -r b7ec9ee75bbb -r d1bc73ce855a src/catedit/persistence.py --- a/src/catedit/persistence.py Wed Jan 07 12:38:57 2015 +0100 +++ b/src/catedit/persistence.py Mon Jan 12 12:00:30 2015 +0100 @@ -80,7 +80,8 @@ """ Saves to a file """ - path_to_save = app.config["FILE_SAVE_DIRECTORY"]+kwargs["name"] + path_to_save = app.config["PERSISTENCE_CONFIG"]["FILE_SAVE_DIRECTORY"] + + kwargs["name"] file_to_save = open(path_to_save, 'wb') file_to_save.write(kwargs["content"]) file_to_save.close() @@ -89,7 +90,8 @@ """ Loads from a file """ - path_to_load = app.config["FILE_SAVE_DIRECTORY"]+kwargs["name"] + path_to_load = app.config["PERSISTENCE_CONFIG"]["FILE_SAVE_DIRECTORY"] + + kwargs["name"] file_to_load = open(path_to_load, 'rb') file_content = file_to_load.read() file_to_load.close() @@ -99,7 +101,8 @@ """ Deletes a file """ - path_to_delete = app.config["FILE_SAVE_DIRECTORY"]+kwargs["name"] + path_to_delete = app.config["PERSISTENCE_CONFIG"] + ["FILE_SAVE_DIRECTORY"] + kwargs["name"] os.remove(path_to_delete) # IDEA: return { file_name: file_content } type dict @@ -108,10 +111,12 @@ Lists all files in file directory (as set in config.py) """ file_content_list = [] - for file_name in os.listdir(app.config["FILE_SAVE_DIRECTORY"]): + for file_name in os.listdir(app.config["PERSISTENCE_CONFIG"] + ["FILE_SAVE_DIRECTORY"]): if not file_name or file_name[0] == ".": continue - path_to_load = open(app.config["FILE_SAVE_DIRECTORY"]+file_name) + path_to_load = open(app.config["PERSISTENCE_CONFIG"] + ["FILE_SAVE_DIRECTORY"] + file_name) file_content = path_to_load.read() path_to_load.close() file_content_list.append(file_content) @@ -189,7 +194,7 @@ try: ref_master = github.get( "repos/" - + app.config["REPOSITORY_OWNER"] + "/" + + app.config["PERSISTENCE_CONFIG"]["REPOSITORY_OWNER"] + "/" + self.repository + "/git/refs/heads/master" ) @@ -200,7 +205,7 @@ logger.debug( "Endpoint: " + "repos/" - + app.config["REPOSITORY_OWNER"] + "/" + + app.config["PERSISTENCE_CONFIG"]["REPOSITORY_OWNER"] + "/" + self.repository + "/git/refs/heads/master" ) @@ -211,7 +216,7 @@ try: last_commit_master = github.get( "repos/" - + app.config["REPOSITORY_OWNER"] + "/" + + app.config["PERSISTENCE_CONFIG"]["REPOSITORY_OWNER"] + "/" + self.repository + "/git/commits/" + ref_master["object"]["sha"] @@ -227,7 +232,7 @@ try: last_commit_tree = github.get( "repos/" - + app.config["REPOSITORY_OWNER"] + "/" + + app.config["PERSISTENCE_CONFIG"]["REPOSITORY_OWNER"] + "/" + self.repository + "/git/trees/" + last_commit_master["tree"]["sha"] @@ -253,7 +258,8 @@ # no point doing anything, the file won't be in the new tree if not( element["path"] in [ - (app.config["CATEGORIES_PATH"] + cat_name) + (app.config["PERSISTENCE_CONFIG"] + ["CATEGORIES_PATH"] + cat_name) for cat_name in deletion_dict.keys() ] ): @@ -265,7 +271,8 @@ # test if element is in modified categories if ( element["path"] in [ - (app.config["CATEGORIES_PATH"] + cat_name) + (app.config["PERSISTENCE_CONFIG"] + ["CATEGORIES_PATH"] + cat_name) for cat_name in modification_dict.keys() ] ): @@ -273,7 +280,8 @@ for (cat_name, cat_content) \ in modification_dict.items(): if element["path"] == ( - app.config["CATEGORIES_PATH"] + cat_name + app.config["PERSISTENCE_CONFIG"] + ["CATEGORIES_PATH"] + cat_name ): # 4-1 for modified files, creating a new blob new_blob_data = { @@ -283,7 +291,8 @@ try: new_blob = github.post( "repos/" - + app.config["REPOSITORY_OWNER"] + "/" + + app.config["PERSISTENCE_CONFIG"] + ["REPOSITORY_OWNER"] + "/" + self.repository + "/git/blobs", data=new_blob_data @@ -313,11 +322,13 @@ # Now we loop over modified categories to find the ones that don't # exist yet in the last commit tree in order to create blobs for them for (cat_name, cat_content) in modification_dict.items(): - logger.debug(app.config["CATEGORIES_PATH"]+cat_content + logger.debug(app.config["PERSISTENCE_CONFIG"] + ["CATEGORIES_PATH"] + cat_content + " should not be in " + str([elt["path"] for elt in last_commit_tree["tree"]])) - if (app.config["CATEGORIES_PATH"] + cat_name not in + if (app.config["PERSISTENCE_CONFIG"] + ["CATEGORIES_PATH"] + cat_name not in [elt["path"] for elt in last_commit_tree["tree"]]): # 4-1 for added files, creating a new blob @@ -326,7 +337,8 @@ try: new_blob = github.post( "repos/" - + app.config["REPOSITORY_OWNER"] + "/" + + app.config["PERSISTENCE_CONFIG"] + ["REPOSITORY_OWNER"] + "/" + self.repository + "/git/blobs", data=new_blob_data @@ -340,7 +352,8 @@ logger.debug(ghe.response.text) logger.debug(str(github.get("rate_limit")["resources"])) new_tree_data["tree"].append({ - "path": app.config["CATEGORIES_PATH"] + cat_name, + "path": app.config["PERSISTENCE_CONFIG"] + ["CATEGORIES_PATH"] + cat_name, "mode": "100644", "type": "blob", "sha": new_blob["sha"] @@ -351,7 +364,7 @@ try: new_tree_response = github.post( "repos/" - + app.config["REPOSITORY_OWNER"]+"/" + + app.config["PERSISTENCE_CONFIG"]["REPOSITORY_OWNER"] + "/" + self.repository + "/git/trees", data=new_tree_data @@ -371,7 +384,7 @@ try: new_commit = github.post( "repos/" - + app.config["REPOSITORY_OWNER"]+"/" + + app.config["PERSISTENCE_CONFIG"]["REPOSITORY_OWNER"] + "/" + self.repository + "/git/commits", data=new_commit_data @@ -391,7 +404,7 @@ try: new_head = github.patch( "repos/" - + app.config["REPOSITORY_OWNER"] + "/" + + app.config["PERSISTENCE_CONFIG"]["REPOSITORY_OWNER"] + "/" + self.repository + "/git/refs/heads/master", data=json.dumps(new_head_data) @@ -413,10 +426,12 @@ """ try: filedict = github.get("repos/" - + app.config["REPOSITORY_OWNER"]+"/" + + app.config["PERSISTENCE_CONFIG"] + ["REPOSITORY_OWNER"] + "/" + self.repository + "/contents/" - + app.config["CATEGORIES_PATH"] + + app.config["PERSISTENCE_CONFIG"] + ["CATEGORIES_PATH"] + kwargs["name"]) file_content = str(b64decode(filedict["content"]), "utf-8") except GitHubError as ghe: @@ -445,10 +460,12 @@ filenames_list = [] try: files_in_repo = github.get("repos/" - + app.config["REPOSITORY_OWNER"]+"/" + + app.config["PERSISTENCE_CONFIG"] + ["REPOSITORY_OWNER"] + "/" + self.repository + "/contents/" - + app.config["CATEGORIES_PATH"]) + + app.config["PERSISTENCE_CONFIG"] + ["CATEGORIES_PATH"]) filenames_list = [github_file["name"] for github_file in files_in_repo] # logger.debug(filenames_list) @@ -465,10 +482,12 @@ for filename in filenames_list: try: filedict = github.get("repos/" - + app.config["REPOSITORY_OWNER"]+"/" + + app.config["PERSISTENCE_CONFIG"] + ["REPOSITORY_OWNER"] + "/" + self.repository + "/contents/" - + app.config["CATEGORIES_PATH"] + + app.config["PERSISTENCE_CONFIG"] + ["CATEGORIES_PATH"] + filename) file_content_list.append(str(b64decode(filedict["content"]), "utf-8")) diff -r b7ec9ee75bbb -r d1bc73ce855a src/catedit/resources.py --- a/src/catedit/resources.py Wed Jan 07 12:38:57 2015 +0100 +++ b/src/catedit/resources.py Mon Jan 12 12:00:30 2015 +0100 @@ -8,7 +8,7 @@ from rdflib import Graph from flask.ext.restful import Resource, reqparse -from flask import request, session +from flask import session from catedit import app, cache from catedit.models import Category, CategoryManager import catedit.persistence @@ -39,7 +39,7 @@ cat_manager_instance = CategoryManager( getattr( catedit.persistence, - app.config["PERSISTENCE_METHOD"] + app.config["PERSISTENCE_CONFIG"]["METHOD"] )(repository=repository), ) if cat_id is not None: @@ -53,16 +53,28 @@ return response # update category cat_id - def put(self, repository, cat_id=None): + def put(self, repository, cat_id=None, cat_data=None): """ API to edit an existing category - * If cat_id is None and persistence support change sets, will + + Args are: + * repository: the repository where the category cat_id is stored + * cat_id : the id of the category to edit + * cat_data : the new data of the category, dict of the form: + { + "label": edited_cat_label, + "description": edited_cat_description, + "properties": [(predicate1, object1), (predicate2, object2)] + } + List of predicate is available in config.py, key PROPERTY_LIST + + Note: If cat_id is None and persistence support change sets, will submit all changes to category list """ cat_manager_instance = CategoryManager( getattr( catedit.persistence, - app.config["PERSISTENCE_METHOD"] + app.config["PERSISTENCE_CONFIG"]["METHOD"] )(repository=repository), ) args = cat_parser.parse_args() @@ -82,28 +94,6 @@ message=args["commit_message"] ) else: - new_property_list = [] - logger.debug(args["property_predicate"]) - logger.debug(args["property_object"]) - if args["property_predicate"] is not None and \ - args["property_object"] is not None: - for property_predicate, property_object in zip( - args["property_predicate"], - args["property_object"]): - if property_object: - property_object_to_append = property_object - # if URIRef category, we must prefix id with namespace - if (app.config["PROPERTY_LIST"] - [property_predicate] - ["object_type"]) == "uriref-category": - property_object_to_append = \ - app.config["CATEGORY_NAMESPACE"] \ - + property_object - logger.debug(property_object_to_append) - new_property_list.append((property_predicate, - property_object_to_append)) - logger.debug(new_property_list) - # is the edition occuring on an already modified category? if cat_id in session.get("modified_categories", {}) \ .get(repository, {}).keys(): @@ -118,9 +108,9 @@ else: cat = cat_manager_instance.load_category(cat_id) - cat.edit_category(new_description=args["description"], - new_label=args["label"], - new_other_properties=new_property_list) + cat.edit_category(new_description=cat_data["description"], + new_label=cat_data["label"], + new_other_properties=cat_data["properties"]) session["modified_categories"][repository][cat.cat_id] = str( cat.cat_graph.serialize(format="turtle"), "utf-8" @@ -138,31 +128,25 @@ cache.clear() return 204 - def post(self, repository): + def post(self, repository, cat_data): """ API to create a new category + + Args are: + * repository: the repository where the category cat_id is stored + * cat_data : the new data of the category, dict of the form: + { + "label": edited_cat_label, + "description": edited_cat_description, + "properties": [(predicate1, object1), (predicate2, object2)] + } + List of predicate is available in config.py, key PROPERTY_LIST """ - args = cat_parser.parse_args() - property_list = [] - logger.debug(args["property_predicate"]) - logger.debug(args["property_object"]) - for property_predicate, property_object in zip( - request.form.getlist('property_predicate'), - request.form.getlist('property_object')): - if property_object: - if (app.config["PROPERTY_LIST"] - [property_predicate] - ["object_type"]) == "uriref-category": - property_list.append((property_predicate, - app.config["CATEGORY_NAMESPACE"] - + property_object)) - else: - property_list.append((property_predicate, - property_object)) - logger.debug(property_list) - cat = Category(label=args["label"], - description=args["description"], - other_properties=property_list) + cat = Category( + label=cat_data["label"], + description=cat_data["description"], + other_properties=cat_data["properties"] + ) if cat.cat_id not in session["modified_categories"][repository].keys(): session["modified_categories"][repository][cat.cat_id] = str( @@ -215,7 +199,7 @@ cat_manager_instance = CategoryManager( getattr( catedit.persistence, - app.config["PERSISTENCE_METHOD"] + app.config["PERSISTENCE_CONFIG"]["METHOD"] )(repository=repository), ) cat_list = cat_manager_instance.list_categories() @@ -320,8 +304,7 @@ """ logger.debug(modified_cat_id) logger.debug(str(session.get("modified_categories", {}))) - logger.debug(str(session.get("modified_categories", {}) - .get(repository, {}))) + if modified_cat_id is None: return { "modified_categories": session.get("modified_categories", {}) diff -r b7ec9ee75bbb -r d1bc73ce855a src/catedit/static/js/property_functions.js --- a/src/catedit/static/js/property_functions.js Wed Jan 07 12:38:57 2015 +0100 +++ b/src/catedit/static/js/property_functions.js Mon Jan 12 12:00:30 2015 +0100 @@ -9,12 +9,12 @@ initPropertyCount : function(divName,initValue){ propertyCount=initValue; - if (propertyCount > 0) { + if (propertyCount >= 0) { document.getElementById(divName).className="visible"; } }, - addProperty : function(divToReveal, divToAddFieldsTo){ + addProperty : function(divToReveal, divToAddFieldsTo, csrfToken){ var selectElement = document.getElementById('property_selector'); if (selectElement.options[selectElement.selectedIndex].id != "property_type_default") { var selectedOptionValue = selectElement.options[selectElement.selectedIndex].value; @@ -43,9 +43,10 @@ ((propertyObjectValue != "") && (propertyOptionObjectType == "uriref-link")) || ((propertyObjectValue != "default") && (propertyOptionObjectType == "uriref-category"))) { var newProperty = document.createElement('tr'); - newProperty.setAttribute('id','property_tr'+(createdProperties+1)); - var newPropertyHTML = ' \ - \ + newProperty.setAttribute('id','properties-'+(createdProperties+1)); + var newPropertyHTML = ' \ + \ + \ \ '+selectedOptionText+' \ \ @@ -61,7 +62,7 @@ parentElement.insertBefore(newProperty, parentElement.firstChild); createdProperties++; propertyCount++; - if (propertyCount > 0) { + if (propertyCount >= 0) { document.getElementById(divToReveal).className="visible"; } } @@ -70,7 +71,7 @@ removeProperty : function(index, divToReveal){ if (propertyCount > 0) { - divItem=document.getElementById("property_tr"+index); + divItem=document.getElementById("properties-"+index); divItem.parentNode.removeChild(divItem); propertyCount--; if (propertyCount == 0) { diff -r b7ec9ee75bbb -r d1bc73ce855a src/catedit/templates/catbase.html --- a/src/catedit/templates/catbase.html Wed Jan 07 12:38:57 2015 +0100 +++ b/src/catedit/templates/catbase.html Mon Jan 12 12:00:30 2015 +0100 @@ -38,7 +38,7 @@