cache optimisation : no more cache.clear() but 2 "keys sets" for each repository: one for the categories and one for the comments and discussion + corrected slight mistake in macro that messed up with sessions
--- a/src/catedit/resources.py Mon Apr 13 12:37:41 2015 +0200
+++ b/src/catedit/resources.py Mon Apr 13 17:33:32 2015 +0200
@@ -43,6 +43,20 @@
The result of this API function goes into the Flask Cache.
"""
cache_key = "categoryapi_get_" + repository + "_" + cat_id
+
+ # Maintaining repo cache key list
+ repo_key = "categories_"+repository+"_keys"
+ if cache.get(repo_key) is None:
+ cache.set(repo_key, [cache_key])
+ else:
+ if cache_key not in cache.get(repo_key):
+ key_list = cache.get(repo_key)
+ key_list.append(cache_key)
+ logger.debug(key_list)
+ cache.set(repo_key, key_list)
+
+ logger.debug("keys in category: "+repo_key+" :"+str(cache.get(repo_key)))
+
if cache.get(cache_key) is None:
rv = None
cat_manager_instance = CategoryManager(
@@ -118,7 +132,6 @@
session.setdefault("tasks", {}).setdefault(repository, []).append(task.id)
session.setdefault("deleted_categories", {})[repository] = {}
session.setdefault("modified_categories", {})[repository] = {}
- cache.clear()
return 204
else:
# is the edition occuring on an already modified category?
--- a/src/catedit/tasks.py Mon Apr 13 12:37:41 2015 +0200
+++ b/src/catedit/tasks.py Mon Apr 13 17:33:32 2015 +0200
@@ -33,7 +33,8 @@
modified_cat_dict=modified_categories,
message=message
)
- r = requests.post(app.config['BASE_URL']+"/meta/cache-clear")
+ registry_key = "categories_"+repository+"_keys"
+ r = requests.post(app.config['BASE_URL']+"/meta/cache-clear/"+registry_key)
try:
r.raise_for_status()
except:
--- a/src/catedit/templates/macros.html Mon Apr 13 12:37:41 2015 +0200
+++ b/src/catedit/templates/macros.html Mon Apr 13 17:33:32 2015 +0200
@@ -94,7 +94,7 @@
{% if cat.state != "deleted" %}
<div class="cat-delete-div" id="delete_confirm_{% if ((cat.state != 'untouched') and (cat.state != 'original')) %}edited_{% endif %}{{cat.cat_id}}{% if buttons_id %}-{{ buttons_id }}{% endif %}">
<form method="POST" action=
- "{% if cat.state == 'modified' %}
+ "{% if cat.state == 'modified' or cat.state == 'created' %}
{{url_for(target, deleted_changes_id=cat.cat_id, repository=current_repository)}}
{% else %}
{{url_for(target, deleted_cat_id=cat.cat_id, repository=current_repository)}}
--- a/src/catedit/views/home.py Mon Apr 13 12:37:41 2015 +0200
+++ b/src/catedit/views/home.py Mon Apr 13 17:33:32 2015 +0200
@@ -51,10 +51,6 @@
session["user_logged"] = True
session["user_login"] = "auth-error"
- session["properties"] = {
- repo: {} for repo
- in app.config["PERSISTENCE_CONFIG"]["REPOSITORY_LIST"]
- }
session["modified_categories"] = {
repo: {} for
repo in app.config["PERSISTENCE_CONFIG"]["REPOSITORY_LIST"]
@@ -67,7 +63,6 @@
repo: [] for repo
in app.config["PERSISTENCE_CONFIG"]["REPOSITORY_LIST"]
}
-
try:
session["user_login"] = github.get(
"user",
@@ -90,21 +85,6 @@
)
session["user_can_edit"] = {}
for repo in session["user_repositories"]:
- try:
- json_file=github.get(
- "repos/"
- + app.config["PERSISTENCE_CONFIG"]["REPOSITORY_OWNER"] + "/"
- + repo + "/contents/properties/properties.json",
- hooks=dict(response=log_api_rate)
- )
- #TODO: vérifier encoding - logger.debug("repo: "+repo+" - properties: "+str(json.loads(str(b64decode(json_file["content"]), "utf-8")))) #wat
- session["properties"][repo] = json.loads(str(b64decode(json_file["content"]), "utf-8"))["property_list"]
- except GitHubError as ghe:
- logger.debug(
- "GitHub Error trying to get the property list. We'll assume " +
- "there is none and use default list as defined in config.py"
- )
- session["properties"][repo] = app.config["PROPERTY_LIST"]
session["user_can_edit"][repo] = True
logger.debug("User can edit: "+str(session["user_can_edit"]))
except GitHubError as ghe:
--- a/src/catedit/views/meta.py Mon Apr 13 12:37:41 2015 +0200
+++ b/src/catedit/views/meta.py Mon Apr 13 17:33:32 2015 +0200
@@ -15,14 +15,17 @@
@csrf.exempt
-@meta.route('/cache-clear', defaults={'cache_keys': None}, methods=["POST"])
-@meta.route('/cache-clear/<cache_keys>', methods=["POST"])
-def cache_clear_endpoint(cache_keys):
+@meta.route('/cache-clear', defaults={'registry_key': None}, methods=["POST"])
+@meta.route('/cache-clear/<string:registry_key>', methods=["POST"])
+def cache_clear_endpoint(registry_key):
if not cache_keys:
cache.clear()
else:
- cache_keys_list = cache_keys.split(",")
- cache.delete_many(*cache_keys_list)
+ if cache.get(registry_key) is not None:
+ for key in cache.get(registry_key):
+ if cache.get(key) is not None:
+ cache.delete(key)
+ cache.delete(registry_key)
return ('', 204)
--- a/src/catedit/views/utils.py Mon Apr 13 12:37:41 2015 +0200
+++ b/src/catedit/views/utils.py Mon Apr 13 17:33:32 2015 +0200
@@ -87,6 +87,18 @@
+ thread_type + "_" \
+ thread_id + "_" \
+ str(page) + "_" + str(per_page)
+
+ # Maintaining repo comments keys list
+ repo_key = "comments_"+repository+"_keys"
+ if cache.get(repo_key) is None:
+ cache.set(repo_key, [cache_key])
+ else:
+ if cache_key not in cache.get(repo_key):
+ key_list = cache.get(repo_key)
+ key_list.append(cache_key)
+ logger.debug(key_list)
+ cache.set(repo_key, key_list)
+
if cache.get(cache_key) is None:
github_comments_data = []
@@ -258,7 +270,11 @@
+ " - comment_body : " + comment_body
)
logger.error(ghe.response.text)
- cache.clear()
+
+ if cache.get("comments_"+repository+"_keys") is not None:
+ for key in cache.get("comments_"+repository+"_keys"):
+ if cache.get(key) is not None:
+ cache.delete(key)
return return_id
@@ -276,6 +292,20 @@
cache_key = "get_commits_" \
+ repository + "_" \
+ str(page) + "_" + str(per_page)
+
+ # Maintaining repo cache key list
+ repo_key = "categories_"+repository+"_keys"
+ if cache.get(repo_key) is None:
+ cache.set(repo_key, [cache_key])
+ else:
+ if cache_key not in cache.get(repo_key):
+ key_list = cache.get(repo_key)
+ key_list.append(cache_key)
+ logger.debug(key_list)
+ cache.set(repo_key, key_list)
+
+ logger.debug("keys in category: "+repo_key+" :"+str(cache.get(repo_key)))
+
if cache.get(cache_key) is None:
commits_data = []
try:
@@ -332,6 +362,20 @@
cache_key = "get_issues_" \
+ repository + "_" \
+ str(page) + "_" + str(per_page)
+
+ # Maintaining repo comments keys list
+ repo_key = "comments_"+repository+"_keys"
+ if cache.get(repo_key) is None:
+ cache.set(repo_key, [cache_key])
+ else:
+ if cache_key not in cache.get(repo_key):
+ key_list = cache.get(repo_key)
+ key_list.append(cache_key)
+ logger.debug(key_list)
+ cache.set(repo_key, key_list)
+
+ logger.debug("keys in comments: "+repo_key+" :"+str(cache.get(repo_key)))
+
if cache.get(cache_key) is None:
issues_data = []
try: