# HG changeset patch # User durandn # Date 1429009285 -7200 # Node ID a55054e72fe46ef9e7e53df7a4841f9648a2aeec # Parent 4a493776539ad667d3078c786390b4cebbbe5e8c Reworking cache keys registries and access diff -r 4a493776539a -r a55054e72fe4 src/catedit/resources.py --- a/src/catedit/resources.py Mon Apr 13 18:13:14 2015 +0200 +++ b/src/catedit/resources.py Tue Apr 14 13:01:25 2015 +0200 @@ -34,22 +34,13 @@ The result of this API function goes into the Flask Cache. """ + + # Cache key for this function and parameters cache_key = "categoryapi_get_" + repository + "_" + cat_id - # Maintaining repo cache key list - repo_key = "categories_"+repository+"_keys" - if cache.get(repo_key) is None: - cache.set(repo_key, [cache_key]) - else: - if cache_key not in cache.get(repo_key): - key_list = cache.get(repo_key) - key_list.append(cache_key) - logger.debug(key_list) - cache.set(repo_key, key_list) - - logger.debug("keys in category: "+repo_key+" :"+str(cache.get(repo_key))) - - if cache.get(cache_key) is None: + # Cache key for category key registry for this repo + registry_key = "categories_"+repository+"_keys" + if (cache.get(registry_key) is None or cache_key not in cache.get(registry_key)) and cache.get(cache_key) is None: rv = None cat_manager_instance = CategoryManager( getattr( @@ -72,7 +63,18 @@ cat.cat_graph.serialize(format='turtle').decode("utf-8") ) rv = response, 200 + # Cache operations + # Setting key for this function cache.set(cache_key, rv, timeout=3600) + # Setting key in the registry + if cache.get(registry_key) is None: + cache.set(registry_key, [cache_key]) + else: + if cache_key not in cache.get(registry_key): + key_list = cache.get(registry_key) + key_list.append(cache_key) + cache.set(registry_key, key_list) + logger.debug("list of key for "+registry_key+" registry: "+str(cache.get(registry_key))) return rv else: return cache.get(cache_key) diff -r 4a493776539a -r a55054e72fe4 src/catedit/views/utils.py --- a/src/catedit/views/utils.py Mon Apr 13 18:13:14 2015 +0200 +++ b/src/catedit/views/utils.py Tue Apr 14 13:01:25 2015 +0200 @@ -82,24 +82,16 @@ "date": date of the comment format dd/mm/yy hh:mm } """ + # Cache key for this function and parameters cache_key = "get_comments_" \ + repository + "_" \ + thread_type + "_" \ + thread_id + "_" \ + str(page) + "_" + str(per_page) + # Cache key for comments key registry for this repo + registry_key = "comments_"+repository+"_keys" - # Maintaining repo comments keys list - repo_key = "comments_"+repository+"_keys" - if cache.get(repo_key) is None: - cache.set(repo_key, [cache_key]) - else: - if cache_key not in cache.get(repo_key): - key_list = cache.get(repo_key) - key_list.append(cache_key) - logger.debug(key_list) - cache.set(repo_key, key_list) - - if cache.get(cache_key) is None: + if (cache.get(registry_key) is None or cache_key not in cache.get(registry_key)) and cache.get(cache_key) is None: github_comments_data = [] try: @@ -193,8 +185,6 @@ ) thread_title = discussion_data.get("title", "") thread_opening_post = discussion_data.get("body", "") - - thread_dict = { "author": thread_author, "title": thread_title, @@ -203,9 +193,21 @@ "opening_post": thread_opening_post, "per_page": per_page } - - cache.set(cache_key, (thread_dict, pagination), timeout=3600) - return (thread_dict, pagination) + rv = (thread_dict, pagination) + + # Cache operations + # Setting key for this function + cache.set(cache_key, rv, timeout=3600) + # Setting key in the registry + if cache.get(registry_key) is None: + cache.set(registry_key, [cache_key]) + else: + if cache_key not in cache.get(registry_key): + key_list = cache.get(registry_key) + key_list.append(cache_key) + logger.debug("list of keys for "+registry_key+" registry: "+str(key_list)) + cache.set(registry_key, key_list) + return rv else: return cache.get(cache_key) @@ -271,10 +273,12 @@ ) logger.error(ghe.response.text) - if cache.get("comments_"+repository+"_keys") is not None: - for key in cache.get("comments_"+repository+"_keys"): + registry_key = "comments_"+repository+"_keys" + if cache.get(registry_key) is not None: + for key in cache.get(registry_key): if cache.get(key) is not None: cache.delete(key) + cache.delete(registry_key) return return_id @@ -289,24 +293,15 @@ comment_count : commit comments count } """ + # Cache key for this function and parameters cache_key = "get_commits_" \ + repository + "_" \ + str(page) + "_" + str(per_page) + + # Cache key for comments key registry for this repo + registry_key = "categories_"+repository+"_keys" - # Maintaining repo cache key list - repo_key = "categories_"+repository+"_keys" - if cache.get(repo_key) is None: - cache.set(repo_key, [cache_key]) - else: - if cache_key not in cache.get(repo_key): - key_list = cache.get(repo_key) - key_list.append(cache_key) - logger.debug(key_list) - cache.set(repo_key, key_list) - - logger.debug("keys in category: "+repo_key+" :"+str(cache.get(repo_key))) - - if cache.get(cache_key) is None: + if (cache.get(registry_key) is None or cache_key not in cache.get(registry_key)) and cache.get(cache_key) is None: commits_data = [] try: commits_data = github.get( @@ -341,8 +336,21 @@ } for commit in commits_data ] - cache.set(cache_key, (changeset_list, commits_pagination), timeout=3600) - return (changeset_list, commits_pagination) + rv = (changeset_list, commits_pagination) + + # Cache operations + # Setting key for this function + cache.set(cache_key, rv, timeout=3600) + # Setting key in the registry + if cache.get(registry_key) is None: + cache.set(registry_key, [cache_key]) + else: + if cache_key not in cache.get(registry_key): + key_list = cache.get(registry_key) + key_list.append(cache_key) + logger.debug("list of keys for "+registry_key+" registry: "+str(key_list)) + cache.set(registry_key, key_list) + return rv else: return cache.get(cache_key) @@ -359,24 +367,16 @@ comment_count: comments count } """ + + # Cache key for this function and parameters cache_key = "get_issues_" \ + repository + "_" \ + str(page) + "_" + str(per_page) - # Maintaining repo comments keys list - repo_key = "comments_"+repository+"_keys" - if cache.get(repo_key) is None: - cache.set(repo_key, [cache_key]) - else: - if cache_key not in cache.get(repo_key): - key_list = cache.get(repo_key) - key_list.append(cache_key) - logger.debug(key_list) - cache.set(repo_key, key_list) + # Cache key for comments key registry for this repo + registry_key = "comments_"+repository+"_keys" - logger.debug("keys in comments: "+repo_key+" :"+str(cache.get(repo_key))) - - if cache.get(cache_key) is None: + if (cache.get(registry_key) is None or cache_key not in cache.get(registry_key)) and cache.get(cache_key) is None: issues_data = [] try: issues_data = github.get( @@ -410,8 +410,21 @@ } for issue in issues_data ] - cache.set(cache_key, (discussion_list, discussions_pagination), timeout=3600) - return (discussion_list, discussions_pagination) + rv = (discussion_list, discussions_pagination) + + # Cache operations + # Setting key for this function + cache.set(cache_key, rv, timeout=3600) + # Setting key in the registry + if cache.get(registry_key) is None: + cache.set(registry_key, [cache_key]) + else: + if cache_key not in cache.get(registry_key): + key_list = cache.get(registry_key) + key_list.append(cache_key) + logger.debug("list of keys for "+registry_key+" registry: "+str(key_list)) + cache.set(registry_key, key_list) + return rv else: return cache.get(cache_key) @@ -421,6 +434,8 @@ Get the category list as it was following the changeset of id changeset_id """ + # Cache key for a given changeset (or its parent) in a given repository + # As this key will never need to be cleared we have no use putting it in a registry cache_key = "get_category_list_for_commit_" \ + repository + "_" \ + changeset_id + "_parent_" + str(get_parent) @@ -544,10 +559,8 @@ modified_cat_dict = {} serialized_cat_list = [] if session.get("user_logged", None) is not None: - serialized_cat_list = cat_api_instance.get(repository=repository) \ - [0] - cat_changes = cat_changes_api_instance.get(repository=repository) \ - [0] + serialized_cat_list = cat_api_instance.get(repository=repository)[0] + cat_changes = cat_changes_api_instance.get(repository=repository)[0] modified_cat_dict = cat_changes["modified_categories"] deleted_cat_dict = cat_changes["deleted_categories"] # logger.debug(serialized_cat_list)