|
1 # -*- coding: utf-8 -*- |
|
2 ''' |
|
3 Created on Mar 22, 2013 |
|
4 |
|
5 @author: tc |
|
6 ''' |
|
7 |
|
8 from django.contrib.auth.models import User, Group |
|
9 from django.core.management import call_command |
|
10 from django.core.management.base import BaseCommand, CommandError |
|
11 from ldt.ldt_utils.models import Content, Project |
|
12 from optparse import make_option |
|
13 import os.path |
|
14 import json |
|
15 from ldt.security.cache import cached_assign |
|
16 |
|
17 |
|
18 class Command(BaseCommand): |
|
19 ''' |
|
20 Load users, medias, contents, project and guardian permissions from json file generated by dumpdata |
|
21 ''' |
|
22 |
|
23 args = 'json_file' |
|
24 help = 'Load users, medias, contents, project and guardian permissions from json file generated by dumpdata' |
|
25 |
|
26 option_list = BaseCommand.option_list + ( |
|
27 make_option('-u', '--ignore-users', |
|
28 dest= 'ignore_users', |
|
29 default= None, |
|
30 help= 'list of usernames to ignore (separated by comma)' |
|
31 ), |
|
32 make_option('-g', '--ignore-groups', |
|
33 dest= 'ignore_groups', |
|
34 default= None, |
|
35 help= 'list of group names to ignore (separated by comma)' |
|
36 ), |
|
37 make_option('-c', '--ignore-contents', |
|
38 dest= 'ignore_contents', |
|
39 default= None, |
|
40 help= 'list of content iri_id to ignore (separated by comma)' |
|
41 ), |
|
42 make_option('-n', '--new-group', |
|
43 dest= 'new_group', |
|
44 default= None, |
|
45 help= 'The script will create a new group and assign view permission for all new media/contents to all the new users' |
|
46 ), |
|
47 ) |
|
48 |
|
49 |
|
50 def __safe_get(self, dict_arg, key, conv = lambda x: x, default= None): |
|
51 val = dict_arg.get(key, default) |
|
52 return conv(val) if val else default |
|
53 |
|
54 def __safe_decode(self, s): |
|
55 if not isinstance(s, basestring): |
|
56 return s |
|
57 try: |
|
58 return s.decode('utf8') |
|
59 except: |
|
60 try: |
|
61 return s.decode('latin1') |
|
62 except: |
|
63 return s.decode('utf8','replace') |
|
64 |
|
65 def handle(self, *args, **options): |
|
66 |
|
67 # Test path |
|
68 if len(args) != 1: |
|
69 raise CommandError("The command has no argument or too much arguments. Only one is needed : the json file path.") |
|
70 |
|
71 # Check if temporary files already exist |
|
72 path = os.path.abspath(args[0]) |
|
73 dir = os.path.dirname(path) |
|
74 path_file1 = os.path.join(dir, 'temp_data1.json') |
|
75 path_file2 = os.path.join(dir, 'temp_data2.json') |
|
76 do_import = True |
|
77 if os.path.exists(path_file1) or os.path.exists(path_file2): |
|
78 confirm = raw_input((""" |
|
79 The folder %s contains the files temp_data1.json or temp_data2.json. These files will be overwritten. |
|
80 |
|
81 Do you want to continue ? |
|
82 |
|
83 Type 'y' to continue, or 'n' to quit: """) % dir) |
|
84 do_import = (confirm == "y") |
|
85 |
|
86 # Continue |
|
87 if do_import: |
|
88 # Init ignore list |
|
89 user_ignore_list = ["admin","AnonymousUser"] |
|
90 group_ignore_list = ["everyone","Hashcut IRI","Hashcut BPI"] |
|
91 content_ignore_list = [] |
|
92 |
|
93 # Update ignore list |
|
94 ignore_users = options.get('ignore_users', None) |
|
95 ignore_groups = options.get('ignore_groups', None) |
|
96 ignore_contents = options.get('ignore_contents', None) |
|
97 if ignore_users: |
|
98 for u in ignore_users.split(","): |
|
99 user_ignore_list.append(u) |
|
100 if ignore_groups: |
|
101 for g in ignore_groups.split(","): |
|
102 group_ignore_list.append(g) |
|
103 if ignore_contents: |
|
104 for c in ignore_contents.split(","): |
|
105 content_ignore_list.append(c) |
|
106 |
|
107 # Begin work... |
|
108 print("Opening file...") |
|
109 json_file = open(path,'rb') |
|
110 print("Loading datas...") |
|
111 data = json.load(json_file) |
|
112 print("%d objects found..." % len(data)) |
|
113 content_pk_id = {} |
|
114 project_pk_id = {} |
|
115 # datas for file 1 : users, medias, contents, projects |
|
116 data_file1 = [] |
|
117 # datas for file 2 : guardian permissions |
|
118 data_file2 = [] |
|
119 # users |
|
120 usernames = [] |
|
121 for obj in data: |
|
122 if "model" in obj: |
|
123 m = obj["model"] |
|
124 if m!="guardian.userobjectpermission" and m!="guardian.groupobjectpermission": |
|
125 # We remove user admin, user AnonymousUser, group everyone and users and contents in ignore list |
|
126 # (a bit fuzzy for media and src but good for others) |
|
127 if not ((m=="auth.user" and "username" in obj["fields"] and obj["fields"]["username"] in user_ignore_list) or \ |
|
128 (m=="auth.group" and "name" in obj["fields"] and obj["fields"]["name"] in group_ignore_list) or \ |
|
129 (m=="ldt_utils.media" and "src" in obj["fields"] and any((s+".") in obj["fields"]["src"] for s in content_ignore_list)) or \ |
|
130 (m=="ldt_utils.content" and "iri_id" in obj["fields"] and obj["fields"]["iri_id"] in content_ignore_list)): |
|
131 data_file1.append(obj) |
|
132 #else: |
|
133 # print("I don't keep from datas %s, pk = %s" % (m, obj["pk"])) |
|
134 if "pk" in obj: |
|
135 # For both contents and projects, we save 2 dicts [id]=pk and [pk]=id |
|
136 # It will enable to parse and replace easily the old pk by the new ones in the permission datas |
|
137 if m=="ldt_utils.project": |
|
138 pk = str(obj["pk"]) |
|
139 id = obj["fields"]["ldt_id"] |
|
140 project_pk_id[pk] = id |
|
141 elif m=="ldt_utils.content": |
|
142 pk = str(obj["pk"]) |
|
143 id = obj["fields"]["iri_id"] |
|
144 content_pk_id[pk] = id |
|
145 obj["pk"] = None |
|
146 else: |
|
147 obj["pk"] = None |
|
148 data_file2.append(obj) |
|
149 # Save usernames except AnonymousUser |
|
150 if m=="auth.user" and "username" in obj["fields"] and obj["fields"]["username"]!="AnonymousUser": |
|
151 usernames.append(obj["fields"]["username"]) |
|
152 json_file.close() |
|
153 #data_file1.append(project_pk_id) |
|
154 #data_file1.append(project_id_pk) |
|
155 #data_file1.append(content_pk_id) |
|
156 #data_file1.append(content_id_pk) |
|
157 |
|
158 # Check if import will fail with the usernames |
|
159 existing_usernames = User.objects.all().values_list("username", flat=True) |
|
160 for un in usernames: |
|
161 if un in existing_usernames and un not in user_ignore_list: |
|
162 print("import will fail with username : %s" % str(un)) |
|
163 do_import = False |
|
164 |
|
165 # Check if import will fail with the contents's iri_id |
|
166 existing_iri_ids = Content.objects.all().values_list("iri_id", flat=True) |
|
167 new_iri_ids = list(content_pk_id.values()) |
|
168 for iri_id in new_iri_ids: |
|
169 if iri_id in existing_iri_ids and iri_id not in content_ignore_list: |
|
170 print("import will fail with iri_id : %s" % str(iri_id)) |
|
171 do_import = False |
|
172 if not do_import: |
|
173 print("Add the usernames and iri_id to the ignore parameters -u and -c") |
|
174 return "" |
|
175 |
|
176 # We save the datas in a file in order to simply call loaddata |
|
177 print("Writing %s..." % path_file1) |
|
178 file1 = open(path_file1, 'w') |
|
179 json.dump(data_file1, file1, indent=2) |
|
180 file1.close() |
|
181 print("Updating permissions ids...") |
|
182 # We replace the old pk by the natural keys in the permission datas |
|
183 ignored_project_pks = [] |
|
184 ignored_content_pks = [] |
|
185 perm_data = [] |
|
186 for obj in data_file2: |
|
187 type = obj["fields"]["content_type"][1] |
|
188 old_pk = obj["fields"]["object_pk"] |
|
189 if type=="project": |
|
190 try: |
|
191 obj["fields"]["object_pk"] = project_pk_id[old_pk] |
|
192 except: |
|
193 # The dumpdata can contain permissions for removed projects |
|
194 ignored_project_pks.append(old_pk) |
|
195 continue |
|
196 # Keeping only valuables objs avoids errors when we we get the new pks |
|
197 perm_data.append(obj) |
|
198 elif type == "content": |
|
199 try: |
|
200 obj["fields"]["object_pk"] = content_pk_id[old_pk] |
|
201 except: |
|
202 # The dumpdata can contain permissions for removed contents |
|
203 ignored_content_pks.append(old_pk) |
|
204 continue |
|
205 # Keeping only valuables objs avoids errors when we we get the new pks |
|
206 obj_id = obj["fields"]["object_pk"] |
|
207 model = obj["model"] # "guardian.groupobjectpermission" or "guardian.userobjectpermission" |
|
208 if obj_id in content_ignore_list: |
|
209 if model=="guardian.groupobjectpermission": |
|
210 if obj["fields"]["group"][0] in group_ignore_list: |
|
211 #print("permissions : j'ignore %s pour le groupe %s ..." % (obj_id, obj["fields"]["group"][0])) |
|
212 continue |
|
213 elif model=="guardian.userobjectpermission": |
|
214 if obj["fields"]["user"][0] in user_ignore_list: |
|
215 #print("permissions : j'ignore %s pour le user %s ..." % (obj_id, obj["fields"]["user"][0])) |
|
216 continue |
|
217 perm_data.append(obj) |
|
218 # We inform the user |
|
219 print("%d project permissions were ignored because projects do not exist in the current datas." % len(ignored_project_pks)) |
|
220 print("%d content permissions were ignored because contents do not exist in the current datas." % len(ignored_content_pks)) |
|
221 print("Loading datas from temporary file %s ..." % path_file1) |
|
222 # Loaddata from file 1 |
|
223 call_command("loaddata", path_file1) |
|
224 |
|
225 # Now users, medias, contents, projects have been saved. |
|
226 # We can get the new pk for contents and projects |
|
227 # Careful: in Python 3, dict.copy().values() will be prefered to list(dict.values()) |
|
228 # We use select_related("media_obj") because it will usefull with the new group |
|
229 contents = Content.objects.filter(iri_id__in=list(content_pk_id.values())).select_related("media_obj")#.values('pk', 'iri_id') |
|
230 content_id_pk = {} |
|
231 for c in contents: |
|
232 content_id_pk[c.iri_id] = str(c.pk) |
|
233 projects = Project.objects.filter(ldt_id__in=list(project_pk_id.values())).values('pk', 'ldt_id') |
|
234 project_id_pk = {} |
|
235 for p in projects: |
|
236 project_id_pk[p["ldt_id"]] = str(p["pk"]) |
|
237 |
|
238 # Now we reparse the perm_data and update with the new pks |
|
239 for obj in perm_data: |
|
240 type = obj["fields"]["content_type"][1] |
|
241 obj_id = obj["fields"]["object_pk"] |
|
242 if type=="project": |
|
243 obj["fields"]["object_pk"] = project_id_pk[obj_id] |
|
244 elif type == "content": |
|
245 obj["fields"]["object_pk"] = content_id_pk[obj_id] |
|
246 |
|
247 |
|
248 # We save the datas in a file in order to simply call loaddata |
|
249 print("Writing %s..." % path_file2) |
|
250 file2 = open(path_file2, 'w') |
|
251 json.dump(perm_data, file2, indent=2) |
|
252 file2.close() |
|
253 print("Loading permissions from temporary file %s ..." % path_file2) |
|
254 call_command("loaddata", path_file2) |
|
255 |
|
256 # Remove temp files |
|
257 print("Removing temporary files...") |
|
258 try: |
|
259 os.remove(path_file1) |
|
260 except: |
|
261 print("Removing temporary files %s failed" % path_file1) |
|
262 try: |
|
263 os.remove(path_file2) |
|
264 except: |
|
265 print("Removing temporary files %s failed" % path_file2) |
|
266 |
|
267 # Now that all datas have been imported we can create the new group and assign permissions if asked |
|
268 new_group = options.get('new_group', None) |
|
269 if new_group and len(usernames)>0: |
|
270 print("Set view permissions for the new group %s ..." % new_group) |
|
271 # Get or create group |
|
272 new_grp, _ = Group.objects.get_or_create(name=new_group) |
|
273 # Add users to the group |
|
274 users = User.objects.filter(username__in=usernames) |
|
275 for u in users: |
|
276 new_grp.user_set.add(u) |
|
277 # Get all contents and medias |
|
278 for c in contents: |
|
279 cached_assign('view_content', new_grp, c) |
|
280 cached_assign('view_media', new_grp, c.media_obj) |
|
281 |
|
282 print("Indexing imported projects ...") |
|
283 call_command('reindex', projects=True, no_content=True) |
|
284 |
|
285 # This is the end |
|
286 print("This is the end") |
|
287 |
|
288 |