diff -r b758351d191f -r cc9b7e14412b web/lib/django/core/management/commands/makemessages.py --- a/web/lib/django/core/management/commands/makemessages.py Wed May 19 17:43:59 2010 +0200 +++ b/web/lib/django/core/management/commands/makemessages.py Tue May 25 02:43:45 2010 +0200 @@ -1,23 +1,17 @@ +import fnmatch +import glob +import os import re -import os import sys -import glob -import warnings from itertools import dropwhile from optparse import make_option +from subprocess import PIPE, Popen from django.core.management.base import CommandError, BaseCommand - -try: - set -except NameError: - from sets import Set as set # For Python 2.3 +from django.utils.text import get_text_list -# Intentionally silence DeprecationWarnings about os.popen3 in Python 2.6. It's -# still sensible for us to use it, since subprocess didn't exist in 2.3. -warnings.filterwarnings('ignore', category=DeprecationWarning, message=r'os\.popen3') - -pythonize_re = re.compile(r'\n\s*//') +pythonize_re = re.compile(r'(?:^|\n)\s*//') +plural_forms_re = re.compile(r'^(?P"Plural-Forms.+?\\n")\s*$', re.MULTILINE | re.DOTALL) def handle_extensions(extensions=('html',)): """ @@ -44,7 +38,84 @@ # trick xgettext to parse them as Python files) return set([x for x in ext_list if x != '.py']) -def make_messages(locale=None, domain='django', verbosity='1', all=False, extensions=None): +def _popen(cmd): + """ + Friendly wrapper around Popen for Windows + """ + p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, close_fds=os.name != 'nt', universal_newlines=True) + return p.communicate() + +def walk(root, topdown=True, onerror=None, followlinks=False): + """ + A version of os.walk that can follow symlinks for Python < 2.6 + """ + for dirpath, dirnames, filenames in os.walk(root, topdown, onerror): + yield (dirpath, dirnames, filenames) + if followlinks: + for d in dirnames: + p = os.path.join(dirpath, d) + if os.path.islink(p): + for link_dirpath, link_dirnames, link_filenames in walk(p): + yield (link_dirpath, link_dirnames, link_filenames) + +def is_ignored(path, ignore_patterns): + """ + Helper function to check if the given path should be ignored or not. + """ + for pattern in ignore_patterns: + if fnmatch.fnmatchcase(path, pattern): + return True + return False + +def find_files(root, ignore_patterns, verbosity, symlinks=False): + """ + Helper function to get all files in the given root. + """ + all_files = [] + for (dirpath, dirnames, filenames) in walk(".", followlinks=symlinks): + for f in filenames: + norm_filepath = os.path.normpath(os.path.join(dirpath, f)) + if is_ignored(norm_filepath, ignore_patterns): + if verbosity > 1: + sys.stdout.write('ignoring file %s in %s\n' % (f, dirpath)) + else: + all_files.extend([(dirpath, f)]) + all_files.sort() + return all_files + +def copy_plural_forms(msgs, locale, domain, verbosity): + """ + Copies plural forms header contents from a Django catalog of locale to + the msgs string, inserting it at the right place. msgs should be the + contents of a newly created .po file. + """ + import django + django_dir = os.path.normpath(os.path.join(os.path.dirname(django.__file__))) + if domain == 'djangojs': + domains = ('djangojs', 'django') + else: + domains = ('django',) + for domain in domains: + django_po = os.path.join(django_dir, 'conf', 'locale', locale, 'LC_MESSAGES', '%s.po' % domain) + if os.path.exists(django_po): + m = plural_forms_re.search(open(django_po, 'rU').read()) + if m: + if verbosity > 1: + sys.stderr.write("copying plural forms: %s\n" % m.group('value')) + lines = [] + seen = False + for line in msgs.split('\n'): + if not line and not seen: + line = '%s\n' % m.group('value') + seen = True + lines.append(line) + msgs = '\n'.join(lines) + break + return msgs + + +def make_messages(locale=None, domain='django', verbosity='1', all=False, + extensions=None, symlinks=False, ignore_patterns=[]): """ Uses the locale directory from the Django SVN tree or an application/ project to process all @@ -58,8 +129,10 @@ from django.utils.translation import templatize + invoked_for_django = False if os.path.isdir(os.path.join('conf', 'locale')): localedir = os.path.abspath(os.path.join('conf', 'locale')) + invoked_for_django = True elif os.path.isdir('locale'): localedir = os.path.abspath('locale') else: @@ -76,29 +149,21 @@ message = "usage: make-messages.py -l \n or: make-messages.py -a\n" raise CommandError(message) - # xgettext versions prior to 0.15 assumed Python source files were encoded - # in iso-8859-1, and produce utf-8 output. In the case where xgettext is - # given utf-8 input (required for Django files with non-ASCII characters), - # this results in a utf-8 re-encoding of the original utf-8 that needs to be - # undone to restore the original utf-8. So we check the xgettext version - # here once and set a flag to remember if a utf-8 decoding needs to be done - # on xgettext's output for Python files. We default to assuming this isn't - # necessary if we run into any trouble determining the version. - xgettext_reencodes_utf8 = False - (stdin, stdout, stderr) = os.popen3('xgettext --version', 't') - match = re.search(r'(?P\d+)\.(?P\d+)', stdout.read()) + # We require gettext version 0.15 or newer. + output = _popen('xgettext --version')[0] + match = re.search(r'(?P\d+)\.(?P\d+)', output) if match: xversion = (int(match.group('major')), int(match.group('minor'))) if xversion < (0, 15): - xgettext_reencodes_utf8 = True - + raise CommandError("Django internationalization requires GNU gettext 0.15 or newer. You are using version %s, please upgrade your gettext toolset." % match.group()) + languages = [] if locale is not None: languages.append(locale) elif all: - locale_dirs = filter(os.path.isdir, glob.glob('%s/*' % localedir)) + locale_dirs = filter(os.path.isdir, glob.glob('%s/*' % localedir)) languages = [os.path.basename(l) for l in locale_dirs] - + for locale in languages: if verbosity > 0: print "processing language", locale @@ -112,23 +177,21 @@ if os.path.exists(potfile): os.unlink(potfile) - all_files = [] - for (dirpath, dirnames, filenames) in os.walk("."): - all_files.extend([(dirpath, f) for f in filenames]) - all_files.sort() - for dirpath, file in all_files: + for dirpath, file in find_files(".", ignore_patterns, verbosity, symlinks=symlinks): file_base, file_ext = os.path.splitext(file) - if domain == 'djangojs' and file_ext == '.js': + if domain == 'djangojs' and file_ext in extensions: if verbosity > 1: sys.stdout.write('processing file %s in %s\n' % (file, dirpath)) src = open(os.path.join(dirpath, file), "rU").read() src = pythonize_re.sub('\n#', src) thefile = '%s.py' % file - open(os.path.join(dirpath, thefile), "w").write(src) + f = open(os.path.join(dirpath, thefile), "w") + try: + f.write(src) + finally: + f.close() cmd = 'xgettext -d %s -L Perl --keyword=gettext_noop --keyword=gettext_lazy --keyword=ngettext_lazy:1,2 --from-code UTF-8 -o - "%s"' % (domain, os.path.join(dirpath, thefile)) - (stdin, stdout, stderr) = os.popen3(cmd, 't') - msgs = stdout.read() - errors = stderr.read() + msgs, errors = _popen(cmd) if errors: raise CommandError("errors happened while running xgettext on %s\n%s" % (file, errors)) old = '#: '+os.path.join(dirpath, thefile)[2:] @@ -140,7 +203,11 @@ else: msgs = msgs.replace('charset=CHARSET', 'charset=UTF-8') if msgs: - open(potfile, 'ab').write(msgs) + f = open(potfile, 'ab') + try: + f.write(msgs) + finally: + f.close() os.unlink(os.path.join(dirpath, thefile)) elif domain == 'django' and (file_ext == '.py' or file_ext in extensions): thefile = file @@ -148,7 +215,11 @@ src = open(os.path.join(dirpath, file), "rU").read() thefile = '%s.py' % file try: - open(os.path.join(dirpath, thefile), "w").write(templatize(src)) + f = open(os.path.join(dirpath, thefile), "w") + try: + f.write(templatize(src)) + finally: + f.close() except SyntaxError, msg: msg = "%s (file: %s)" % (msg, os.path.join(dirpath, file)) raise SyntaxError(msg) @@ -156,15 +227,10 @@ sys.stdout.write('processing file %s in %s\n' % (file, dirpath)) cmd = 'xgettext -d %s -L Python --keyword=gettext_noop --keyword=gettext_lazy --keyword=ngettext_lazy:1,2 --keyword=ugettext_noop --keyword=ugettext_lazy --keyword=ungettext_lazy:1,2 --from-code UTF-8 -o - "%s"' % ( domain, os.path.join(dirpath, thefile)) - (stdin, stdout, stderr) = os.popen3(cmd, 't') - msgs = stdout.read() - errors = stderr.read() + msgs, errors = _popen(cmd) if errors: raise CommandError("errors happened while running xgettext on %s\n%s" % (file, errors)) - if xgettext_reencodes_utf8: - msgs = msgs.decode('utf-8').encode('iso-8859-1') - if thefile != file: old = '#: '+os.path.join(dirpath, thefile)[2:] new = '#: '+os.path.join(dirpath, file)[2:] @@ -175,24 +241,34 @@ else: msgs = msgs.replace('charset=CHARSET', 'charset=UTF-8') if msgs: - open(potfile, 'ab').write(msgs) + f = open(potfile, 'ab') + try: + f.write(msgs) + finally: + f.close() if thefile != file: os.unlink(os.path.join(dirpath, thefile)) if os.path.exists(potfile): - (stdin, stdout, stderr) = os.popen3('msguniq --to-code=utf-8 "%s"' % potfile, 't') - msgs = stdout.read() - errors = stderr.read() + msgs, errors = _popen('msguniq --to-code=utf-8 "%s"' % potfile) if errors: raise CommandError("errors happened while running msguniq\n%s" % errors) - open(potfile, 'w').write(msgs) + f = open(potfile, 'w') + try: + f.write(msgs) + finally: + f.close() if os.path.exists(pofile): - (stdin, stdout, stderr) = os.popen3('msgmerge -q "%s" "%s"' % (pofile, potfile), 't') - msgs = stdout.read() - errors = stderr.read() + msgs, errors = _popen('msgmerge -q "%s" "%s"' % (pofile, potfile)) if errors: raise CommandError("errors happened while running msgmerge\n%s" % errors) - open(pofile, 'wb').write(msgs) + elif not invoked_for_django: + msgs = copy_plural_forms(msgs, locale, domain, verbosity) + f = open(pofile, 'wb') + try: + f.write(msgs) + finally: + f.close() os.unlink(potfile) @@ -207,6 +283,12 @@ make_option('--extension', '-e', dest='extensions', help='The file extension(s) to examine (default: ".html", separate multiple extensions with commas, or use -e multiple times)', action='append'), + make_option('--symlinks', '-s', action='store_true', dest='symlinks', + default=False, help='Follows symlinks to directories when examining source code and templates for translation strings.'), + make_option('--ignore', '-i', action='append', dest='ignore_patterns', + default=[], metavar='PATTERN', help='Ignore files or directories matching this glob-style pattern. Use multiple times to ignore more.'), + make_option('--no-default-ignore', action='store_false', dest='use_default_ignore_patterns', + default=True, help="Don't ignore the common glob-style patterns 'CVS', '.*' and '*~'."), ) help = "Runs over the entire source tree of the current directory and pulls out all strings marked for translation. It creates (or updates) a message file in the conf/locale (in the django tree) or locale (for project and application) directory." @@ -221,14 +303,19 @@ domain = options.get('domain') verbosity = int(options.get('verbosity')) process_all = options.get('all') - extensions = options.get('extensions') or ['html'] + extensions = options.get('extensions') + symlinks = options.get('symlinks') + ignore_patterns = options.get('ignore_patterns') + if options.get('use_default_ignore_patterns'): + ignore_patterns += ['CVS', '.*', '*~'] + ignore_patterns = list(set(ignore_patterns)) if domain == 'djangojs': - extensions = [] + extensions = handle_extensions(extensions or ['js']) else: - extensions = handle_extensions(extensions) + extensions = handle_extensions(extensions or ['html']) - if '.js' in extensions: - raise CommandError("JavaScript files should be examined by using the special 'djangojs' domain only.") + if verbosity > 1: + sys.stdout.write('examining files with the extensions: %s\n' % get_text_list(list(extensions), 'and')) - make_messages(locale, domain, verbosity, process_all, extensions) + make_messages(locale, domain, verbosity, process_all, extensions, symlinks, ignore_patterns)