--- a/web/lib/django/core/management/commands/makemessages.py Wed May 19 17:43:59 2010 +0200
+++ b/web/lib/django/core/management/commands/makemessages.py Tue May 25 02:43:45 2010 +0200
@@ -1,23 +1,17 @@
+import fnmatch
+import glob
+import os
import re
-import os
import sys
-import glob
-import warnings
from itertools import dropwhile
from optparse import make_option
+from subprocess import PIPE, Popen
from django.core.management.base import CommandError, BaseCommand
-
-try:
- set
-except NameError:
- from sets import Set as set # For Python 2.3
+from django.utils.text import get_text_list
-# Intentionally silence DeprecationWarnings about os.popen3 in Python 2.6. It's
-# still sensible for us to use it, since subprocess didn't exist in 2.3.
-warnings.filterwarnings('ignore', category=DeprecationWarning, message=r'os\.popen3')
-
-pythonize_re = re.compile(r'\n\s*//')
+pythonize_re = re.compile(r'(?:^|\n)\s*//')
+plural_forms_re = re.compile(r'^(?P<value>"Plural-Forms.+?\\n")\s*$', re.MULTILINE | re.DOTALL)
def handle_extensions(extensions=('html',)):
"""
@@ -44,7 +38,84 @@
# trick xgettext to parse them as Python files)
return set([x for x in ext_list if x != '.py'])
-def make_messages(locale=None, domain='django', verbosity='1', all=False, extensions=None):
+def _popen(cmd):
+ """
+ Friendly wrapper around Popen for Windows
+ """
+ p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, close_fds=os.name != 'nt', universal_newlines=True)
+ return p.communicate()
+
+def walk(root, topdown=True, onerror=None, followlinks=False):
+ """
+ A version of os.walk that can follow symlinks for Python < 2.6
+ """
+ for dirpath, dirnames, filenames in os.walk(root, topdown, onerror):
+ yield (dirpath, dirnames, filenames)
+ if followlinks:
+ for d in dirnames:
+ p = os.path.join(dirpath, d)
+ if os.path.islink(p):
+ for link_dirpath, link_dirnames, link_filenames in walk(p):
+ yield (link_dirpath, link_dirnames, link_filenames)
+
+def is_ignored(path, ignore_patterns):
+ """
+ Helper function to check if the given path should be ignored or not.
+ """
+ for pattern in ignore_patterns:
+ if fnmatch.fnmatchcase(path, pattern):
+ return True
+ return False
+
+def find_files(root, ignore_patterns, verbosity, symlinks=False):
+ """
+ Helper function to get all files in the given root.
+ """
+ all_files = []
+ for (dirpath, dirnames, filenames) in walk(".", followlinks=symlinks):
+ for f in filenames:
+ norm_filepath = os.path.normpath(os.path.join(dirpath, f))
+ if is_ignored(norm_filepath, ignore_patterns):
+ if verbosity > 1:
+ sys.stdout.write('ignoring file %s in %s\n' % (f, dirpath))
+ else:
+ all_files.extend([(dirpath, f)])
+ all_files.sort()
+ return all_files
+
+def copy_plural_forms(msgs, locale, domain, verbosity):
+ """
+ Copies plural forms header contents from a Django catalog of locale to
+ the msgs string, inserting it at the right place. msgs should be the
+ contents of a newly created .po file.
+ """
+ import django
+ django_dir = os.path.normpath(os.path.join(os.path.dirname(django.__file__)))
+ if domain == 'djangojs':
+ domains = ('djangojs', 'django')
+ else:
+ domains = ('django',)
+ for domain in domains:
+ django_po = os.path.join(django_dir, 'conf', 'locale', locale, 'LC_MESSAGES', '%s.po' % domain)
+ if os.path.exists(django_po):
+ m = plural_forms_re.search(open(django_po, 'rU').read())
+ if m:
+ if verbosity > 1:
+ sys.stderr.write("copying plural forms: %s\n" % m.group('value'))
+ lines = []
+ seen = False
+ for line in msgs.split('\n'):
+ if not line and not seen:
+ line = '%s\n' % m.group('value')
+ seen = True
+ lines.append(line)
+ msgs = '\n'.join(lines)
+ break
+ return msgs
+
+
+def make_messages(locale=None, domain='django', verbosity='1', all=False,
+ extensions=None, symlinks=False, ignore_patterns=[]):
"""
Uses the locale directory from the Django SVN tree or an application/
project to process all
@@ -58,8 +129,10 @@
from django.utils.translation import templatize
+ invoked_for_django = False
if os.path.isdir(os.path.join('conf', 'locale')):
localedir = os.path.abspath(os.path.join('conf', 'locale'))
+ invoked_for_django = True
elif os.path.isdir('locale'):
localedir = os.path.abspath('locale')
else:
@@ -76,29 +149,21 @@
message = "usage: make-messages.py -l <language>\n or: make-messages.py -a\n"
raise CommandError(message)
- # xgettext versions prior to 0.15 assumed Python source files were encoded
- # in iso-8859-1, and produce utf-8 output. In the case where xgettext is
- # given utf-8 input (required for Django files with non-ASCII characters),
- # this results in a utf-8 re-encoding of the original utf-8 that needs to be
- # undone to restore the original utf-8. So we check the xgettext version
- # here once and set a flag to remember if a utf-8 decoding needs to be done
- # on xgettext's output for Python files. We default to assuming this isn't
- # necessary if we run into any trouble determining the version.
- xgettext_reencodes_utf8 = False
- (stdin, stdout, stderr) = os.popen3('xgettext --version', 't')
- match = re.search(r'(?P<major>\d+)\.(?P<minor>\d+)', stdout.read())
+ # We require gettext version 0.15 or newer.
+ output = _popen('xgettext --version')[0]
+ match = re.search(r'(?P<major>\d+)\.(?P<minor>\d+)', output)
if match:
xversion = (int(match.group('major')), int(match.group('minor')))
if xversion < (0, 15):
- xgettext_reencodes_utf8 = True
-
+ raise CommandError("Django internationalization requires GNU gettext 0.15 or newer. You are using version %s, please upgrade your gettext toolset." % match.group())
+
languages = []
if locale is not None:
languages.append(locale)
elif all:
- locale_dirs = filter(os.path.isdir, glob.glob('%s/*' % localedir))
+ locale_dirs = filter(os.path.isdir, glob.glob('%s/*' % localedir))
languages = [os.path.basename(l) for l in locale_dirs]
-
+
for locale in languages:
if verbosity > 0:
print "processing language", locale
@@ -112,23 +177,21 @@
if os.path.exists(potfile):
os.unlink(potfile)
- all_files = []
- for (dirpath, dirnames, filenames) in os.walk("."):
- all_files.extend([(dirpath, f) for f in filenames])
- all_files.sort()
- for dirpath, file in all_files:
+ for dirpath, file in find_files(".", ignore_patterns, verbosity, symlinks=symlinks):
file_base, file_ext = os.path.splitext(file)
- if domain == 'djangojs' and file_ext == '.js':
+ if domain == 'djangojs' and file_ext in extensions:
if verbosity > 1:
sys.stdout.write('processing file %s in %s\n' % (file, dirpath))
src = open(os.path.join(dirpath, file), "rU").read()
src = pythonize_re.sub('\n#', src)
thefile = '%s.py' % file
- open(os.path.join(dirpath, thefile), "w").write(src)
+ f = open(os.path.join(dirpath, thefile), "w")
+ try:
+ f.write(src)
+ finally:
+ f.close()
cmd = 'xgettext -d %s -L Perl --keyword=gettext_noop --keyword=gettext_lazy --keyword=ngettext_lazy:1,2 --from-code UTF-8 -o - "%s"' % (domain, os.path.join(dirpath, thefile))
- (stdin, stdout, stderr) = os.popen3(cmd, 't')
- msgs = stdout.read()
- errors = stderr.read()
+ msgs, errors = _popen(cmd)
if errors:
raise CommandError("errors happened while running xgettext on %s\n%s" % (file, errors))
old = '#: '+os.path.join(dirpath, thefile)[2:]
@@ -140,7 +203,11 @@
else:
msgs = msgs.replace('charset=CHARSET', 'charset=UTF-8')
if msgs:
- open(potfile, 'ab').write(msgs)
+ f = open(potfile, 'ab')
+ try:
+ f.write(msgs)
+ finally:
+ f.close()
os.unlink(os.path.join(dirpath, thefile))
elif domain == 'django' and (file_ext == '.py' or file_ext in extensions):
thefile = file
@@ -148,7 +215,11 @@
src = open(os.path.join(dirpath, file), "rU").read()
thefile = '%s.py' % file
try:
- open(os.path.join(dirpath, thefile), "w").write(templatize(src))
+ f = open(os.path.join(dirpath, thefile), "w")
+ try:
+ f.write(templatize(src))
+ finally:
+ f.close()
except SyntaxError, msg:
msg = "%s (file: %s)" % (msg, os.path.join(dirpath, file))
raise SyntaxError(msg)
@@ -156,15 +227,10 @@
sys.stdout.write('processing file %s in %s\n' % (file, dirpath))
cmd = 'xgettext -d %s -L Python --keyword=gettext_noop --keyword=gettext_lazy --keyword=ngettext_lazy:1,2 --keyword=ugettext_noop --keyword=ugettext_lazy --keyword=ungettext_lazy:1,2 --from-code UTF-8 -o - "%s"' % (
domain, os.path.join(dirpath, thefile))
- (stdin, stdout, stderr) = os.popen3(cmd, 't')
- msgs = stdout.read()
- errors = stderr.read()
+ msgs, errors = _popen(cmd)
if errors:
raise CommandError("errors happened while running xgettext on %s\n%s" % (file, errors))
- if xgettext_reencodes_utf8:
- msgs = msgs.decode('utf-8').encode('iso-8859-1')
-
if thefile != file:
old = '#: '+os.path.join(dirpath, thefile)[2:]
new = '#: '+os.path.join(dirpath, file)[2:]
@@ -175,24 +241,34 @@
else:
msgs = msgs.replace('charset=CHARSET', 'charset=UTF-8')
if msgs:
- open(potfile, 'ab').write(msgs)
+ f = open(potfile, 'ab')
+ try:
+ f.write(msgs)
+ finally:
+ f.close()
if thefile != file:
os.unlink(os.path.join(dirpath, thefile))
if os.path.exists(potfile):
- (stdin, stdout, stderr) = os.popen3('msguniq --to-code=utf-8 "%s"' % potfile, 't')
- msgs = stdout.read()
- errors = stderr.read()
+ msgs, errors = _popen('msguniq --to-code=utf-8 "%s"' % potfile)
if errors:
raise CommandError("errors happened while running msguniq\n%s" % errors)
- open(potfile, 'w').write(msgs)
+ f = open(potfile, 'w')
+ try:
+ f.write(msgs)
+ finally:
+ f.close()
if os.path.exists(pofile):
- (stdin, stdout, stderr) = os.popen3('msgmerge -q "%s" "%s"' % (pofile, potfile), 't')
- msgs = stdout.read()
- errors = stderr.read()
+ msgs, errors = _popen('msgmerge -q "%s" "%s"' % (pofile, potfile))
if errors:
raise CommandError("errors happened while running msgmerge\n%s" % errors)
- open(pofile, 'wb').write(msgs)
+ elif not invoked_for_django:
+ msgs = copy_plural_forms(msgs, locale, domain, verbosity)
+ f = open(pofile, 'wb')
+ try:
+ f.write(msgs)
+ finally:
+ f.close()
os.unlink(potfile)
@@ -207,6 +283,12 @@
make_option('--extension', '-e', dest='extensions',
help='The file extension(s) to examine (default: ".html", separate multiple extensions with commas, or use -e multiple times)',
action='append'),
+ make_option('--symlinks', '-s', action='store_true', dest='symlinks',
+ default=False, help='Follows symlinks to directories when examining source code and templates for translation strings.'),
+ make_option('--ignore', '-i', action='append', dest='ignore_patterns',
+ default=[], metavar='PATTERN', help='Ignore files or directories matching this glob-style pattern. Use multiple times to ignore more.'),
+ make_option('--no-default-ignore', action='store_false', dest='use_default_ignore_patterns',
+ default=True, help="Don't ignore the common glob-style patterns 'CVS', '.*' and '*~'."),
)
help = "Runs over the entire source tree of the current directory and pulls out all strings marked for translation. It creates (or updates) a message file in the conf/locale (in the django tree) or locale (for project and application) directory."
@@ -221,14 +303,19 @@
domain = options.get('domain')
verbosity = int(options.get('verbosity'))
process_all = options.get('all')
- extensions = options.get('extensions') or ['html']
+ extensions = options.get('extensions')
+ symlinks = options.get('symlinks')
+ ignore_patterns = options.get('ignore_patterns')
+ if options.get('use_default_ignore_patterns'):
+ ignore_patterns += ['CVS', '.*', '*~']
+ ignore_patterns = list(set(ignore_patterns))
if domain == 'djangojs':
- extensions = []
+ extensions = handle_extensions(extensions or ['js'])
else:
- extensions = handle_extensions(extensions)
+ extensions = handle_extensions(extensions or ['html'])
- if '.js' in extensions:
- raise CommandError("JavaScript files should be examined by using the special 'djangojs' domain only.")
+ if verbosity > 1:
+ sys.stdout.write('examining files with the extensions: %s\n' % get_text_list(list(extensions), 'and'))
- make_messages(locale, domain, verbosity, process_all, extensions)
+ make_messages(locale, domain, verbosity, process_all, extensions, symlinks, ignore_patterns)