"""
Sync Media to S3
================
Django command that scans all files in your settings.MEDIA_ROOT folder and
uploads them to S3 with the same directory structure.
This command can optionally do the following but it is off by default:
* gzip compress any CSS and Javascript files it finds and adds the appropriate
'Content-Encoding' header.
* set a far future 'Expires' header for optimal caching.
Note: This script requires the Python boto library and valid Amazon Web
Services API keys.
Required settings.py variables:
AWS_ACCESS_KEY_ID = ''
AWS_SECRET_ACCESS_KEY = ''
AWS_BUCKET_NAME = ''
Command options are:
-p PREFIX, --prefix=PREFIX
The prefix to prepend to the path on S3.
--gzip Enables gzipping CSS and Javascript files.
--expires Enables setting a far future expires header.
--force Skip the file mtime check to force upload of all
files.
--filter-list Override default directory and file exclusion
filters. (enter as comma seperated line)
TODO:
* Use fnmatch (or regex) to allow more complex FILTER_LIST rules.
"""
import datetime
import email
import mimetypes
import optparse
import os
import sys
import time
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
# Make sure boto is available
try:
import boto
import boto.exception
except ImportError:
raise ImportError, "The boto Python library is not installed."
class Command(BaseCommand):
# Extra variables to avoid passing these around
AWS_ACCESS_KEY_ID = ''
AWS_SECRET_ACCESS_KEY = ''
AWS_BUCKET_NAME = ''
DIRECTORY = ''
FILTER_LIST = ['.DS_Store', '.svn', '.hg', '.git', 'Thumbs.db']
GZIP_CONTENT_TYPES = (
'text/css',
'application/javascript',
'application/x-javascript'
)
upload_count = 0
skip_count = 0
option_list = BaseCommand.option_list + (
optparse.make_option('-p', '--prefix',
dest='prefix', default='',
help="The prefix to prepend to the path on S3."),
optparse.make_option('-d', '--dir',
dest='dir', default=settings.MEDIA_ROOT,
help="The root directory to use instead of your MEDIA_ROOT"),
optparse.make_option('--gzip',
action='store_true', dest='gzip', default=False,
help="Enables gzipping CSS and Javascript files."),
optparse.make_option('--expires',
action='store_true', dest='expires', default=False,
help="Enables setting a far future expires header."),
optparse.make_option('--force',
action='store_true', dest='force', default=False,
help="Skip the file mtime check to force upload of all files."),
optparse.make_option('--filter-list', dest='filter_list',
action='store', default='',
help="Override default directory and file exclusion filters. (enter as comma seperated line)"),
)
help = 'Syncs the complete MEDIA_ROOT structure and files to S3 into the given bucket name.'
args = 'bucket_name'
can_import_settings = True
def handle(self, *args, **options):
# Check for AWS keys in settings
if not hasattr(settings, 'AWS_ACCESS_KEY_ID') or \
not hasattr(settings, 'AWS_SECRET_ACCESS_KEY'):
raise CommandError('Missing AWS keys from settings file. Please' +
'supply both AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY.')
else:
self.AWS_ACCESS_KEY_ID = settings.AWS_ACCESS_KEY_ID
self.AWS_SECRET_ACCESS_KEY = settings.AWS_SECRET_ACCESS_KEY
if not hasattr(settings, 'AWS_BUCKET_NAME'):
raise CommandError('Missing bucket name from settings file. Please' +
' add the AWS_BUCKET_NAME to your settings file.')
else:
if not settings.AWS_BUCKET_NAME:
raise CommandError('AWS_BUCKET_NAME cannot be empty.')
self.AWS_BUCKET_NAME = settings.AWS_BUCKET_NAME
if not hasattr(settings, 'MEDIA_ROOT'):
raise CommandError('MEDIA_ROOT must be set in your settings.')
else:
if not settings.MEDIA_ROOT:
raise CommandError('MEDIA_ROOT must be set in your settings.')
self.verbosity = int(options.get('verbosity'))
self.prefix = options.get('prefix')
self.do_gzip = options.get('gzip')
self.do_expires = options.get('expires')
self.do_force = options.get('force')
self.DIRECTORY = options.get('dir')
self.FILTER_LIST = getattr(settings, 'FILTER_LIST', self.FILTER_LIST)
filter_list = options.get('filter_list').split(',')
if filter_list:
# command line option overrides default filter_list and
# settings.filter_list
self.FILTER_LIST = filter_list
# Now call the syncing method to walk the MEDIA_ROOT directory and
# upload all files found.
self.sync_s3()
print
print "%d files uploaded." % (self.upload_count)
print "%d files skipped." % (self.skip_count)
def sync_s3(self):
"""
Walks the media directory and syncs files to S3
"""
bucket, key = self.open_s3()
os.path.walk(self.DIRECTORY, self.upload_s3,
(bucket, key, self.AWS_BUCKET_NAME, self.DIRECTORY))
def compress_string(self, s):
"""Gzip a given string."""
import cStringIO, gzip
zbuf = cStringIO.StringIO()
zfile = gzip.GzipFile(mode='wb', compresslevel=6, fileobj=zbuf)
zfile.write(s)
zfile.close()
return zbuf.getvalue()
def open_s3(self):
"""
Opens connection to S3 returning bucket and key
"""
conn = boto.connect_s3(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY)
try:
bucket = conn.get_bucket(self.AWS_BUCKET_NAME)
except boto.exception.S3ResponseError:
bucket = conn.create_bucket(self.AWS_BUCKET_NAME)
return bucket, boto.s3.key.Key(bucket)
def upload_s3(self, arg, dirname, names):
"""
This is the callback to os.path.walk and where much of the work happens
"""
bucket, key, bucket_name, root_dir = arg # expand arg tuple
# Skip directories we don't want to sync
if os.path.basename(dirname) in self.FILTER_LIST:
# prevent walk from processing subfiles/subdirs below the ignored one
del names[:]
return
# Later we assume the MEDIA_ROOT ends with a trailing slash
if not root_dir.endswith(os.path.sep):
root_dir = root_dir + os.path.sep
for file in names:
headers = {}
if file in self.FILTER_LIST:
continue # Skip files we don't want to sync
filename = os.path.join(dirname, file)
if os.path.isdir(filename):
continue # Don't try to upload directories
file_key = filename[len(root_dir):]
if self.prefix:
file_key = '%s/%s' % (self.prefix, file_key)
# Check if file on S3 is older than local file, if so, upload
if not self.do_force:
s3_key = bucket.get_key(file_key)
if s3_key:
s3_datetime = datetime.datetime(*time.strptime(
s3_key.last_modified, '%a, %d %b %Y %H:%M:%S %Z')[0:6])
local_datetime = datetime.datetime.utcfromtimestamp(
os.stat(filename).st_mtime)
if local_datetime < s3_datetime:
self.skip_count += 1
if self.verbosity > 1:
print "File %s hasn't been modified since last " \
"being uploaded" % (file_key)
continue
# File is newer, let's process and upload
if self.verbosity > 0:
print "Uploading %s..." % (file_key)
content_type = mimetypes.guess_type(filename)[0]
if content_type:
headers['Content-Type'] = content_type
file_obj = open(filename, 'rb')
file_size = os.fstat(file_obj.fileno()).st_size
filedata = file_obj.read()
if self.do_gzip:
# Gzipping only if file is large enough (>1K is recommended)
# and only if file is a common text type (not a binary file)
if file_size > 1024 and content_type in self.GZIP_CONTENT_TYPES:
filedata = self.compress_string(filedata)
headers['Content-Encoding'] = 'gzip'
if self.verbosity > 1:
print "\tgzipped: %dk to %dk" % \
(file_size/1024, len(filedata)/1024)
if self.do_expires:
# HTTP/1.0
headers['Expires'] = '%s GMT' % (email.Utils.formatdate(
time.mktime((datetime.datetime.now() +
datetime.timedelta(days=365*2)).timetuple())))
# HTTP/1.1
headers['Cache-Control'] = 'max-age %d' % (3600 * 24 * 365 * 2)
if self.verbosity > 1:
print "\texpires: %s" % (headers['Expires'])
print "\tcache-control: %s" % (headers['Cache-Control'])
try:
key.name = file_key
key.set_contents_from_string(filedata, headers, replace=True)
key.set_acl('public-read')
except boto.s3.connection.S3CreateError, e:
print "Failed: %s" % e
except Exception, e:
print e
raise
else:
self.upload_count += 1
file_obj.close()
# Backwards compatibility for Django r9110
if not [opt for opt in Command.option_list if opt.dest=='verbosity']:
Command.option_list += (
optparse.make_option('-v', '--verbosity',
dest='verbosity', default=1, action='count',
help="Verbose mode. Multiple -v options increase the verbosity."),
)