From 7342d2831829d5550965dfd18ef56998bf20665e Mon Sep 17 00:00:00 2001 From: jsa Date: Tue, 15 Mar 2016 15:08:09 -0400 Subject: [PATCH] add resizing image field for program banner images ECOM-3196 --- .gitignore | 4 +- programs/apps/api/serializers.py | 14 +- programs/apps/api/v1/tests/test_views.py | 52 ++++- programs/apps/core/s3utils.py | 13 ++ programs/apps/core/tests/test_s3utils.py | 23 ++ programs/apps/programs/admin.py | 2 +- programs/apps/programs/fields.py | 206 ++++++++++++++++ programs/apps/programs/image_helpers.py | 219 ++++++++++++++++++ .../migrations/0007_auto_20160318_1859.py | 26 +++ programs/apps/programs/models.py | 16 ++ programs/apps/programs/tests/helpers.py | 62 +++++ programs/apps/programs/tests/test_fields.py | 169 ++++++++++++++ .../apps/programs/tests/test_image_helpers.py | 162 +++++++++++++ programs/settings/production.py | 7 + programs/urls.py | 3 +- requirements/base.txt | 4 + 16 files changed, 976 insertions(+), 6 deletions(-) create mode 100644 programs/apps/core/s3utils.py create mode 100644 programs/apps/core/tests/test_s3utils.py create mode 100644 programs/apps/programs/fields.py create mode 100644 programs/apps/programs/image_helpers.py create mode 100644 programs/apps/programs/migrations/0007_auto_20160318_1859.py create mode 100644 programs/apps/programs/tests/helpers.py create mode 100644 programs/apps/programs/tests/test_fields.py create mode 100644 programs/apps/programs/tests/test_image_helpers.py diff --git a/.gitignore b/.gitignore index bec4182..58a158e 100644 --- a/.gitignore +++ b/.gitignore @@ -73,8 +73,9 @@ coverage/ override.cfg private.py -# JetBrains +# IDEs .idea +.ropeproject # OS X .DS_Store @@ -87,3 +88,4 @@ private.py docs/_build/ node_modules/ programs/static/bower_components/ +programs/media/ diff --git a/programs/apps/api/serializers.py b/programs/apps/api/serializers.py index fdbe44f..6259fcc 100644 --- a/programs/apps/api/serializers.py +++ b/programs/apps/api/serializers.py @@ -261,13 +261,23 @@ class Meta(object): # pylint: disable=missing-docstring model = models.Program fields = ( 'id', 'name', 'subtitle', 'category', 'status', 'marketing_slug', 'organizations', 'course_codes', - 'created', 'modified' + 'created', 'modified', 'banner_image_urls' ) - read_only_fields = ('id', 'created', 'modified') + read_only_fields = ('id', 'created', 'modified', 'banner_image_urls') + banner_image_urls = serializers.SerializerMethodField() organizations = ProgramOrganizationSerializer(many=True, source='programorganization_set') course_codes = ProgramCourseCodeSerializer(many=True, source='programcoursecode_set', required=False) + def get_banner_image_urls(self, instance): + """ + Render public-facing URLs for the available banner images. + """ + url_items = instance.banner_image.resized_urls.items() + # in case MEDIA_URL does not include scheme+host, ensure that the URLs are absolute and not relative + url_items = [[size, self.context['request'].build_absolute_uri(url)] for size, url in url_items] + return {'{}x{}'.format(*size): url for size, url in url_items} + def create(self, validated_data): """ Create a Program and link it with the provided organization. diff --git a/programs/apps/api/v1/tests/test_views.py b/programs/apps/api/v1/tests/test_views.py index 4b45e7c..c2355f6 100644 --- a/programs/apps/api/v1/tests/test_views.py +++ b/programs/apps/api/v1/tests/test_views.py @@ -1,14 +1,17 @@ """ Tests for Programs API views (v1). """ +from cStringIO import StringIO import datetime import itertools import json import ddt from django.core.urlresolvers import reverse -from django.test import TestCase +from django.core.files.uploadedfile import SimpleUploadedFile +from django.test import override_settings, TestCase from mock import ANY +from PIL import Image import pytz from programs.apps.api.v1.tests.mixins import AuthClientMixin, JwtMixin @@ -188,6 +191,7 @@ def test_create(self): u"created": ANY, u"modified": ANY, u'marketing_slug': '', + u'banner_image_urls': {} } ) @@ -287,6 +291,7 @@ def test_view_admin(self, status): u"created": program.created.strftime(DRF_DATE_FORMAT), u"modified": program.modified.strftime(DRF_DATE_FORMAT), u'marketing_slug': program.marketing_slug, + u'banner_image_urls': {} } ) @@ -314,9 +319,53 @@ def test_view_learner(self, status): u"created": program.created.strftime(DRF_DATE_FORMAT), u"modified": program.modified.strftime(DRF_DATE_FORMAT), u'marketing_slug': program.marketing_slug, + u'banner_image_urls': {} } ) + def make_banner_image_file(self, name): + """ + Helper to generate values for Program.banner_image + """ + image = Image.new('RGB', (1440, 900), 'green') + sio = StringIO() + image.save(sio, format='JPEG') + return SimpleUploadedFile(name, sio.getvalue(), content_type='image/jpeg') + + def assert_correct_banner_image_urls(self, url_prepend=''): + """ + DRY test helper. Ensure that the serializer generates a complete set + of absolute URLs for the banner_image when one is set. + """ + program = ProgramFactory.create(status=ProgramStatus.ACTIVE) + program.banner_image = self.make_banner_image_file('test_filename.jpg') + program.save() + + response = self._make_request(program_id=program.id) + self.assertEqual(response.status_code, 200) + + expected_urls = { + '{}x{}'.format(*size): '{}{}__{}x{}.jpg'.format(url_prepend, program.banner_image.url, *size) + for size in program.banner_image.field.sizes + } + self.assertEqual(response.data[u'banner_image_urls'], expected_urls) + + @override_settings(MEDIA_URL='/test/media/url/') + def test_banner_image_urls(self): + """ + Ensure that the request is used to generate absolute URLs for banner + images, when MEDIA_ROOT does not specify an explicit host. + """ + self.assert_correct_banner_image_urls(url_prepend='http://testserver') + + @override_settings(MEDIA_URL='https://example.com/test/media/url/') + def test_banner_image_urls_with_absolute_media_url(self): + """ + Ensure that banner image URLs are correctly presented when storage + is configured to use absolute URLs. + """ + self.assert_correct_banner_image_urls() + def test_view_with_nested(self): """ Ensure that nested serializers are working in program detail views. @@ -380,6 +429,7 @@ def test_view_with_nested(self): u"created": program.created.strftime(DRF_DATE_FORMAT), u"modified": program.modified.strftime(DRF_DATE_FORMAT), u'marketing_slug': program.marketing_slug, + u'banner_image_urls': {} } ) diff --git a/programs/apps/core/s3utils.py b/programs/apps/core/s3utils.py new file mode 100644 index 0000000..27dadc9 --- /dev/null +++ b/programs/apps/core/s3utils.py @@ -0,0 +1,13 @@ +""" +Custom S3 storage backends. +""" +from functools import partial + +from django.conf import settings +from storages.backends.s3boto import S3BotoStorage + + +MediaS3BotoStorage = partial( + S3BotoStorage, + location=settings.MEDIA_ROOT.strip('/') +) diff --git a/programs/apps/core/tests/test_s3utils.py b/programs/apps/core/tests/test_s3utils.py new file mode 100644 index 0000000..178170a --- /dev/null +++ b/programs/apps/core/tests/test_s3utils.py @@ -0,0 +1,23 @@ +""" +Test s3 utilities +""" +from django.test import TestCase +from django.conf import settings +from storages.backends.s3boto import S3BotoStorage + +from programs.apps.core.s3utils import MediaS3BotoStorage + + +class MediaS3BotoStorageTestCase(TestCase): + """ + Test the MediaS3BotoStorage django storage driver + """ + + def test_storage_init(self): + """ + The object is just a partial to S3BotoStorage from django-storages, + with some settings piped in. Ensure this works as expected. + """ + storage = MediaS3BotoStorage() + self.assertIsInstance(storage, S3BotoStorage) + self.assertEqual(storage.location, settings.MEDIA_ROOT.strip('/')) diff --git a/programs/apps/programs/admin.py b/programs/apps/programs/admin.py index 940140a..8679e4e 100644 --- a/programs/apps/programs/admin.py +++ b/programs/apps/programs/admin.py @@ -27,7 +27,7 @@ class ProgramAdmin(admin.ModelAdmin): list_display = ('name', 'status', 'category') list_filter = ('status', 'category') search_fields = ('name',) - fields = ('name', 'category', 'status', 'subtitle', 'marketing_slug',) + fields = ('name', 'category', 'status', 'subtitle', 'marketing_slug', 'banner_image') inlines = (ProgramOrganizationInline,) diff --git a/programs/apps/programs/fields.py b/programs/apps/programs/fields.py new file mode 100644 index 0000000..1e09095 --- /dev/null +++ b/programs/apps/programs/fields.py @@ -0,0 +1,206 @@ +""" +Custom fields used in models in the programs django app. +""" +from contextlib import closing + +from django.db import models +from django.db.models.fields.files import ImageFieldFile +from PIL import Image + +from .image_helpers import ( + create_image_file, + crop_image_to_aspect_ratio, + scale_image, + set_color_mode_to_rgb, + validate_image_size, + validate_image_type, +) + + +class ResizingImageFieldFile(ImageFieldFile): + """ + Custom field value behavior for images stored in `ResizingImageField` + fields on model instances. See `ResizingImageField` docs for more info. + """ + @property + def resized_names(self): + """ + Return the names of the resized copies of this image (if any), in a + dictionary keyed by tuples of (width, height). + + Returns: + dict + """ + if not self.name: + return {} + else: + return { + (width, height): '{}__{}x{}.jpg'.format(self.name, width, height) + for width, height in self.field.sizes + } + + @property + def resized_urls(self): + """ + Return the URLs of the resized copies of this image (if any), in a + dictionary keyed by tuples of (width, height). + + Returns: + dict + """ + if not self.name: + return {} + else: + return {size: self.storage.url(name) for size, name in self.resized_names.items()} + + @property + def minimum_original_size(self): + """ + Return the minimum acceptable width and height of an uploaded image + (which is the same as the greatest (width, height) pair in + self.field.sizes). + + Returns: + tuple(int, int) + """ + return sorted(self.field.sizes)[-1] + + def create_resized_copies(self): + """ + Generate and store resized copies of the original image, using the + django storage API. + + Returns: + None + """ + original = Image.open(self.file) + image = set_color_mode_to_rgb(original) + ref_width, ref_height = self.minimum_original_size + image = crop_image_to_aspect_ratio(image, float(ref_width) / float(ref_height)) + + for size, name in self.resized_names.items(): + scaled = scale_image(image, *size) + with closing(create_image_file(scaled)) as scaled_image_file: + self.storage.save(name, scaled_image_file) + + +class ResizingImageField(models.ImageField): + """ + Customized ImageField that automatically generates and stores a set of + resized copies along with the original image files. + + WARNING: this does not presently correct for orientation - processed images + taken directly from digital cameras may appear with unexpected rotation. + + TODO: purge stale copies. + """ + attr_class = ResizingImageFieldFile + + def __init__(self, path_template, sizes, *a, **kw): + """ + Arguments: + + path_template (basestring): + A format string that will be templated against model + instances to produce a directory name for stored files. + For example, if your model has a unique "name" field, you + could use '/mymodel/{name}/' as the path template. + + To facilitate management/cleanup of stale copies, it's + important to use a template that will result in a unique and + immutable value for each model object. + + Note that using the primary key ('id') in your template is + dangerous, however, because this will evaluate to `None` when + initially storing a new model instance which has not yet been + assigned an id by the database. Therefore, choose a value or + values which can be assigned before the object is physically + saved, for example a UUID or an application-generated timestamp. + + sizes: + A sequence of tuples of (width, height) at which to resize + copies of the original image. + + The largest of the sizes will be used as the minimum allowed + dimensions of a newly-stored file. + + WARNING: presently, all of the sizes must have the same aspect + ratio. + """ + if callable(kw.get('upload_to')): + # if an upload_to kwarg is passed with a callable value, the + # superclass will use it to overwrite the value of + # self.generate_filename (which is redefined below). + # Since that will lead to unexpected behavior, prevent it from + # happening. + raise Exception( + 'ResizingImageField does not support passing a custom callable ' + 'for the `upload_to` keyword arg.' + ) + super(ResizingImageField, self).__init__(*a, **kw) + self.path_template = path_template.rstrip('/') + self.sizes = sizes + + def generate_filename(self, model_instance, filename): # pylint: disable=method-hidden + """ + Join our path template with the filename assigned by django storage to + generate a filename for newly-uploaded file. + + Arguments: + + model_instance (Model): + The model instance whose value is about to be saved. + + filename (basestring): + The filename assigned to a newly uploaded file by django. + + Returns: + + ResizingImageFieldFile + + """ + pathname = self.path_template.format(**model_instance.__dict__) # pylint: disable=no-member + return '{}/{}'.format(pathname, filename) + + def pre_save(self, model_instance, add): + """ + Override pre_save to create resized copies of the original upload + when necessary (i.e. a newly stored file). + + Arguments: + + model_instance (Model): + The model instance whose value is about to be saved. + + add (bool): + Whether the model instance is being added (inserted) for the + first time. + + Returns: + + ResizingImageFieldFile + + """ + # before invoking super, determine if we are dealing with a file that has previously been saved to storage. + # we have to check this before calling super since that will store a new file and set _committed to True. + original_field_value = getattr(model_instance, self.attname) + originally_committed = getattr(original_field_value, '_committed', False) + + field_value = super(ResizingImageField, self).pre_save(model_instance, add) + + # if we just stored a new file, do additional validation, then generate and save resized copies. + if not originally_committed: + validate_image_type(field_value.file) + validate_image_size(field_value.file, *field_value.minimum_original_size) + field_value.create_resized_copies() + + return field_value + + def deconstruct(self): + """ + Provide instantiation metadata for the migrations framework. + """ + name, path, args, kwargs = super(ResizingImageField, self).deconstruct() + kwargs['sizes'] = self.sizes + kwargs['path_template'] = self.path_template + return name, path, args, kwargs diff --git a/programs/apps/programs/image_helpers.py b/programs/apps/programs/image_helpers.py new file mode 100644 index 0000000..bab5187 --- /dev/null +++ b/programs/apps/programs/image_helpers.py @@ -0,0 +1,219 @@ +""" +Image file manipulation functions. + +This module was based heavily on +edx-platform/openedx/core/djangoapps/profile_images/images.py, but needed +further generalization to provide correct functionality in this app. +""" +from collections import namedtuple +from cStringIO import StringIO + +from django.core.exceptions import ValidationError +from django.core.files.base import ContentFile +from django.utils.translation import ugettext as _ +from PIL import Image + + +class ImageValidationError(ValidationError): # pylint: disable=missing-docstring + pass + + +ImageType = namedtuple('ImageType', ('extensions', 'mimetypes', 'file_signatures')) + +IMAGE_TYPES = { + 'jpeg': ImageType( + extensions=['.jpeg', '.jpg'], + mimetypes=['image/jpeg', 'image/pjpeg'], + file_signatures=['ffd8'], + ), + 'png': ImageType( + extensions=[".png"], + mimetypes=['image/png'], + file_signatures=["89504e470d0a1a0a"], + ), + 'gif': ImageType( + extensions=[".gif"], + mimetypes=['image/gif'], + file_signatures=["474946383961", "474946383761"], + ), +} + + +def validate_image_type(uploaded_file): + """ + Raises ImageValidationError if the server should refuse to use this + uploaded file based on its apparent type/metadata. Otherwise, returns + nothing. + + Arguments: + + uploaded_file (UploadedFile): A user-supplied image file + + Returns: + None + + Raises: + ImageValidationError: + when the image file is an unsupported or invalid type. + + Note: + Based on original code by @pmitros, adapted from https://github.com/pmitros/ProfileXBlock + + See Also: + http://en.wikipedia.org/wiki/Magic_number_%28programming%29 + https://en.wikipedia.org/wiki/List_of_file_signatures + """ + uploaded_file.seek(0) + + # check the file extension looks acceptable + filename = unicode(uploaded_file.name).lower() + filetypes = [ + filetype for filetype, imagetype in IMAGE_TYPES.items() + if any(filename.endswith(ext) for ext in imagetype.extensions) + ] + if not filetypes: + file_upload_bad_type = _( + u'The file must be one of the following types: {valid_file_types}.' + ).format(valid_file_types=_get_valid_file_types()) + raise ImageValidationError(file_upload_bad_type) + image_type = IMAGE_TYPES[filetypes[0]] + + # check mimetype matches expected file type + if uploaded_file.content_type not in image_type.mimetypes: + file_upload_bad_mimetype = _( + u'The Content-Type header for this file does not match ' + u'the file data. The file may be corrupted.' + ) + raise ImageValidationError(file_upload_bad_mimetype) + + # check file signature matches expected file type + headers = image_type.file_signatures + if uploaded_file.read(len(headers[0]) / 2).encode('hex') not in headers: + file_upload_bad_ext = _( + u'The file name extension for this file does not match ' + u'the file data. The file may be corrupted.' + ) + raise ImageValidationError(file_upload_bad_ext) + # avoid unexpected errors from subsequent modules expecting the fp to be at 0 + uploaded_file.seek(0) + + +def validate_image_size(uploaded_file, minimum_width, minimum_height): + """ + Raises ImageValidationError if the uploaded file is not at least as wide + and tall as the specified dimensions. + + Arguments: + uploaded_file (UploadedFile): A user-supplied image file + minimum_width (int): minimum width of the image in pixels + minimum_height (int): minimum height of the image in pixels + + Returns: + None + + Raises: + ImageValidationError: + when the image file is an unsupported or invalid type. + """ + image_width, image_height = Image.open(uploaded_file).size + if image_width < minimum_width or image_height < minimum_height: + file_upload_too_small = _( + u'The file must be at least {minimum_width} pixels wide ' + u'and {minimum_height} pixels high.' + ).format(minimum_width=minimum_width, minimum_height=minimum_height) + raise ImageValidationError(file_upload_too_small) + + +def crop_image_to_aspect_ratio(image, aspect_ratio): + """ + Given a PIL.Image object, return a copy cropped horizontally around the + center and vertically from the top, using the specified aspect ratio. + + Arguments: + image (Image): a PIL.Image + aspect_ratio (float): desired aspect ratio of the cropped image. + + Returns: + Image + """ + width, height = image.size + current_ratio = float(width) / float(height) + + # defaults + left = 0 + top = 0 + right = width + bottom = height + + if current_ratio > aspect_ratio: + # image is too wide and must be cropped horizontally (from center) + new_width = height * aspect_ratio + left = (width - new_width) // 2 + right = (width + new_width) // 2 + elif current_ratio < aspect_ratio: + # image is too tall and must be cropped vertically (from top) + bottom = width // aspect_ratio + else: + # cropping will be a no-op but we'll go ahead, since we promised to + # return a copy of the input image. + pass + + image = image.crop(map(int, (left, top, right, bottom))) + return image + + +def set_color_mode_to_rgb(image): + """ + Given a PIL.Image object, return a copy with the color mode set to RGB. + + Arguments: + image (Image) + + Returns: + Image + """ + return image.convert('RGB') + + +def scale_image(image, width, height): + """ + Given a PIL.Image object, return a copy resized to the given dimensions. + + Arguments: + image (Image) + width (int) + height (int) + + Returns: + Image + """ + return image.resize((width, height), Image.ANTIALIAS) + + +def create_image_file(image): + """ + Given a PIL.Image object, create and return a file-like object containing + the data saved as a JPEG that is compatible with django's storage API. + + Note that the file object returned is a django ContentFile which holds data + in memory (not on disk). Because ContentFile does not support a `write` + call (which is required by PIL.Image.save to serialize itself), we use + StringIO as an intermediary buffer for the written data, and initialize + the ContentFile from that. + + Arguments: + image (Image) + + Returns: + ContentFile + """ + string_io = StringIO() + image.save(string_io, format='JPEG') + return ContentFile(string_io.getvalue()) + + +def _get_valid_file_types(): + """ + Return comma separated string of valid file types. + """ + return ', '.join([', '.join(IMAGE_TYPES[ft].extensions) for ft in IMAGE_TYPES]) diff --git a/programs/apps/programs/migrations/0007_auto_20160318_1859.py b/programs/apps/programs/migrations/0007_auto_20160318_1859.py new file mode 100644 index 0000000..162e7b9 --- /dev/null +++ b/programs/apps/programs/migrations/0007_auto_20160318_1859.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +import programs.apps.programs.fields +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('programs', '0006_auto_20160104_1920'), + ] + + operations = [ + migrations.AddField( + model_name='program', + name='banner_image', + field=programs.apps.programs.fields.ResizingImageField(path_template=b'program/banner/{uuid}', null=True, upload_to=b'', sizes=[(1440, 480), (480, 160), (360, 120), (180, 60)], blank=True), + ), + migrations.AddField( + model_name='program', + name='uuid', + field=models.UUIDField(default=uuid.uuid4, editable=False, blank=True), + ), + ] diff --git a/programs/apps/programs/models.py b/programs/apps/programs/models.py index 25b893a..03f56ce 100644 --- a/programs/apps/programs/models.py +++ b/programs/apps/programs/models.py @@ -1,6 +1,8 @@ """ Models for the programs app. """ +from uuid import uuid4 + from django.db import models from django.core.exceptions import ValidationError from django_extensions.db.models import TimeStampedModel @@ -9,6 +11,7 @@ from opaque_keys.edx.keys import CourseKey from programs.apps.programs import constants +from programs.apps.programs.fields import ResizingImageField def _choices(*values): @@ -23,6 +26,12 @@ class Program(TimeStampedModel): Representation of a Program. """ + uuid = models.UUIDField( + blank=True, + default=uuid4, + editable=False, + ) + name = models.CharField( help_text=_('The user-facing display name for this Program.'), max_length=255, @@ -63,6 +72,13 @@ class Program(TimeStampedModel): max_length=255 ) + banner_image = ResizingImageField( + path_template='program/banner/{uuid}', + sizes=[(1440, 480), (480, 160), (360, 120), (180, 60)], + null=True, + blank=True, + ) + def save(self, *a, **kw): """ Verify that the marketing slug is not empty if the user has attempted diff --git a/programs/apps/programs/tests/helpers.py b/programs/apps/programs/tests/helpers.py new file mode 100644 index 0000000..91f57dc --- /dev/null +++ b/programs/apps/programs/tests/helpers.py @@ -0,0 +1,62 @@ +""" +Helper methods for testing the processing of image files. + +TODO: + this module was copied (mostly) verbatim from + edx-platform/master/openedx/core/djangoapps/profile_images/tests/helpers.py + and could ultimately be moved (with related modules) into a shared utility package. +""" +from contextlib import contextmanager +import os +from tempfile import NamedTemporaryFile + +from django.core.files.uploadedfile import UploadedFile +import piexif +from PIL import Image + + +@contextmanager +def make_image_file(dimensions=(320, 240), extension=".jpeg", force_size=None, orientation=None): + """ + Yields a named temporary file created with the specified image type and + options. + + The temporary file will be closed and deleted automatically upon exiting + the `with` block. + """ + image = Image.new('RGB', dimensions, "green") + image_file = NamedTemporaryFile(suffix=extension) + try: + if orientation and orientation in xrange(1, 9): + exif_bytes = piexif.dump({'0th': {piexif.ImageIFD.Orientation: orientation}}) + image.save(image_file, exif=exif_bytes) + else: + image.save(image_file) + if force_size is not None: + image_file.seek(0, os.SEEK_END) + bytes_to_pad = force_size - image_file.tell() + # write in hunks of 256 bytes + hunk, byte_ = bytearray([0] * 256), bytearray([0]) + num_hunks, remainder = divmod(bytes_to_pad, 256) + for _ in xrange(num_hunks): + image_file.write(hunk) + for _ in xrange(remainder): + image_file.write(byte_) + image_file.flush() + image_file.seek(0) + yield image_file + finally: + image_file.close() + + +@contextmanager +def make_uploaded_file(content_type, *a, **kw): + """ + Wrap the result of make_image_file in a django UploadedFile. + """ + with make_image_file(*a, **kw) as image_file: + yield UploadedFile( + image_file, + content_type=content_type, + size=os.path.getsize(image_file.name), + ) diff --git a/programs/apps/programs/tests/test_fields.py b/programs/apps/programs/tests/test_fields.py new file mode 100644 index 0000000..a734518 --- /dev/null +++ b/programs/apps/programs/tests/test_fields.py @@ -0,0 +1,169 @@ +""" +Tests for custom fields. +""" + +from django.test import override_settings, TestCase +import ddt +import mock +from PIL import Image + +from programs.apps.programs.fields import ResizingImageField, ResizingImageFieldFile +from .helpers import make_image_file, make_uploaded_file + +TEST_SIZES = [(1, 1), (999, 999)] +PATCH_MODULE = 'programs.apps.programs.fields' + + +@override_settings(MEDIA_URL='/test/media/url/') +class ResizingImageFieldFileTestCase(TestCase): + """ + Test the behavior of values of our custom field in the context of a model + instance. + """ + # pylint: disable=too-many-function-args + # the above is because pylint doesn't seem to understand the method signature of ResizingImageFieldFile.__init__ + # TODO: figure out why this happens + + def setUp(self): + super(ResizingImageFieldFileTestCase, self).setUp() + self.model_instance = mock.Mock() + self.field = ResizingImageField('dummy', TEST_SIZES) + + def test_resized_names(self): + """ + Ensure the names for resized copies of the image are generated + correctly. + """ + field_value = ResizingImageFieldFile(self.model_instance, self.field, 'path/to/test-filename') + self.assertEqual( + field_value.resized_names, + { + (1, 1): 'path/to/test-filename__1x1.jpg', + (999, 999): 'path/to/test-filename__999x999.jpg' + } + ) + + def test_resized_names_no_file(self): + """ + Ensure the result of generating names is empty when there is no + original file. + """ + field_value = ResizingImageFieldFile(self.model_instance, self.field, None) + self.assertEqual(field_value.resized_names, {}) + + def test_resized_urls(self): + """ + Ensure the URLs for resized copies of the image are generated + correctly. + """ + field_value = ResizingImageFieldFile(self.model_instance, self.field, 'path/to/test-filename') + self.assertEqual( + field_value.resized_urls, + { + (1, 1): '/test/media/url/path/to/test-filename__1x1.jpg', + (999, 999): '/test/media/url/path/to/test-filename__999x999.jpg' + } + ) + + def test_resized_urls_no_file(self): + """ + Ensure the result of generating URLs is empty when there is no + original file. + """ + field_value = ResizingImageFieldFile(self.model_instance, self.field, None) + self.assertEqual(field_value.resized_urls, {}) + + def test_minimum_original_size(self): + """ + Ensure the minimum original size is computed correctly. + """ + field_value = ResizingImageFieldFile(self.model_instance, self.field, None) + self.assertEqual(field_value.minimum_original_size, (999, 999)) + + def test_create_resized_copies(self): + """ + Ensure the create_resized_copies function produces and stores copies + with the correct sizes and data. + """ + field_value = ResizingImageFieldFile(self.model_instance, self.field, 'test_name') + with mock.patch.object(field_value, 'storage') as mock_storage: + with make_image_file((300, 300)) as image_file: + field_value.file = image_file + field_value.create_resized_copies() + + self.assertEqual(mock_storage.save.call_count, len(TEST_SIZES)) + actual_calls = dict((v[1] for v in mock_storage.save.mock_calls)) + for width, height in TEST_SIZES: + expected_name = 'test_name__{}x{}.jpg'.format(width, height) + actual_data = actual_calls[expected_name] + image_object = Image.open(actual_data) + self.assertEqual(image_object.size, (width, height)) + + +@ddt.ddt +class ResizingImageFieldTestCase(TestCase): + """ + Test the behavior of the definition of our custom field in the context of a + model instance. + """ + def setUp(self): + super(ResizingImageFieldTestCase, self).setUp() + self.model_instance = mock.Mock(attr='test-attr') + self.field = ResizingImageField('testing/{attr}/path', TEST_SIZES) + + def test_generate_filename(self): + """ + Ensure that the path_template is used to generate filenames correctly. + """ + self.assertEqual( + self.field.generate_filename(self.model_instance, 'test-filename'), + 'testing/test-attr/path/test-filename' + ) + + @mock.patch(PATCH_MODULE + '.validate_image_type') + @mock.patch(PATCH_MODULE + '.validate_image_size') + @ddt.data( + (None, False), + ('test-filename', False), + ('test-filename', True), + ) + @ddt.unpack + def test_pre_save(self, filename, is_existing_file, mock_validate_size, mock_validate_type): + """ + Ensure that image validation and resizing take place only when a new + file is being stored. + """ + # pylint: disable=too-many-function-args + + field_value = ResizingImageFieldFile(self.model_instance, self.field, filename) + self.model_instance.resized_image = field_value + self.field.attname = 'resized_image' + self.field.name = 'resized_image' + + with mock.patch(PATCH_MODULE + '.ResizingImageFieldFile.create_resized_copies') as mock_resize: + # actual file data is needed for this test to work + with make_uploaded_file('image/jpeg', (1000, 1000)) as image_file: + if filename: + field_value.file = image_file + field_value._committed = is_existing_file # pylint: disable=protected-access + self.field.pre_save(self.model_instance, False) + + expected_called = bool(filename) and not is_existing_file + for actual_called in (mock_validate_size.called, mock_validate_type.called, mock_resize.called): + self.assertEqual(actual_called, expected_called) + + def test_upload_to(self): + """ + Ensure that the field cannot be initialized with a callable `upload_to` + as this will break the filename-generation template logic. + """ + def dummy_upload_to(instance, filename): # pylint: disable=missing-docstring, unused-argument + return 'foo' + + with self.assertRaises(Exception) as exc_context: + self.field = ResizingImageField('testing/{attr}/path', TEST_SIZES, upload_to=dummy_upload_to) + + self.assertEquals( + exc_context.exception.message, + 'ResizingImageField does not support passing a custom callable for the `upload_to` keyword arg.', + ) diff --git a/programs/apps/programs/tests/test_image_helpers.py b/programs/apps/programs/tests/test_image_helpers.py new file mode 100644 index 0000000..e89fb35 --- /dev/null +++ b/programs/apps/programs/tests/test_image_helpers.py @@ -0,0 +1,162 @@ +""" +Test cases for image processing helpers. +""" +from contextlib import closing +import os +from tempfile import NamedTemporaryFile + +from django.core.files.uploadedfile import UploadedFile +from django.test import TestCase +import ddt +from PIL import Image + +from ..image_helpers import ( + ImageValidationError, + validate_image_type, + validate_image_size, + crop_image_to_aspect_ratio, +) +from .helpers import make_image_file, make_uploaded_file + + +@ddt.ddt +class TestValidateImageType(TestCase): + """ + Test validate_image_type + """ + FILE_UPLOAD_BAD_TYPE = ( + u'The file must be one of the following types: .gif, .png, .jpeg, .jpg.' + ) + + def check_validation_result(self, uploaded_file, expected_failure_message): + """ + Internal DRY helper. + """ + if expected_failure_message is not None: + with self.assertRaises(ImageValidationError) as ctx: + validate_image_type(uploaded_file) + self.assertEqual(ctx.exception.message, expected_failure_message) + else: + validate_image_type(uploaded_file) + self.assertEqual(uploaded_file.tell(), 0) + + @ddt.data( + (".gif", "image/gif"), + (".jpg", "image/jpeg"), + (".jpeg", "image/jpeg"), + (".png", "image/png"), + (".bmp", "image/bmp", FILE_UPLOAD_BAD_TYPE), + (".tif", "image/tiff", FILE_UPLOAD_BAD_TYPE), + ) + @ddt.unpack + def test_extension(self, extension, content_type, expected_failure_message=None): + """ + Ensure that files whose extension is not supported fail validation. + """ + with make_uploaded_file(extension=extension, content_type=content_type) as uploaded_file: + self.check_validation_result(uploaded_file, expected_failure_message) + + def test_extension_mismatch(self): + """ + Ensure that validation fails when the file extension does not match the + file data. + """ + file_upload_bad_ext = ( + u'The file name extension for this file does not match ' + u'the file data. The file may be corrupted.' + ) + # make a bmp, try to fool the function into thinking it's a jpeg + with make_image_file(extension=".bmp") as bmp_file: + with closing(NamedTemporaryFile(suffix=".jpeg")) as fake_jpeg_file: + fake_jpeg_file.write(bmp_file.read()) + fake_jpeg_file.seek(0) + uploaded_file = UploadedFile( + fake_jpeg_file, + content_type="image/jpeg", + size=os.path.getsize(fake_jpeg_file.name) + ) + with self.assertRaises(ImageValidationError) as ctx: + validate_image_type(uploaded_file) + self.assertEqual(ctx.exception.message, file_upload_bad_ext) + + def test_content_type(self): + """ + Ensure that validation fails when the content_type header and file + extension do not match + """ + file_upload_bad_mimetype = ( + u'The Content-Type header for this file does not match ' + u'the file data. The file may be corrupted.' + ) + with make_uploaded_file(extension=".jpeg", content_type="image/gif") as uploaded_file: + with self.assertRaises(ImageValidationError) as ctx: + validate_image_type(uploaded_file) + self.assertEqual(ctx.exception.message, file_upload_bad_mimetype) + + +@ddt.ddt +class TestValidateImageSize(TestCase): + """ + Test validate_image_size + """ + + @ddt.data( + ((100, 200), (99, 200)), + ((100, 200), (100, 199)), + ((2, 2), (1, 2)), + ((2, 2), (2, 1)), + ((1000, 1000), (1, 1)), + ) + @ddt.unpack + def test_validate_image_size_invalid(self, required_dimensions, actual_dimensions): + + expected_message = u'The file must be at least {} pixels wide and {} pixels high.'.format(*required_dimensions) + + with make_uploaded_file("image/jpeg", actual_dimensions) as uploaded_file: + with self.assertRaises(ImageValidationError) as ctx: + validate_image_size(uploaded_file, *required_dimensions) + self.assertEqual(ctx.exception.message, expected_message) + + @ddt.data( + ((100, 200), (100, 200)), + ((100, 200), (101, 201)), + ((1, 1), (1, 1)), + ((100, 100), (1000, 1000)), + ) + @ddt.unpack + def test_validate_image_size_valid(self, required_dimensions, actual_dimensions): + + with make_uploaded_file("image/jpeg", actual_dimensions) as uploaded_file: + self.assertIsNone(validate_image_size(uploaded_file, *required_dimensions)) + + +class TestCropImageToAspectRatio(TestCase): + """ + Test crop_image_to_aspect_ratio + + TODO: this does not test where images are being cropped from, just that the + sizing math is correct. + """ + + def test_crop_image(self): + """ + Ensure the resulting cropped Image has the correct dimensions. + """ + with make_image_file((300, 200)) as image_file: + with closing(Image.open(image_file)) as image_obj: + + # reduce lesser dimension (height) to achieve 2:1 + cropped = crop_image_to_aspect_ratio(image_obj, 2) + self.assertEqual(cropped.size, (300, 150)) + + # reduce greater dimension (width) to achieve 0.5:1 + cropped = crop_image_to_aspect_ratio(image_obj, 0.5) + self.assertEqual(cropped.size, (100, 200)) + + # reduce greater dimension (width) to achieve 1:1 + cropped = crop_image_to_aspect_ratio(image_obj, 1) + self.assertEqual(cropped.size, (200, 200)) + + # no cropping necessary, aspect ratio already correct + cropped = crop_image_to_aspect_ratio(image_obj, 1.5) + self.assertEqual(cropped.size, (300, 200)) diff --git a/programs/settings/production.py b/programs/settings/production.py index 524348a..e5b4107 100644 --- a/programs/settings/production.py +++ b/programs/settings/production.py @@ -21,11 +21,18 @@ LOGGING = get_logger_config() +# This may be overridden by the yaml in PROGRAMS_CFG, but it should +# be here as a default. +MEDIA_STORAGE_BACKEND = {} + CONFIG_FILE = get_env_setting('PROGRAMS_CFG') with open(CONFIG_FILE) as f: config_from_yaml = yaml.load(f) vars().update(config_from_yaml) + # Load settings for media storage + vars().update(MEDIA_STORAGE_BACKEND) + DB_OVERRIDES = dict( PASSWORD=environ.get('DB_MIGRATION_PASS', DATABASES['default']['PASSWORD']), ENGINE=environ.get('DB_MIGRATION_ENGINE', DATABASES['default']['ENGINE']), diff --git a/programs/urls.py b/programs/urls.py index f82cc27..f96b81a 100644 --- a/programs/urls.py +++ b/programs/urls.py @@ -17,6 +17,7 @@ from django.conf import settings from django.conf.urls import include, url +from django.conf.urls.static import static from django.contrib import admin from django.contrib.auth.views import logout from django.core.urlresolvers import reverse_lazy @@ -60,7 +61,7 @@ urlpatterns += [ # Drops into the Programs authoring app, which handles its own routing. url(r'^program/*', program_view, name='program'), - ] + ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) if os.environ.get('ENABLE_DJANGO_TOOLBAR', False): import debug_toolbar # pylint: disable=import-error diff --git a/requirements/base.txt b/requirements/base.txt index cec1790..54142d2 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,8 +1,10 @@ +boto==2.39.0 django==1.8.10 django_compressor==1.5 django-cors-headers==1.1.0 django-extensions==1.5.5 django-libsass==0.4 +django-storages==1.4 django-waffle==0.10.1 djangorestframework==3.2.3 djangorestframework-jwt==1.7.2 @@ -14,4 +16,6 @@ edx-rest-api-client==1.5.0 # Pinning to 0.10.0 is a temporary workaround, but the underlying violation(s) should be fixed. libsass==0.10.0 Markdown==2.6.2 +piexif==1.0.3 +Pillow==3.1.1 pytz==2015.4