From fe48a318e8c4fe495710e3c648118770394a8fc9 Mon Sep 17 00:00:00 2001 From: Bryan Date: Fri, 31 Jan 2014 21:08:43 -0500 Subject: [PATCH] refactoring Filepicker usage, closes #308 --- karmaworld/apps/courses/views.py | 4 + karmaworld/apps/notes/forms.py | 14 ++ karmaworld/apps/notes/gdrive.py | 2 +- karmaworld/apps/notes/models.py | 104 ++++++---- karmaworld/secret/filepicker.py.example | 1 + karmaworld/templates/partial/filepicker.html | 194 +++++++++---------- karmaworld/utils/filepicker.py | 28 +++ reqs/common.txt | 2 +- 8 files changed, 203 insertions(+), 146 deletions(-) create mode 100644 karmaworld/utils/filepicker.py diff --git a/karmaworld/apps/courses/views.py b/karmaworld/apps/courses/views.py index bb12926..d485ea3 100644 --- a/karmaworld/apps/courses/views.py +++ b/karmaworld/apps/courses/views.py @@ -20,6 +20,7 @@ from karmaworld.apps.courses.models import Course from karmaworld.apps.courses.models import School from karmaworld.apps.notes.models import Note from karmaworld.apps.users.models import CourseKarmaEvent +from karmaworld.apps.notes.forms import FileUploadForm from karmaworld.utils import ajax_increment, format_session_increment_field FLAG_FIELD = 'flags' @@ -76,6 +77,9 @@ class CourseDetailView(DetailView): # Include "Add Note" button in header kwargs['display_add_note'] = True + # For the Filepicker Partial template + kwargs['file_upload_form'] = FileUploadForm() + if self.request.session.get(format_session_increment_field(Course, self.object.id, FLAG_FIELD), False): kwargs['already_flagged'] = True diff --git a/karmaworld/apps/notes/forms.py b/karmaworld/apps/notes/forms.py index 1207b02..0759ffb 100644 --- a/karmaworld/apps/notes/forms.py +++ b/karmaworld/apps/notes/forms.py @@ -5,6 +5,9 @@ from django.forms import ModelForm from django.forms import TextInput +from django_filepicker.forms import FPFileField +from django_filepicker.widgets import FPFileWidget + from karmaworld.apps.notes.models import Note class NoteForm(ModelForm): @@ -14,3 +17,14 @@ class NoteForm(ModelForm): widgets = { 'name': TextInput() } + +class FileUploadForm(ModelForm): + auto_id = False + class Meta: + model = Note + fields = ('fp_file',) + widgets = { + 'fp_file': FPFileWidget(attrs={ + 'id': 'filepicker-file-upload', + }), + } diff --git a/karmaworld/apps/notes/gdrive.py b/karmaworld/apps/notes/gdrive.py index 24af3d0..2870133 100644 --- a/karmaworld/apps/notes/gdrive.py +++ b/karmaworld/apps/notes/gdrive.py @@ -251,7 +251,7 @@ def convert_raw_document(raw_document, user=None): note.save() NoteKarmaEvent.create_event(mapping.user, note, NoteKarmaEvent.UPLOAD) except (ObjectDoesNotExist, MultipleObjectsReturned): - logger.info("Zero or multiple mappings found with fp_file " + raw_document.fp_file.url) + logger.info("Zero or multiple mappings found with fp_file " + raw_document.fp_file.name) # Finally, save whatever data we got back from google note.save() diff --git a/karmaworld/apps/notes/models.py b/karmaworld/apps/notes/models.py index 23007cf..5cc5097 100644 --- a/karmaworld/apps/notes/models.py +++ b/karmaworld/apps/notes/models.py @@ -11,13 +11,17 @@ import traceback import logging from allauth.account.signals import user_logged_in from django.contrib.auth.models import User +from django.utils.safestring import mark_safe from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned from django.core.files.storage import default_storage from django.db.models import SET_NULL from django.db.models.signals import post_save, post_delete, pre_save from django.dispatch import receiver from karmaworld.apps.users.models import NoteKarmaEvent, GenericKarmaEvent +from karmaworld.secret.filepicker import FILEPICKER_API_KEY +from karmaworld.utils.filepicker import encode_fp_policy, sign_fp_policy import os +import time import urllib from django.conf import settings @@ -49,15 +53,6 @@ s3_upload_headers = { # https://github.com/FinalsClub/karmaworld/issues/273#issuecomment-32572169 all_read_xml_acl = '\n710efc05767903a0eae5064bbc541f1c8e68f8f344fa809dc92682146b401d9cAndrew710efc05767903a0eae5064bbc541f1c8e68f8f344fa809dc92682146b401d9cAndrewREAD710efc05767903a0eae5064bbc541f1c8e68f8f344fa809dc92682146b401d9cAndrewWRITE710efc05767903a0eae5064bbc541f1c8e68f8f344fa809dc92682146b401d9cAndrewREAD_ACP710efc05767903a0eae5064bbc541f1c8e68f8f344fa809dc92682146b401d9cAndrewWRITE_ACPhttp://acs.amazonaws.com/groups/global/AllUsersREAD' -def _choose_upload_to(instance, filename): - # /school/course/year/month/day - return u"{school}/{course}/{year}/{month}/{day}".format( - school=instance.course.school.slug, - course=instance.course.slug, - year=instance.uploaded_at.year, - month=instance.uploaded_at.month, - day=instance.uploaded_at.day) - class Document(models.Model): """ @@ -85,11 +80,51 @@ class Document(models.Model): # WARNING: This may throw an error on migration is_hidden = models.BooleanField(default=False) + ### + # Everything Filepicker, now in one small area + + # Allow pick (choose files), store (upload to S3), read (from FP repo), + # stat (status of FP repo files) for 1 year (current time + 365 * 24 * 3600 + # seconds). Generated one time, at class definition upon import. So the + # server will need to be rebooted at least one time each year or this will + # go stale. + fp_policy_json = '{{"expiry": {0}, "call": ["pick","store","read","stat"]}}' + fp_policy_json = fp_policy_json.format(int(time.time() + 31536000)) + fp_policy = encode_fp_policy(fp_policy_json) + fp_signature = sign_fp_policy(fp_policy) + + # Hack because mimetypes conflict with extensions, but there is no way to + # disable mimetypes. + # https://github.com/Ink/django-filepicker/issues/22 + django_filepicker.forms.FPFieldMixin.default_mimetypes = '' + # Now let django-filepicker do the heavy lifting. Sort of. Look at all those + # parameters! fp_file = django_filepicker.models.FPFileField( - upload_to=_choose_upload_to, - storage=fs, - null=True, blank=True, - help_text=u"An uploaded file reference from Filepicker.io") + # FPFileField settings + apikey=FILEPICKER_API_KEY, + services='COMPUTER,DROPBOX,URL,GOOGLE_DRIVE,EVERNOTE,GMAIL,BOX,FACEBOOK,FLICKR,PICASA,IMAGE_SEARCH,WEBCAM,FTP', + additional_params={ + 'data-fp-multiple': 'true', + 'data-fp-folders': 'true', + 'data-fp-button-class': + 'add-note-btn small-10 columns large-4', + 'data-fp-button-text': + mark_safe(" add notes"), + 'data-fp-drag-class': + 'dragdrop show-for-medium-up large-7 columns', + 'data-fp-drag-text': 'Drop Some Knowledge', + 'data-fp-extensions': + '.pdf,.doc,.docx,.txt,.html,.rtf,.odt,.png,.jpg,.jpeg,.ppt,.pptx', + 'data-fp-store-location': 'S3', + 'data-fp-policy': fp_policy, + 'data-fp-signature': fp_signature, + 'onchange': "got_file(event)", + }, + # FileField settings + null=True, blank=True, + upload_to='nil', # field ignored because S3, but required. + verbose_name='', # prevent a label from showing up + ) mimetype = models.CharField(max_length=255, blank=True, null=True) class Meta: @@ -110,31 +145,24 @@ class Document(models.Model): def get_file(self): """ Downloads the file from filepicker.io and returns a Django File wrapper object """ - # clean up any old downloads that are still hanging around - if hasattr(self, 'tempfile'): - self.tempfile.close() - delattr(self, 'tempfile') - - if hasattr(self, 'filename'): - # the file might have been moved in the meantime so - # check first - if os.path.exists(self.filename): - os.remove(self.filename) - delattr(self, 'filename') - - # The temporary file will be created in a directory set by the - # environment (TEMP_DIR, TEMP or TMP) - self.filename, header = urllib.urlretrieve(self.fp_file.name) - name = os.path.basename(self.filename) - disposition = header.get('Content-Disposition') - if disposition: - name = disposition.rpartition("filename=")[2].strip('" ') - filename = header.get('X-File-Name') - if filename: - name = filename - - self.tempfile = open(self.filename, 'r') - return File(self.tempfile, name=name) + fpf = django_filepicker.utils.FilepickerFile(self.fp_file.name) + fd = fpf.get_file(self.fp_file.field.additional_params) + # temporary workaround. FilepickerFile closes/deletes the fd when + # garbage collected, so write it to another fd. + # https://github.com/Ink/django-filepicker/issues/25 + newfd = os.tmpfile() + buff_size = 1048576 # read 1 MiB at a time + buff = fd.read(buff_size) + while buff != '': + # write until EOF + newfd.write(buff) + buff = fd.read(buff_size) + # replace the Django File's python file with the reset temp file. + newfd.seek(0) + fd.file = newfd + # force FilepickerFile to be garbage collected now, why not + del fpf + return fd def save(self, *args, **kwargs): if self.name and not self.slug: diff --git a/karmaworld/secret/filepicker.py.example b/karmaworld/secret/filepicker.py.example index bab7b50..4369a28 100644 --- a/karmaworld/secret/filepicker.py.example +++ b/karmaworld/secret/filepicker.py.example @@ -7,3 +7,4 @@ that file. DO NOT check filepicker.py into source control. """ FILEPICKER_API_KEY = 0 +SECRET = 0 diff --git a/karmaworld/templates/partial/filepicker.html b/karmaworld/templates/partial/filepicker.html index 2a133ad..d00e2f8 100644 --- a/karmaworld/templates/partial/filepicker.html +++ b/karmaworld/templates/partial/filepicker.html @@ -1,24 +1,10 @@ {% load url from future %} {% load socialaccount %} +{{ file_upload_form.media }}
- -
- + {{ file_upload_form.as_p }}
@@ -87,109 +73,105 @@ diff --git a/karmaworld/utils/filepicker.py b/karmaworld/utils/filepicker.py new file mode 100644 index 0000000..f816c76 --- /dev/null +++ b/karmaworld/utils/filepicker.py @@ -0,0 +1,28 @@ +import hmac +import json +import base64 + +from hashlib import sha256 +from karmaworld.secret.filepicker import SECRET + +def encode_fp_policy(policy): + """ Return URL-safe Base64 encoded JSON Filepicker policy. """ + # Validate the JSON before trying to sign it and send it off to FP. It'll + # be easier to trap exceptions in Python than read errors out of FP. + + # drop an exception bomb if the policy is not valid JSON. + pypolicy = json.loads(policy) + # ensure expiry is included. drop excepbomb if it isn't. + pypolicy['expiry'] + + # https://developers.inkfilepicker.com/docs/security/#signPolicy + # encode and return + return base64.urlsafe_b64encode(policy) + +def sign_fp_policy(policy): + """ Return a signature appropriate for the given encoded policy. """ + # https://developers.inkfilepicker.com/docs/security/#signPolicy + # hash it up, bra! + engine = hmac.new(SECRET, digestmod=sha256) + engine.update(policy) + return engine.hexdigest() diff --git a/reqs/common.txt b/reqs/common.txt index dc5f553..f2715e7 100644 --- a/reqs/common.txt +++ b/reqs/common.txt @@ -9,7 +9,7 @@ urllib3==1.5 google-api-python-client==1.0 django-grappelli==2.4.8 git+https://github.com/FinalsClub/django-taggit.git -django-filepicker==0.1.5 +git+https://github.com/btbonval/django-filepicker filemagic==1.6 requests beautifulsoup4 -- 2.25.1