From 775fb46d7ebf78d595baba60404fbf9269a5f303 Mon Sep 17 00:00:00 2001 From: Bryan Date: Sat, 31 Jan 2015 20:45:04 -0500 Subject: [PATCH] #396 pull static s3 html into NoteMarkdown.html --- .../notes/migrations/0021_get_html_from_s3.py | 192 ++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 karmaworld/apps/notes/migrations/0021_get_html_from_s3.py diff --git a/karmaworld/apps/notes/migrations/0021_get_html_from_s3.py b/karmaworld/apps/notes/migrations/0021_get_html_from_s3.py new file mode 100644 index 0000000..a173e47 --- /dev/null +++ b/karmaworld/apps/notes/migrations/0021_get_html_from_s3.py @@ -0,0 +1,192 @@ +# -*- coding: utf-8 -*- +from south.utils import datetime_utils as datetime +from south.db import db +from south.v2 import DataMigration +from django.db import models +from django.core.files.storage import default_storage +import requests + +class Migration(DataMigration): + + def forwards(self, orm): + "Write your forwards methods here." + # Note: Don't use "from appname.models import ModelName". + # Use orm.ModelName to refer to models in this application, + # and orm['appname.ModelName'] for models in other applications. + + # keep score + good = [] + bad = [] + + # find each Note without an html field, download its S3 html, and + # store it in the local database. + for note in orm['notes.Note'].objects.filter(notemarkdown__html__isnull=True): + # download the s3 content + html = '' + # copy/pasted from model code for Note.get_relative_s3_path + note_s3_path = 'html/{0}.html'.format(note.slug) + key = default_storage.bucket.get_key(note_s3_path) + if key: + html = key.read() + + if not html: + bad.append(note.slug) + continue + else: + good.append(note.slug) + + # store the html in the corresponding NoteMarkdown object + nmd = orm['notes.NoteMarkdown'].objects.get_or_create(note=note)[0] + nmd.html = html + nmd.save() + + # Display the score + print "Migrated {0} notes and failed to migrate {1} notes.".format( + len(good), len(bad)) + + print "Failed list:" + for slug in bad: + print slug + + def backwards(self, orm): + "Write your backwards methods here." + + models = { + u'auth.group': { + 'Meta': {'object_name': 'Group'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + u'auth.permission': { + 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + u'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + u'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + u'courses.course': { + 'Meta': {'ordering': "['-file_count', 'school', 'name']", 'unique_together': "(('name', 'school'),)", 'object_name': 'Course'}, + 'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'department': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['courses.Department']", 'null': 'True', 'blank': 'True'}), + 'desc': ('django.db.models.fields.TextField', [], {'max_length': '511', 'null': 'True', 'blank': 'True'}), + 'file_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'flags': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instructor_email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'instructor_name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'professor': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': u"orm['courses.Professor']", 'null': 'True', 'blank': 'True'}), + 'school': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['courses.School']", 'null': 'True', 'blank': 'True'}), + 'slug': ('django.db.models.fields.SlugField', [], {'max_length': '150', 'null': 'True'}), + 'thank_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'updated_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.utcnow'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '511', 'null': 'True', 'blank': 'True'}) + }, + u'courses.department': { + 'Meta': {'unique_together': "(('name', 'school'),)", 'object_name': 'Department'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'school': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['courses.School']"}), + 'slug': ('django.db.models.fields.SlugField', [], {'max_length': '150', 'null': 'True'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '511', 'null': 'True', 'blank': 'True'}) + }, + u'courses.professor': { + 'Meta': {'unique_together': "(('name', 'email'),)", 'object_name': 'Professor'}, + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + }, + u'courses.school': { + 'Meta': {'ordering': "['-file_count', '-priority', 'name']", 'object_name': 'School'}, + 'alias': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'facebook_id': ('django.db.models.fields.BigIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'file_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'hashtag': ('django.db.models.fields.CharField', [], {'max_length': '16', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'location': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'priority': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'slug': ('django.db.models.fields.SlugField', [], {'max_length': '150', 'null': 'True'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '511', 'blank': 'True'}), + 'usde_id': ('django.db.models.fields.BigIntegerField', [], {'unique': 'True', 'null': 'True', 'blank': 'True'}) + }, + u'licenses.license': { + 'Meta': {'object_name': 'License'}, + 'html': ('django.db.models.fields.TextField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}) + }, + u'notes.note': { + 'Meta': {'ordering': "['-uploaded_at']", 'unique_together': "(('fp_file', 'upstream_link'),)", 'object_name': 'Note'}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'course': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['courses.Course']"}), + 'flags': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'fp_file': ('django_filepicker.models.FPFileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'gdrive_url': ('django.db.models.fields.URLField', [], {'max_length': '1024', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.GenericIPAddressField', [], {'max_length': '39', 'null': 'True', 'blank': 'True'}), + 'is_hidden': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'license': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['licenses.License']", 'null': 'True', 'blank': 'True'}), + 'mimetype': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '255'}), + 'text': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'thanks': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}), + 'tweeted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'uploaded_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.utcnow', 'null': 'True'}), + 'upstream_link': ('django.db.models.fields.URLField', [], {'max_length': '1024', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']", 'null': 'True', 'on_delete': 'models.SET_NULL', 'blank': 'True'}) + }, + u'notes.notemarkdown': { + 'Meta': {'object_name': 'NoteMarkdown'}, + 'html': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'markdown': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'note': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['notes.Note']", 'unique': 'True', 'primary_key': 'True'}) + }, + u'notes.useruploadmapping': { + 'Meta': {'unique_together': "(('user', 'fp_file'),)", 'object_name': 'UserUploadMapping'}, + 'fp_file': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}) + }, + u'taggit.tag': { + 'Meta': {'object_name': 'Tag'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}), + 'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '100'}) + }, + u'taggit.taggeditem': { + 'Meta': {'object_name': 'TaggedItem'}, + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_tagged_items'", 'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}), + 'tag': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_items'", 'to': u"orm['taggit.Tag']"}) + } + } + + complete_apps = ['notes'] + symmetrical = True -- 2.25.1