workaround to deal with migration transaction

need to ensure that we don't lose hours of work because one note fails.

diff --git a/karmaworld/apps/notes/migrations/0021_get_html_from_s3.py b/karmaworld/apps/notes/migrations/0021_get_html_from_s3.py
index 85afec1033a8eda79d61713b25f04ce2301a3212..b746075f79db01bfb4228c40a92759077e2f6b5d 100644 (file)
--- a/karmaworld/apps/notes/migrations/0021_get_html_from_s3.py
+++ b/karmaworld/apps/notes/migrations/0021_get_html_from_s3.py
@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+import os
 import sys
 
 from karmaworld.apps.notes import sanitizer
@@ -9,6 +10,9 @@ from django.db import models
 from django.core.files.storage import default_storage
 import requests
 
+def display_counts(current, total):
+    sys.stdout.write('\n{0:04}/{1:04} '.format(current, total))
+
 class Migration(DataMigration):
 
     def forwards(self, orm):
@@ -36,9 +40,20 @@ class Migration(DataMigration):
         necessary_notes = orm['notes.Note'].objects.filter(notemarkdown__html__isnull=True)
         n_notes = necessary_notes.count()
 
+        # perform migration in discrete chunks to deal with the transaction
+        # (just delete the migration from the south table and run again)
+        limitkey = 'NOTE_LIMIT_0021'
+        sys.stdout.write('Running until ')
+        if os.environ.has_key(limitkey):
+            max_notes = int(os.environ[limitkey])
+            display_counts(max_notes, n_notes)
+        else:
+            max_notes = n_notes
+            display_counts(n_notes, n_notes)
+
         # visualiation to show how well this is moving through a large database.
         counter = 0
-        sys.stdout.write('\n{0:04}/{1:04} '.format(counter, n_notes))
+        display_counts(counter, max_notes)
         # find each Note without an html field, download its S3 html, and
         # store it in the local database.
         for note in necessary_notes:
@@ -56,6 +71,7 @@ class Migration(DataMigration):
             if not html:
                 sys.stdout.write('( ')
                 bad.append(note.slug)
+                counter = counter + 1
                 continue
             else:
                 good.append(note.slug)
@@ -86,10 +102,14 @@ class Migration(DataMigration):
             # track 20 notes per line
             if counter % 20 == 0:
                 # finish off previous line and start new line
-                sys.stdout.write('\n{0:04}/{1:04} '.format(counter, n_notes))
+                display_counts(counter, max_notes)
                 # flush per line, just in case it isn't outputting
                 sys.stdout.flush()
 
+            # perform migration in discrete chunks to deal with the transaction
+            if counter == max_notes:
+                break
+
         # Display the score
         print "Migrated {0} notes and failed to migrate {1} notes.".format(
           len(good), len(bad))