From: Bryan <btbonval@gmail.com>
Date: Sat, 22 Feb 2014 21:22:24 +0000 (-0500)
Subject: hotfix from prod to upload MIT courses in a more reliable manner.
X-Git-Tag: release-20150131~163^2~8
X-Git-Url: https://git.librecmc.org/?a=commitdiff_plain;h=07e4e870a43c5c25bf7d64868aa67374ebf234b3;p=oweals%2Fkarmaworld.git

hotfix from prod to upload MIT courses in a more reliable manner.
---

diff --git a/karmaworld/apps/notes/management/commands/import_ocw_json.py b/karmaworld/apps/notes/management/commands/import_ocw_json.py
index 85a9d3c..4e43e34 100644
--- a/karmaworld/apps/notes/management/commands/import_ocw_json.py
+++ b/karmaworld/apps/notes/management/commands/import_ocw_json.py
@@ -142,7 +142,12 @@ class Command(BaseCommand):
                             ulresp = requests.post(fpurl, data={
                               'url': url,
                             })
-                            ulresp.raise_for_status()
+                            try:
+                                ulresp.raise_for_status()
+                            except Exception, e:
+                                print "Failed to upload note: " + str(e)
+                                print "Skipping."
+                                continue
                             # Filepicker returns JSON, so use that
                             uljson = ulresp.json()
 
@@ -167,21 +172,33 @@ class Command(BaseCommand):
                         dbnote.tags.add('mit-ocw','karma')
 
                         print "Converting document and saving note to S3."
-                        try:
-                            convert_raw_document(dbnote)
-                        except ValueError, e:
-                            # only catch one specific error
-                            if not str(e).startswith('PDF file could not be'):
-                                raise e
-                            # write the link to file.
-                            with open('pdferrors.log', 'a') as pdferrs:
-                                pdferrs.write(url + '\n')
-                            # delete the partial Note created in convert_raw_doc
-                            dbnote = Note.objects.filter(upstream_link=url)[0]
-                            dbnote.tags.set()
-                            dbnote.delete()
-                            print "This note errored, so it is removed :("
-                        else:
-                            print "This note is done."
+                        while True:
+                            try:
+                                convert_raw_document(dbnote)
+                            except ValueError, e:
+                                # only catch one specific error
+                                if not str(e).startswith('PDF file could not be'):
+                                    raise e
+                                # write the link to file.
+                                with open('pdferrors.log', 'a') as pdferrs:
+                                    pdferrs.write(url + '\n')
+                                # delete the partial Note created in convert_raw_doc
+                                dbnote = Note.objects.filter(upstream_link=url)[0]
+                                dbnote.tags.set()
+                                dbnote.delete()
+                                print "This note errored, so it is removed :("
+                                break
+                            except Exception, e:
+                                if '403' in str(e):
+                                    print "Failed: " + str(e)
+                                    print "Trying again."
+                                    continue
+                                else:
+                                    print "Failed: " + str(e)
+                                    print "Aborting."
+                                    break
+                            else:
+                                print "This note is done."
+                                break
 
                     print "Notes for {0} are done.".format(dbcourse.name)