From 07e4e870a43c5c25bf7d64868aa67374ebf234b3 Mon Sep 17 00:00:00 2001 From: Bryan Date: Sat, 22 Feb 2014 16:22:24 -0500 Subject: [PATCH] hotfix from prod to upload MIT courses in a more reliable manner. --- .../management/commands/import_ocw_json.py | 51 ++++++++++++------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/karmaworld/apps/notes/management/commands/import_ocw_json.py b/karmaworld/apps/notes/management/commands/import_ocw_json.py index 85a9d3c..4e43e34 100644 --- a/karmaworld/apps/notes/management/commands/import_ocw_json.py +++ b/karmaworld/apps/notes/management/commands/import_ocw_json.py @@ -142,7 +142,12 @@ class Command(BaseCommand): ulresp = requests.post(fpurl, data={ 'url': url, }) - ulresp.raise_for_status() + try: + ulresp.raise_for_status() + except Exception, e: + print "Failed to upload note: " + str(e) + print "Skipping." + continue # Filepicker returns JSON, so use that uljson = ulresp.json() @@ -167,21 +172,33 @@ class Command(BaseCommand): dbnote.tags.add('mit-ocw','karma') print "Converting document and saving note to S3." - try: - convert_raw_document(dbnote) - except ValueError, e: - # only catch one specific error - if not str(e).startswith('PDF file could not be'): - raise e - # write the link to file. - with open('pdferrors.log', 'a') as pdferrs: - pdferrs.write(url + '\n') - # delete the partial Note created in convert_raw_doc - dbnote = Note.objects.filter(upstream_link=url)[0] - dbnote.tags.set() - dbnote.delete() - print "This note errored, so it is removed :(" - else: - print "This note is done." + while True: + try: + convert_raw_document(dbnote) + except ValueError, e: + # only catch one specific error + if not str(e).startswith('PDF file could not be'): + raise e + # write the link to file. + with open('pdferrors.log', 'a') as pdferrs: + pdferrs.write(url + '\n') + # delete the partial Note created in convert_raw_doc + dbnote = Note.objects.filter(upstream_link=url)[0] + dbnote.tags.set() + dbnote.delete() + print "This note errored, so it is removed :(" + break + except Exception, e: + if '403' in str(e): + print "Failed: " + str(e) + print "Trying again." + continue + else: + print "Failed: " + str(e) + print "Aborting." + break + else: + print "This note is done." + break print "Notes for {0} are done.".format(dbcourse.name) -- 2.25.1