Track HITs locally

diff --git a/karmaworld/apps/quizzes/admin.py b/karmaworld/apps/quizzes/admin.py
index de5a7afd78937babeeb5d4275c49d225800e3fcc..c7e669f3a8652f20091bcc26de1544657cdb2bf7 100644 (file)
--- a/karmaworld/apps/quizzes/admin.py
+++ b/karmaworld/apps/quizzes/admin.py
@@ -2,6 +2,7 @@
 # -*- coding:utf8 -*-
 # Copyright (C) 2014  FinalsClub Foundation
 from django.contrib import admin
-from karmaworld.apps.quizzes.models import Keyword
+from karmaworld.apps.quizzes.models import Keyword, HIT
 
 admin.site.register(Keyword)
+admin.site.register(HIT)

diff --git a/karmaworld/apps/quizzes/migrations/0007_auto__add_hit.py b/karmaworld/apps/quizzes/migrations/0007_auto__add_hit.py
new file mode 100644 (file)
index 0000000..8c737ad
--- /dev/null
+++ b/karmaworld/apps/quizzes/migrations/0007_auto__add_hit.py
@@ -0,0 +1,168 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'HIT'
+        db.create_table(u'quizzes_hit', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('HITId', self.gf('django.db.models.fields.CharField')(unique=True, max_length=100)),
+            ('note', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['notes.Note'])),
+            ('processed', self.gf('django.db.models.fields.BooleanField')(default=False)),
+        ))
+        db.send_create_signal(u'quizzes', ['HIT'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'HIT'
+        db.delete_table(u'quizzes_hit')
+
+
+    models = {
+        u'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        u'auth.permission': {
+            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        u'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        u'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        u'courses.course': {
+            'Meta': {'ordering': "['-file_count', 'school', 'name']", 'unique_together': "(('name', 'school'),)", 'object_name': 'Course'},
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'department': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['courses.Department']", 'null': 'True', 'blank': 'True'}),
+            'desc': ('django.db.models.fields.TextField', [], {'max_length': '511', 'null': 'True', 'blank': 'True'}),
+            'file_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'flags': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'instructor_email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}),
+            'instructor_name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'professor': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': u"orm['courses.Professor']", 'null': 'True', 'blank': 'True'}),
+            'school': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['courses.School']", 'null': 'True', 'blank': 'True'}),
+            'slug': ('django.db.models.fields.SlugField', [], {'max_length': '150', 'null': 'True'}),
+            'thank_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'updated_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.utcnow'}),
+            'url': ('django.db.models.fields.URLField', [], {'max_length': '511', 'null': 'True', 'blank': 'True'})
+        },
+        u'courses.department': {
+            'Meta': {'unique_together': "(('name', 'school'),)", 'object_name': 'Department'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'school': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['courses.School']"}),
+            'slug': ('django.db.models.fields.SlugField', [], {'max_length': '150', 'null': 'True'}),
+            'url': ('django.db.models.fields.URLField', [], {'max_length': '511', 'null': 'True', 'blank': 'True'})
+        },
+        u'courses.professor': {
+            'Meta': {'unique_together': "(('name', 'email'),)", 'object_name': 'Professor'},
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '255'})
+        },
+        u'courses.school': {
+            'Meta': {'ordering': "['-file_count', '-priority', 'name']", 'object_name': 'School'},
+            'alias': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'facebook_id': ('django.db.models.fields.BigIntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'file_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'hashtag': ('django.db.models.fields.CharField', [], {'max_length': '16', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'location': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'priority': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'slug': ('django.db.models.fields.SlugField', [], {'max_length': '150', 'null': 'True'}),
+            'url': ('django.db.models.fields.URLField', [], {'max_length': '511', 'blank': 'True'}),
+            'usde_id': ('django.db.models.fields.BigIntegerField', [], {'unique': 'True', 'null': 'True', 'blank': 'True'})
+        },
+        u'licenses.license': {
+            'Meta': {'object_name': 'License'},
+            'html': ('django.db.models.fields.TextField', [], {}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'})
+        },
+        u'notes.note': {
+            'Meta': {'ordering': "['-uploaded_at']", 'unique_together': "(('fp_file', 'upstream_link'),)", 'object_name': 'Note'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'course': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['courses.Course']"}),
+            'flags': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'fp_file': ('django_filepicker.models.FPFileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'gdrive_url': ('django.db.models.fields.URLField', [], {'max_length': '1024', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'ip': ('django.db.models.fields.GenericIPAddressField', [], {'max_length': '39', 'null': 'True', 'blank': 'True'}),
+            'is_hidden': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'license': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['licenses.License']", 'null': 'True', 'blank': 'True'}),
+            'mimetype': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '255'}),
+            'text': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'thanks': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}),
+            'tweeted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'uploaded_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.utcnow', 'null': 'True'}),
+            'upstream_link': ('django.db.models.fields.URLField', [], {'max_length': '1024', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']", 'null': 'True', 'on_delete': 'models.SET_NULL', 'blank': 'True'})
+        },
+        u'quizzes.hit': {
+            'HITId': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}),
+            'Meta': {'object_name': 'HIT'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'note': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['notes.Note']"}),
+            'processed': ('django.db.models.fields.BooleanField', [], {'default': 'False'})
+        },
+        u'quizzes.keyword': {
+            'Meta': {'unique_together': "(('word', 'note', 'ranges'),)", 'object_name': 'Keyword'},
+            'definition': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'note': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['notes.Note']", 'null': 'True', 'blank': 'True'}),
+            'ranges': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
+            'timestamp': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.utcnow'}),
+            'unreviewed': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'word': ('django.db.models.fields.CharField', [], {'max_length': '1024'})
+        },
+        u'taggit.tag': {
+            'Meta': {'object_name': 'Tag'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}),
+            'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '100'})
+        },
+        u'taggit.taggeditem': {
+            'Meta': {'object_name': 'TaggedItem'},
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_tagged_items'", 'to': u"orm['contenttypes.ContentType']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'object_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}),
+            'tag': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_items'", 'to': u"orm['taggit.Tag']"})
+        }
+    }
+
+    complete_apps = ['quizzes']
\ No newline at end of file

diff --git a/karmaworld/apps/quizzes/models.py b/karmaworld/apps/quizzes/models.py
index 36186a4d792fe821c1e4e5337a44370a83fe1e97..933b4ceb14411d932c21b3df5b31530cc432ce13 100644 (file)
--- a/karmaworld/apps/quizzes/models.py
+++ b/karmaworld/apps/quizzes/models.py
@@ -19,3 +19,11 @@ class Keyword(models.Model):
     def __unicode__(self):
         return u"{w}: {d}".format(w=self.word, d=self.definition)
 
+
+class HIT(models.Model):
+    HITId = models.CharField(max_length=100, unique=True, null=False)
+    note = models.ForeignKey(to='notes.Note', null=False)
+    processed = models.BooleanField(default=False)
+
+    def __unicode__(self):
+        return u'{i}: {p}'.format(i=self.HITId, p='processed' if self.processed else 'unprocessed')

diff --git a/karmaworld/apps/quizzes/tasks.py b/karmaworld/apps/quizzes/tasks.py
index b886e9e8a0180178cf19590a50805132e0720c60..fca872f4a3ed1bbf29b181b7cc88302019a0c37d 100644 (file)
--- a/karmaworld/apps/quizzes/tasks.py
+++ b/karmaworld/apps/quizzes/tasks.py
@@ -11,7 +11,7 @@ from boto.mturk.connection import MTurkConnection
 from django.contrib.sites.models import Site
 from django.core.exceptions import ObjectDoesNotExist
 from karmaworld.apps.notes.models import Note
-from karmaworld.apps.quizzes.models import Keyword
+from karmaworld.apps.quizzes.models import Keyword, HIT
 from django.conf import settings
 
 logger = get_task_logger(__name__)
@@ -133,11 +133,14 @@ def submit_extract_keywords_hit(note):
                                        is_required=False)
         question_form.append(definition_question)
 
-    connection.create_hit(questions=question_form, max_assignments=1,
+    hit = connection.create_hit(questions=question_form, max_assignments=1,
                           title=title, description=HIT_DESCRIPTION,
                           keywords=HIT_KEYWORDS, duration=HIT_DURATION,
                           reward=HIT_REWARD, qualifications=HIT_QUALIFICATION,
-                          annotation=str(note.id))
+                          annotation=str(note.id))[0]
+
+    HIT.objects.create(HITId=hit.HITId, note=note, processed=False)
+
 
 
 @task(name='get_extract_keywords_results')
@@ -152,27 +155,13 @@ def get_extract_keywords_results():
     connection = MTurkConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY,
                                  host=MTURK_HOST)
 
-    reviewable_hits = connection.get_reviewable_hits(status='Reviewable', page_size=100,
-                                                     sort_by='CreationTime', sort_direction='Descending')
-    for hit in reviewable_hits:
-        logger.info('Found HIT {0}'.format(hit.HITId))
-        try:
-            note_id = connection.get_hit(hit.HITId)[0].RequesterAnnotation
-            note_id = int(note_id)
-        except (AttributeError, ValueError):
-            logger.error('HIT {0} does not have a valid RequesterAnnotation, '
-                         'so we cannot determine which note it references'.format(hit.HITId))
-            return
-
-        try:
-            note = Note.objects.get(id=note_id)
-        except ObjectDoesNotExist:
-            logger.error('Could not find note {0} which was referenced by HIT {1}'.format(note_id, hit.HITId))
-            return
+    for hit_object in HIT.objects.filter(processed=False):
+        logger.info('Found unprocessed HIT {0}'.format(hit_object.HITId))
 
         answers = {}
-        assignments = [a for a in connection.get_assignments(hit.HITId) if a.AssignmentStatus == 'Approved']
+        assignments = [a for a in connection.get_assignments(hit_object.HITId) if a.AssignmentStatus == 'Approved']
         for assignment in assignments:
+            logger.info('Found approved assignment for HIT {0}'.format(hit_object.HITId))
             for question_form_answer in assignment.answers[0]:
                 answers[question_form_answer.qid] = question_form_answer.fields[0]
 
@@ -183,6 +172,11 @@ def get_extract_keywords_results():
                 keyword = answers[keyword_qid]
                 definition = answers[definition_qid]
                 if keyword:
-                    Keyword.objects.create(word=keyword, definition=definition, note=note, unreviewed=True)
+                    Keyword.objects.create(word=keyword, definition=definition, note=hit_object.note, unreviewed=True)
             except KeyError:
                 pass
+
+        if assignments:
+            logger.info('Done processing HIT {0}'.format(hit_object.HITId))
+            hit_object.processed = True
+            hit_object.save()