From 402d1ea78060a98d1869677d7c46666f85683f92 Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Tue, 3 Jun 2014 16:00:16 -0400 Subject: [PATCH] Track HITs locally --- karmaworld/apps/quizzes/admin.py | 3 +- .../quizzes/migrations/0007_auto__add_hit.py | 168 ++++++++++++++++++ karmaworld/apps/quizzes/models.py | 8 + karmaworld/apps/quizzes/tasks.py | 38 ++-- 4 files changed, 194 insertions(+), 23 deletions(-) create mode 100644 karmaworld/apps/quizzes/migrations/0007_auto__add_hit.py diff --git a/karmaworld/apps/quizzes/admin.py b/karmaworld/apps/quizzes/admin.py index de5a7af..c7e669f 100644 --- a/karmaworld/apps/quizzes/admin.py +++ b/karmaworld/apps/quizzes/admin.py @@ -2,6 +2,7 @@ # -*- coding:utf8 -*- # Copyright (C) 2014 FinalsClub Foundation from django.contrib import admin -from karmaworld.apps.quizzes.models import Keyword +from karmaworld.apps.quizzes.models import Keyword, HIT admin.site.register(Keyword) +admin.site.register(HIT) diff --git a/karmaworld/apps/quizzes/migrations/0007_auto__add_hit.py b/karmaworld/apps/quizzes/migrations/0007_auto__add_hit.py new file mode 100644 index 0000000..8c737ad --- /dev/null +++ b/karmaworld/apps/quizzes/migrations/0007_auto__add_hit.py @@ -0,0 +1,168 @@ +# -*- coding: utf-8 -*- +from south.utils import datetime_utils as datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'HIT' + db.create_table(u'quizzes_hit', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('HITId', self.gf('django.db.models.fields.CharField')(unique=True, max_length=100)), + ('note', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['notes.Note'])), + ('processed', self.gf('django.db.models.fields.BooleanField')(default=False)), + )) + db.send_create_signal(u'quizzes', ['HIT']) + + + def backwards(self, orm): + # Deleting model 'HIT' + db.delete_table(u'quizzes_hit') + + + models = { + u'auth.group': { + 'Meta': {'object_name': 'Group'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + u'auth.permission': { + 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + u'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + u'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + u'courses.course': { + 'Meta': {'ordering': "['-file_count', 'school', 'name']", 'unique_together': "(('name', 'school'),)", 'object_name': 'Course'}, + 'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'department': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['courses.Department']", 'null': 'True', 'blank': 'True'}), + 'desc': ('django.db.models.fields.TextField', [], {'max_length': '511', 'null': 'True', 'blank': 'True'}), + 'file_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'flags': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instructor_email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'instructor_name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'professor': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': u"orm['courses.Professor']", 'null': 'True', 'blank': 'True'}), + 'school': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['courses.School']", 'null': 'True', 'blank': 'True'}), + 'slug': ('django.db.models.fields.SlugField', [], {'max_length': '150', 'null': 'True'}), + 'thank_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'updated_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.utcnow'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '511', 'null': 'True', 'blank': 'True'}) + }, + u'courses.department': { + 'Meta': {'unique_together': "(('name', 'school'),)", 'object_name': 'Department'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'school': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['courses.School']"}), + 'slug': ('django.db.models.fields.SlugField', [], {'max_length': '150', 'null': 'True'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '511', 'null': 'True', 'blank': 'True'}) + }, + u'courses.professor': { + 'Meta': {'unique_together': "(('name', 'email'),)", 'object_name': 'Professor'}, + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + }, + u'courses.school': { + 'Meta': {'ordering': "['-file_count', '-priority', 'name']", 'object_name': 'School'}, + 'alias': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'facebook_id': ('django.db.models.fields.BigIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'file_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'hashtag': ('django.db.models.fields.CharField', [], {'max_length': '16', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'location': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'priority': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'slug': ('django.db.models.fields.SlugField', [], {'max_length': '150', 'null': 'True'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '511', 'blank': 'True'}), + 'usde_id': ('django.db.models.fields.BigIntegerField', [], {'unique': 'True', 'null': 'True', 'blank': 'True'}) + }, + u'licenses.license': { + 'Meta': {'object_name': 'License'}, + 'html': ('django.db.models.fields.TextField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}) + }, + u'notes.note': { + 'Meta': {'ordering': "['-uploaded_at']", 'unique_together': "(('fp_file', 'upstream_link'),)", 'object_name': 'Note'}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'course': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['courses.Course']"}), + 'flags': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'fp_file': ('django_filepicker.models.FPFileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'gdrive_url': ('django.db.models.fields.URLField', [], {'max_length': '1024', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.GenericIPAddressField', [], {'max_length': '39', 'null': 'True', 'blank': 'True'}), + 'is_hidden': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'license': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['licenses.License']", 'null': 'True', 'blank': 'True'}), + 'mimetype': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '255'}), + 'text': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'thanks': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}), + 'tweeted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'uploaded_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.utcnow', 'null': 'True'}), + 'upstream_link': ('django.db.models.fields.URLField', [], {'max_length': '1024', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']", 'null': 'True', 'on_delete': 'models.SET_NULL', 'blank': 'True'}) + }, + u'quizzes.hit': { + 'HITId': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}), + 'Meta': {'object_name': 'HIT'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'note': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['notes.Note']"}), + 'processed': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + }, + u'quizzes.keyword': { + 'Meta': {'unique_together': "(('word', 'note', 'ranges'),)", 'object_name': 'Keyword'}, + 'definition': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'note': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['notes.Note']", 'null': 'True', 'blank': 'True'}), + 'ranges': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}), + 'timestamp': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.utcnow'}), + 'unreviewed': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'word': ('django.db.models.fields.CharField', [], {'max_length': '1024'}) + }, + u'taggit.tag': { + 'Meta': {'object_name': 'Tag'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}), + 'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '100'}) + }, + u'taggit.taggeditem': { + 'Meta': {'object_name': 'TaggedItem'}, + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_tagged_items'", 'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}), + 'tag': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_items'", 'to': u"orm['taggit.Tag']"}) + } + } + + complete_apps = ['quizzes'] \ No newline at end of file diff --git a/karmaworld/apps/quizzes/models.py b/karmaworld/apps/quizzes/models.py index 36186a4..933b4ce 100644 --- a/karmaworld/apps/quizzes/models.py +++ b/karmaworld/apps/quizzes/models.py @@ -19,3 +19,11 @@ class Keyword(models.Model): def __unicode__(self): return u"{w}: {d}".format(w=self.word, d=self.definition) + +class HIT(models.Model): + HITId = models.CharField(max_length=100, unique=True, null=False) + note = models.ForeignKey(to='notes.Note', null=False) + processed = models.BooleanField(default=False) + + def __unicode__(self): + return u'{i}: {p}'.format(i=self.HITId, p='processed' if self.processed else 'unprocessed') diff --git a/karmaworld/apps/quizzes/tasks.py b/karmaworld/apps/quizzes/tasks.py index b886e9e..fca872f 100644 --- a/karmaworld/apps/quizzes/tasks.py +++ b/karmaworld/apps/quizzes/tasks.py @@ -11,7 +11,7 @@ from boto.mturk.connection import MTurkConnection from django.contrib.sites.models import Site from django.core.exceptions import ObjectDoesNotExist from karmaworld.apps.notes.models import Note -from karmaworld.apps.quizzes.models import Keyword +from karmaworld.apps.quizzes.models import Keyword, HIT from django.conf import settings logger = get_task_logger(__name__) @@ -133,11 +133,14 @@ def submit_extract_keywords_hit(note): is_required=False) question_form.append(definition_question) - connection.create_hit(questions=question_form, max_assignments=1, + hit = connection.create_hit(questions=question_form, max_assignments=1, title=title, description=HIT_DESCRIPTION, keywords=HIT_KEYWORDS, duration=HIT_DURATION, reward=HIT_REWARD, qualifications=HIT_QUALIFICATION, - annotation=str(note.id)) + annotation=str(note.id))[0] + + HIT.objects.create(HITId=hit.HITId, note=note, processed=False) + @task(name='get_extract_keywords_results') @@ -152,27 +155,13 @@ def get_extract_keywords_results(): connection = MTurkConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY, host=MTURK_HOST) - reviewable_hits = connection.get_reviewable_hits(status='Reviewable', page_size=100, - sort_by='CreationTime', sort_direction='Descending') - for hit in reviewable_hits: - logger.info('Found HIT {0}'.format(hit.HITId)) - try: - note_id = connection.get_hit(hit.HITId)[0].RequesterAnnotation - note_id = int(note_id) - except (AttributeError, ValueError): - logger.error('HIT {0} does not have a valid RequesterAnnotation, ' - 'so we cannot determine which note it references'.format(hit.HITId)) - return - - try: - note = Note.objects.get(id=note_id) - except ObjectDoesNotExist: - logger.error('Could not find note {0} which was referenced by HIT {1}'.format(note_id, hit.HITId)) - return + for hit_object in HIT.objects.filter(processed=False): + logger.info('Found unprocessed HIT {0}'.format(hit_object.HITId)) answers = {} - assignments = [a for a in connection.get_assignments(hit.HITId) if a.AssignmentStatus == 'Approved'] + assignments = [a for a in connection.get_assignments(hit_object.HITId) if a.AssignmentStatus == 'Approved'] for assignment in assignments: + logger.info('Found approved assignment for HIT {0}'.format(hit_object.HITId)) for question_form_answer in assignment.answers[0]: answers[question_form_answer.qid] = question_form_answer.fields[0] @@ -183,6 +172,11 @@ def get_extract_keywords_results(): keyword = answers[keyword_qid] definition = answers[definition_qid] if keyword: - Keyword.objects.create(word=keyword, definition=definition, note=note, unreviewed=True) + Keyword.objects.create(word=keyword, definition=definition, note=hit_object.note, unreviewed=True) except KeyError: pass + + if assignments: + logger.info('Done processing HIT {0}'.format(hit_object.HITId)) + hit_object.processed = True + hit_object.save() -- 2.25.1