From 2374f495f91b45345df7752aefa825ff51d0f098 Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Sat, 10 May 2014 15:55:26 -0400 Subject: [PATCH] Use mechanical turk to get keywords of notes --- fabfile.py | 9 - karmaworld/apps/notes/views.py | 6 + ...l_multiplechoiceoption__del_quiz__del_m.py | 218 ++++++++++++++++++ ...0006_auto__add_field_keyword_unreviewed.py | 156 +++++++++++++ karmaworld/apps/quizzes/models.py | 4 +- karmaworld/apps/quizzes/tasks.py | 161 +++++++++++++ karmaworld/secret/mturk.py.example | 7 + karmaworld/settings/prod.py | 4 + reqs/common.txt | 2 +- 9 files changed, 554 insertions(+), 13 deletions(-) create mode 100644 karmaworld/apps/quizzes/migrations/0005_auto__del_truefalsequestion__del_multiplechoiceoption__del_quiz__del_m.py create mode 100644 karmaworld/apps/quizzes/migrations/0006_auto__add_field_keyword_unreviewed.py create mode 100644 karmaworld/apps/quizzes/tasks.py create mode 100644 karmaworld/secret/mturk.py.example diff --git a/fabfile.py b/fabfile.py index 8ce4315..7bee315 100644 --- a/fabfile.py +++ b/fabfile.py @@ -287,13 +287,6 @@ def import_usde(): virtenv_exec('{0}/manage.py import_usde_csv {1}'.format(env.code_root, env.usde_csv)) virtenv_exec('{0}/manage.py sanitize_usde_schools'.format(env.code_root)) -@task -def nltk_download(): - """ - Initialize corpa used by NLTK - """ - virtenv_exec('{0}/manage.py nltk_download'.format(env.code_root)) - @task def first_deploy(): """ @@ -307,7 +300,6 @@ def first_deploy(): syncdb() compress_static() collect_static() - nltk_download() fetch_usde() import_usde() flush_memcache() @@ -325,7 +317,6 @@ def deploy(): syncdb() compress_static() collect_static() - nltk_download() flush_memcache() restart_supervisord() ########## END COMMANDS diff --git a/karmaworld/apps/notes/views.py b/karmaworld/apps/notes/views.py index 2a8eb82..6a4ea6e 100644 --- a/karmaworld/apps/notes/views.py +++ b/karmaworld/apps/notes/views.py @@ -14,6 +14,7 @@ from karmaworld.apps.notes.search import SearchIndex from karmaworld.apps.quizzes.create_quiz import quiz_from_keywords from karmaworld.apps.quizzes.forms import KeywordForm from karmaworld.apps.quizzes.models import Keyword +from karmaworld.apps.quizzes.tasks import submit_extract_keywords_hit, get_extract_keywords_results from karmaworld.apps.users.models import NoteKarmaEvent from karmaworld.utils.ajax_utils import * @@ -334,6 +335,11 @@ def process_note_thank_events(request_user, note): if note.user != request_user and note.user: NoteKarmaEvent.create_event(note.user, note, NoteKarmaEvent.THANKS) + # If note thanks exceeds a threshold, create a Mechanical + # Turk task to get some keywords for it + if note.thanks == 3: + submit_extract_keywords_hit(note) + def thank_note(request, pk): """Record that somebody has thanked a note.""" diff --git a/karmaworld/apps/quizzes/migrations/0005_auto__del_truefalsequestion__del_multiplechoiceoption__del_quiz__del_m.py b/karmaworld/apps/quizzes/migrations/0005_auto__del_truefalsequestion__del_multiplechoiceoption__del_quiz__del_m.py new file mode 100644 index 0000000..8d2e5b0 --- /dev/null +++ b/karmaworld/apps/quizzes/migrations/0005_auto__del_truefalsequestion__del_multiplechoiceoption__del_quiz__del_m.py @@ -0,0 +1,218 @@ +# -*- coding: utf-8 -*- +from south.utils import datetime_utils as datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Deleting model 'TrueFalseQuestion' + db.delete_table(u'quizzes_truefalsequestion') + + # Deleting model 'MultipleChoiceOption' + db.delete_table(u'quizzes_multiplechoiceoption') + + # Deleting model 'Quiz' + db.delete_table(u'quizzes_quiz') + + # Deleting model 'MultipleChoiceQuestion' + db.delete_table(u'quizzes_multiplechoicequestion') + + # Deleting model 'FlashCardQuestion' + db.delete_table(u'quizzes_flashcardquestion') + + + def backwards(self, orm): + # Adding model 'TrueFalseQuestion' + db.create_table(u'quizzes_truefalsequestion', ( + ('category', self.gf('django.db.models.fields.CharField')(max_length=50, null=True, blank=True)), + ('difficulty', self.gf('django.db.models.fields.CharField')(max_length=50, null=True, blank=True)), + ('timestamp', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.utcnow)), + ('text', self.gf('django.db.models.fields.CharField')(max_length=2048)), + ('explanation', self.gf('django.db.models.fields.CharField')(max_length=2048, null=True, blank=True)), + ('true', self.gf('django.db.models.fields.BooleanField')(default=False)), + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('quiz', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['quizzes.Quiz'])), + )) + db.send_create_signal(u'quizzes', ['TrueFalseQuestion']) + + # Adding model 'MultipleChoiceOption' + db.create_table(u'quizzes_multiplechoiceoption', ( + ('text', self.gf('django.db.models.fields.CharField')(max_length=2048)), + ('question', self.gf('django.db.models.fields.related.ForeignKey')(related_name='choices', to=orm['quizzes.MultipleChoiceQuestion'])), + ('correct', self.gf('django.db.models.fields.BooleanField')(default=False)), + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + )) + db.send_create_signal(u'quizzes', ['MultipleChoiceOption']) + + # Adding model 'Quiz' + db.create_table(u'quizzes_quiz', ( + ('note', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['notes.Note'], null=True, blank=True)), + ('timestamp', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.utcnow)), + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=512)), + )) + db.send_create_signal(u'quizzes', ['Quiz']) + + # Adding model 'MultipleChoiceQuestion' + db.create_table(u'quizzes_multiplechoicequestion', ( + ('category', self.gf('django.db.models.fields.CharField')(max_length=50, null=True, blank=True)), + ('difficulty', self.gf('django.db.models.fields.CharField')(max_length=50, null=True, blank=True)), + ('question_text', self.gf('django.db.models.fields.CharField')(max_length=2048)), + ('timestamp', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.utcnow)), + ('explanation', self.gf('django.db.models.fields.CharField')(max_length=2048, null=True, blank=True)), + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('quiz', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['quizzes.Quiz'])), + )) + db.send_create_signal(u'quizzes', ['MultipleChoiceQuestion']) + + # Adding model 'FlashCardQuestion' + db.create_table(u'quizzes_flashcardquestion', ( + ('category', self.gf('django.db.models.fields.CharField')(max_length=50, null=True, blank=True)), + ('difficulty', self.gf('django.db.models.fields.CharField')(max_length=50, null=True, blank=True)), + ('timestamp', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.utcnow)), + ('keyword_side', self.gf('django.db.models.fields.CharField')(max_length=2048)), + ('explanation', self.gf('django.db.models.fields.CharField')(max_length=2048, null=True, blank=True)), + ('definition_side', self.gf('django.db.models.fields.CharField')(max_length=2048)), + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('quiz', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['quizzes.Quiz'])), + )) + db.send_create_signal(u'quizzes', ['FlashCardQuestion']) + + + models = { + u'auth.group': { + 'Meta': {'object_name': 'Group'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + u'auth.permission': { + 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + u'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + u'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + u'courses.course': { + 'Meta': {'ordering': "['-file_count', 'school', 'name']", 'unique_together': "(('name', 'school'),)", 'object_name': 'Course'}, + 'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'department': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['courses.Department']", 'null': 'True', 'blank': 'True'}), + 'desc': ('django.db.models.fields.TextField', [], {'max_length': '511', 'null': 'True', 'blank': 'True'}), + 'file_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'flags': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instructor_email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'instructor_name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'professor': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': u"orm['courses.Professor']", 'null': 'True', 'blank': 'True'}), + 'school': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['courses.School']", 'null': 'True', 'blank': 'True'}), + 'slug': ('django.db.models.fields.SlugField', [], {'max_length': '150', 'null': 'True'}), + 'updated_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.utcnow'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '511', 'null': 'True', 'blank': 'True'}) + }, + u'courses.department': { + 'Meta': {'unique_together': "(('name', 'school'),)", 'object_name': 'Department'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'school': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['courses.School']"}), + 'slug': ('django.db.models.fields.SlugField', [], {'max_length': '150', 'null': 'True'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '511', 'null': 'True', 'blank': 'True'}) + }, + u'courses.professor': { + 'Meta': {'unique_together': "(('name', 'email'),)", 'object_name': 'Professor'}, + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + }, + u'courses.school': { + 'Meta': {'ordering': "['-file_count', '-priority', 'name']", 'object_name': 'School'}, + 'alias': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'facebook_id': ('django.db.models.fields.BigIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'file_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'hashtag': ('django.db.models.fields.CharField', [], {'max_length': '16', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'location': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'priority': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'slug': ('django.db.models.fields.SlugField', [], {'max_length': '150', 'null': 'True'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '511', 'blank': 'True'}), + 'usde_id': ('django.db.models.fields.BigIntegerField', [], {'unique': 'True', 'null': 'True', 'blank': 'True'}) + }, + u'licenses.license': { + 'Meta': {'object_name': 'License'}, + 'html': ('django.db.models.fields.TextField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}) + }, + u'notes.note': { + 'Meta': {'ordering': "['-uploaded_at']", 'unique_together': "(('fp_file', 'upstream_link'),)", 'object_name': 'Note'}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'course': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['courses.Course']"}), + 'flags': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'fp_file': ('django_filepicker.models.FPFileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'gdrive_url': ('django.db.models.fields.URLField', [], {'max_length': '1024', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.GenericIPAddressField', [], {'max_length': '39', 'null': 'True', 'blank': 'True'}), + 'is_hidden': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'license': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['licenses.License']", 'null': 'True', 'blank': 'True'}), + 'mimetype': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '255'}), + 'text': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'thanks': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}), + 'tweeted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'uploaded_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.utcnow', 'null': 'True'}), + 'upstream_link': ('django.db.models.fields.URLField', [], {'max_length': '1024', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']", 'null': 'True', 'on_delete': 'models.SET_NULL', 'blank': 'True'}) + }, + u'quizzes.keyword': { + 'Meta': {'unique_together': "(('word', 'note', 'ranges'),)", 'object_name': 'Keyword'}, + 'definition': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'note': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['notes.Note']", 'null': 'True', 'blank': 'True'}), + 'ranges': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}), + 'timestamp': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.utcnow'}), + 'word': ('django.db.models.fields.CharField', [], {'max_length': '1024'}) + }, + u'taggit.tag': { + 'Meta': {'object_name': 'Tag'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}), + 'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '100'}) + }, + u'taggit.taggeditem': { + 'Meta': {'object_name': 'TaggedItem'}, + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_tagged_items'", 'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}), + 'tag': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_items'", 'to': u"orm['taggit.Tag']"}) + } + } + + complete_apps = ['quizzes'] \ No newline at end of file diff --git a/karmaworld/apps/quizzes/migrations/0006_auto__add_field_keyword_unreviewed.py b/karmaworld/apps/quizzes/migrations/0006_auto__add_field_keyword_unreviewed.py new file mode 100644 index 0000000..d1124bc --- /dev/null +++ b/karmaworld/apps/quizzes/migrations/0006_auto__add_field_keyword_unreviewed.py @@ -0,0 +1,156 @@ +# -*- coding: utf-8 -*- +from south.utils import datetime_utils as datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'Keyword.unreviewed' + db.add_column(u'quizzes_keyword', 'unreviewed', + self.gf('django.db.models.fields.BooleanField')(default=False), + keep_default=False) + + + def backwards(self, orm): + # Deleting field 'Keyword.unreviewed' + db.delete_column(u'quizzes_keyword', 'unreviewed') + + + models = { + u'auth.group': { + 'Meta': {'object_name': 'Group'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + u'auth.permission': { + 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + u'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + u'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + u'courses.course': { + 'Meta': {'ordering': "['-file_count', 'school', 'name']", 'unique_together': "(('name', 'school'),)", 'object_name': 'Course'}, + 'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'department': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['courses.Department']", 'null': 'True', 'blank': 'True'}), + 'desc': ('django.db.models.fields.TextField', [], {'max_length': '511', 'null': 'True', 'blank': 'True'}), + 'file_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'flags': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instructor_email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'instructor_name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'professor': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': u"orm['courses.Professor']", 'null': 'True', 'blank': 'True'}), + 'school': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['courses.School']", 'null': 'True', 'blank': 'True'}), + 'slug': ('django.db.models.fields.SlugField', [], {'max_length': '150', 'null': 'True'}), + 'updated_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.utcnow'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '511', 'null': 'True', 'blank': 'True'}) + }, + u'courses.department': { + 'Meta': {'unique_together': "(('name', 'school'),)", 'object_name': 'Department'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'school': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['courses.School']"}), + 'slug': ('django.db.models.fields.SlugField', [], {'max_length': '150', 'null': 'True'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '511', 'null': 'True', 'blank': 'True'}) + }, + u'courses.professor': { + 'Meta': {'unique_together': "(('name', 'email'),)", 'object_name': 'Professor'}, + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + }, + u'courses.school': { + 'Meta': {'ordering': "['-file_count', '-priority', 'name']", 'object_name': 'School'}, + 'alias': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'facebook_id': ('django.db.models.fields.BigIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'file_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'hashtag': ('django.db.models.fields.CharField', [], {'max_length': '16', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'location': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'priority': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'slug': ('django.db.models.fields.SlugField', [], {'max_length': '150', 'null': 'True'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '511', 'blank': 'True'}), + 'usde_id': ('django.db.models.fields.BigIntegerField', [], {'unique': 'True', 'null': 'True', 'blank': 'True'}) + }, + u'licenses.license': { + 'Meta': {'object_name': 'License'}, + 'html': ('django.db.models.fields.TextField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}) + }, + u'notes.note': { + 'Meta': {'ordering': "['-uploaded_at']", 'unique_together': "(('fp_file', 'upstream_link'),)", 'object_name': 'Note'}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'course': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['courses.Course']"}), + 'flags': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'fp_file': ('django_filepicker.models.FPFileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'gdrive_url': ('django.db.models.fields.URLField', [], {'max_length': '1024', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.GenericIPAddressField', [], {'max_length': '39', 'null': 'True', 'blank': 'True'}), + 'is_hidden': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'license': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['licenses.License']", 'null': 'True', 'blank': 'True'}), + 'mimetype': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '255'}), + 'text': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'thanks': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}), + 'tweeted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'uploaded_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.utcnow', 'null': 'True'}), + 'upstream_link': ('django.db.models.fields.URLField', [], {'max_length': '1024', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']", 'null': 'True', 'on_delete': 'models.SET_NULL', 'blank': 'True'}) + }, + u'quizzes.keyword': { + 'Meta': {'unique_together': "(('word', 'note', 'ranges'),)", 'object_name': 'Keyword'}, + 'definition': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'note': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['notes.Note']", 'null': 'True', 'blank': 'True'}), + 'ranges': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}), + 'timestamp': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.utcnow'}), + 'unreviewed': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'word': ('django.db.models.fields.CharField', [], {'max_length': '1024'}) + }, + u'taggit.tag': { + 'Meta': {'object_name': 'Tag'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}), + 'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '100'}) + }, + u'taggit.taggeditem': { + 'Meta': {'object_name': 'TaggedItem'}, + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_tagged_items'", 'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}), + 'tag': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_items'", 'to': u"orm['taggit.Tag']"}) + } + } + + complete_apps = ['quizzes'] \ No newline at end of file diff --git a/karmaworld/apps/quizzes/models.py b/karmaworld/apps/quizzes/models.py index b618209..c9a89a6 100644 --- a/karmaworld/apps/quizzes/models.py +++ b/karmaworld/apps/quizzes/models.py @@ -8,12 +8,10 @@ from django.db import models class Keyword(models.Model): word = models.CharField(max_length=1024) definition = models.CharField(max_length=2048, blank=True, null=True) - note = models.ForeignKey('notes.Note', blank=True, null=True) - ranges = models.CharField(max_length=1024, blank=True, null=True) - timestamp = models.DateTimeField(default=datetime.datetime.utcnow) + unreviewed = models.BooleanField(default=False) class Meta: unique_together = ('word', 'note', 'ranges') diff --git a/karmaworld/apps/quizzes/tasks.py b/karmaworld/apps/quizzes/tasks.py new file mode 100644 index 0000000..c234add --- /dev/null +++ b/karmaworld/apps/quizzes/tasks.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python +# -*- coding:utf8 -*- +# Copyright (C) 2013 FinalsClub Foundation +from boto.mturk.qualification import PercentAssignmentsApprovedRequirement, Qualifications +from boto.mturk.question import Overview, FormattedContent, QuestionContent, Question, FreeTextAnswer, QuestionForm, \ + AnswerSpecification +from celery import task +from celery.utils.log import get_task_logger +from boto.mturk.connection import MTurkConnection +from django.contrib.sites.models import Site +from django.core.exceptions import ObjectDoesNotExist +from karmaworld.apps.notes.models import Note +from karmaworld.apps.quizzes.models import Keyword + +logger = get_task_logger(__name__) + +HIT_TITLE = 'Choose keywords and descriptions from course notes' +HIT_DESCRIPTION = "Read students' course notes on Karmanotes.org and " \ + "identify 10 keywords along with descriptions of them" +HIT_OVERVIEW_TEMPLATE = \ + '

Go to the page at KarmaNotes.org by clicking the link provided below. ' \ + 'Identify 10 key words (or short phrases) that are the most important to the document ' \ + 'on the page. This requires reading and understanding the document. Write these words in the boxes below. ' \ + 'Then, write definitions or descriptions of these keywords as they are provided in the page. ' \ + 'If the page does not provide a definition or description, leave the box blank. ' \ + 'For example, keywords might be "John Locke," "the Protestant reformation," ' \ + 'or "existentialism." Their respective definitions or descriptions might be "life, ' \ + 'liberty, and property," "schism in Christianity started by Martin Luther," and ' \ + '"existence precedes essence."

' \ + '

Notes link: ' \ + 'http://{domain}{link}

' +HIT_KEYWORDS = 'writing, summary, keywords' +HIT_DURATION = 60 * 60 * 24 * 7 +HIT_REWARD = 1.0 +HIT_PERCENT_APPROVED_REQUIREMENT = PercentAssignmentsApprovedRequirement(comparator='GreaterThan', integer_value=95) +HIT_QUALIFICATION = Qualifications(requirements=[HIT_PERCENT_APPROVED_REQUIREMENT]) + +KEYWORD_FIELDS = [ + ('keyword01', 'Keyword 1'), + ('keyword02', 'Keyword 2'), + ('keyword03', 'Keyword 3'), + ('keyword04', 'Keyword 4'), + ('keyword05', 'Keyword 5'), + ('keyword06', 'Keyword 6'), + ('keyword07', 'Keyword 7'), + ('keyword08', 'Keyword 8'), + ('keyword09', 'Keyword 9'), + ('keyword10', 'Keyword 10'), +] + +DEFINITION_FIELDS = [ + ('definition01', 'Definition 1'), + ('definition02', 'Definition 2'), + ('definition03', 'Definition 3'), + ('definition04', 'Definition 4'), + ('definition05', 'Definition 5'), + ('definition06', 'Definition 6'), + ('definition07', 'Definition 7'), + ('definition08', 'Definition 8'), + ('definition09', 'Definition 9'), + ('definition10', 'Definition 10'), +] + +@task() +def submit_extract_keywords_hit(note): + """Create a Mechanical Turk HIT that asks a worker to + choose keywords and definitions from the given note.""" + + try: + from karmaworld.secret.mturk import AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, MTURK_HOST + except ImportError: + logger.warn('Could not find Mechanical Turk secrets, not running submit_extract_keywords_hit') + return + + connection = MTurkConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, + host=MTURK_HOST) + + overview = Overview() + overview.append(FormattedContent(HIT_OVERVIEW_TEMPLATE.format(domain=Site.objects.get_current(), + link=note.get_absolute_url()))) + + keyword_fta = FreeTextAnswer() + keyword_fta.num_lines = 1 + + definition_fta = FreeTextAnswer() + definition_fta.num_lines = 3 + + question_form = QuestionForm() + question_form.append(overview) + + for i in range(min(len(KEYWORD_FIELDS), len(DEFINITION_FIELDS))): + keyword_content = QuestionContent() + keyword_content.append_field('Title', KEYWORD_FIELDS[i][1]) + keyword_question = Question(identifier=KEYWORD_FIELDS[i][0], + content=keyword_content, + answer_spec=AnswerSpecification(keyword_fta), + is_required=True) + question_form.append(keyword_question) + + definition_content = QuestionContent() + definition_content.append_field('Title', DEFINITION_FIELDS[i][1]) + definition_question = Question(identifier=DEFINITION_FIELDS[i][0], + content=definition_content, + answer_spec=AnswerSpecification(definition_fta), + is_required=False) + question_form.append(definition_question) + + connection.create_hit(questions=question_form, max_assignments=1, + title=HIT_TITLE, description=HIT_DESCRIPTION, + keywords=HIT_KEYWORDS, duration=HIT_DURATION, + reward=HIT_REWARD, qualifications=HIT_QUALIFICATION, + annotation=str(note.id)) + + +@task(name='get_extract_keywords_results') +def get_extract_keywords_results(): + + try: + from karmaworld.secret.mturk import AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, MTURK_HOST + except ImportError: + logger.warn('Could not find Mechanical Turk secrets, not running get_extract_keywords_results') + return + + connection = MTurkConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, + host=MTURK_HOST) + + reviewable_hits = connection.get_reviewable_hits(page_size=100) + for hit in reviewable_hits: + try: + note_id = connection.get_hit(hit.HITId)[0].RequesterAnnotation + except AttributeError: + logger.error('HIT {0} does not have a RequesterAnnotation, ' + 'so we cannot determine which note it references'.format(hit.HITId)) + return + + try: + note = Note.objects.get(id=note_id) + except ObjectDoesNotExist: + logger.error('Could not find note {0} which was referenced by HIT {1}'.format(note_id, hit.HITId)) + return + + answers = {} + assignments = [a for a in connection.get_assignments(hit.HITId) if a.AssignmentStatus == 'Submitted'] + for assignment in assignments: + for question_form_answer in assignment.answers[0]: + answers[question_form_answer.qid] = question_form_answer.fields[0] + + for i in range(min(len(KEYWORD_FIELDS), len(DEFINITION_FIELDS))): + keyword_qid = KEYWORD_FIELDS[i][0] + definition_qid = DEFINITION_FIELDS[i][0] + try: + keyword = answers[keyword_qid] + definition = answers[definition_qid] + Keyword.objects.create(word=keyword, definition=definition, note=note, unreviewed=True) + except KeyError: + pass + + for assignment in assignments: + connection.approve_assignment(assignment.AssignmentId) + + connection.dispose_hit(hit.HITId) diff --git a/karmaworld/secret/mturk.py.example b/karmaworld/secret/mturk.py.example new file mode 100644 index 0000000..1be49e8 --- /dev/null +++ b/karmaworld/secret/mturk.py.example @@ -0,0 +1,7 @@ +#!/usr/bin/env python +# -*- coding:utf8 -*- +# Copyright (C) 2014 FinalsClub Foundation + +AWS_ACCESS_KEY_ID = '' +AWS_SECRET_ACCESS_KEY = '' +MTURK_HOST = 'mechanicalturk.sandbox.amazonaws.com' diff --git a/karmaworld/settings/prod.py b/karmaworld/settings/prod.py index 1367ade..8ded87d 100644 --- a/karmaworld/settings/prod.py +++ b/karmaworld/settings/prod.py @@ -107,6 +107,10 @@ CELERYBEAT_SCHEDULE = { 'task': 'tweet_note', 'schedule': timedelta(minutes=60), }, + 'check-mturk-results': { + 'task': 'get_extract_keywords_results', + 'schedule': timedelta(minutes=20), + }, } CELERY_TIMEZONE = 'UTC' diff --git a/reqs/common.txt b/reqs/common.txt index 55dbbda..feeb14e 100644 --- a/reqs/common.txt +++ b/reqs/common.txt @@ -19,7 +19,7 @@ gdshortener git+https://github.com/flaptor/indextank-py.git git+https://github.com/jhilker/html2text.git django-allauth -boto==2.6.0 +boto django-storages==1.1.4 django-reversion django-ajax-selects -- 2.25.1