From bf508f8e2a7cc6f657aaf031dccc522e7be07b3c Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Mon, 17 Feb 2014 15:53:23 -0500 Subject: [PATCH] If we can't test search indexing properly, let's not test it at all --- karmaworld/apps/notes/gdrive.py | 9 ++++++- karmaworld/apps/notes/search.py | 43 +++++++++++++++++++++++---------- karmaworld/apps/notes/tests.py | 32 ------------------------ karmaworld/apps/users/tests.py | 13 ---------- karmaworld/settings/common.py | 6 +++++ 5 files changed, 44 insertions(+), 59 deletions(-) diff --git a/karmaworld/apps/notes/gdrive.py b/karmaworld/apps/notes/gdrive.py index 2870133..859e1d9 100644 --- a/karmaworld/apps/notes/gdrive.py +++ b/karmaworld/apps/notes/gdrive.py @@ -5,6 +5,7 @@ import datetime import logging from django.contrib.auth.models import User +from django.conf import settings from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned from karmaworld.apps.notes.models import UserUploadMapping from karmaworld.apps.users.models import NoteKarmaEvent @@ -74,6 +75,7 @@ def build_api_service(): return build('drive', 'v2', http=credentials.authorize(httplib2.Http())) + def pdf2html(content): pdf_file = tempfile.NamedTemporaryFile() pdf_file.write(content) @@ -83,8 +85,13 @@ def pdf2html(content): html_file_path = os.path.join(tmp_dir, html_file_name) command = ['pdf2htmlEX', pdf_file.name, html_file_name] - call = subprocess.Popen(command, shell=False, cwd=tmp_dir) + devnull = open('/dev/null', 'w') + if settings.TESTING: + call = subprocess.Popen(command, shell=False, cwd=tmp_dir, stdout=devnull, stderr=devnull) + else: + call = subprocess.Popen(command, shell=False, cwd=tmp_dir) call.wait() + devnull.close() if call.returncode != 0: raise ValueError("PDF file could not be processed") diff --git a/karmaworld/apps/notes/search.py b/karmaworld/apps/notes/search.py index 629b73c..b18ba21 100644 --- a/karmaworld/apps/notes/search.py +++ b/karmaworld/apps/notes/search.py @@ -5,14 +5,18 @@ import calendar import time import uuid +from django.core.exceptions import ImproperlyConfigured import indextank.client as itc +from django.conf import settings import karmaworld.secret.indexden as secret import logging PAGE_SIZE = 10 +MOCK_MODE = settings.TESTING + logging.basicConfig() logger = logging.getLogger(__name__) @@ -52,15 +56,15 @@ class SearchIndex(object): __metaclass__ = Singleton - def __init__(self, testing=False): - if not testing: - self.setup() + def __init__(self): + self.index_name = secret.INDEX - def setup(self, testing=False): - if testing: - self.index_name = uuid.uuid4().hex - else: - self.index_name = secret.INDEX + # If we're in production mode, + # or if we're in testing mode with indexing + # explicity turned on, + # do index setup stuff + if MOCK_MODE: + return self.api_client = itc.ApiClient(secret.PRIVATE_URL) if not self.api_client.get_index(self.index_name).exists(): @@ -78,11 +82,6 @@ class SearchIndex(object): # "Relevance" is a black box provided by IndexDen. self.index.add_function(0, 'relevance * log(doc.var[0])') - def delete_index(self): - """This is meant for test cases that want to clean up - after themselves.""" - self.api_client.delete_index(self.index_name) - @staticmethod def _tags_to_str(tags): return ' '.join([str(tag) for tag in tags.all()]) @@ -105,9 +104,20 @@ class SearchIndex(object): return d + def delete_index(self): + """This is meant for test cases that want to clean up + after themselves.""" + if MOCK_MODE: + return + + self.api_client.delete_index(self.index_name) + def add_note(self, note): """Add a note to the index. If the note is already in the index, it will be overwritten.""" + if MOCK_MODE: + return + if note.text: logger.info("Indexing {n}".format(n=note)) self.index.add_document(note.id, SearchIndex._note_to_dict(note), variables={0: note.thanks}) @@ -118,6 +128,9 @@ class SearchIndex(object): """Update a note. Will only truly update the search index if it needs to. Compares the fields in new_note with old_note to see what has changed.""" + if MOCK_MODE: + return + if not new_note.text: logger.info("Note {n} has no text, will not add to IndexDen".format(n=new_note)) return @@ -144,12 +157,16 @@ class SearchIndex(object): def remove_note(self, note): """Remove a note from the search index.""" + if MOCK_MODE: + return logger.info("Removing from index: {n}".format(n=note)) self.index.delete_document(note.id) def search(self, query, course_id=None, page=0): """Returns an instance of SearchResult for your query.""" + if MOCK_MODE: + raise ImproperlyConfigured("Attempting to use SearchIndex while in test mode.") if course_id: real_query = '("%s" OR name:"%s") AND course_id:%s' % (query, query, course_id) diff --git a/karmaworld/apps/notes/tests.py b/karmaworld/apps/notes/tests.py index c313c8d..9572668 100644 --- a/karmaworld/apps/notes/tests.py +++ b/karmaworld/apps/notes/tests.py @@ -42,24 +42,11 @@ class TestNotes(TestCase): self.note = Note() self.note.course = self.course self.note.name = u"Lecture notes concerning the use of therefore ∴" - #self.note.slug := do not set for test_remake_slug() behavior self.note.file_type = 'doc' self.note.uploaded_at = self.now self.note.text = "This is the plaintext version of a note. It's pretty cool. Alpaca." self.note.save() - @classmethod - def setUpClass(cls): - index = SearchIndex(testing=True) - index.setup(testing=True) - - @classmethod - def tearDownClass(cls): - """Delete the test index that was automatically - created by notes/search.py""" - index = SearchIndex() - index.delete_index() - def test_course_fkey(self): self.assertEqual(self.course, self.note.course) @@ -92,22 +79,3 @@ class TestNotes(TestCase): self.note.slug = None url = self.expected_url_prefix + str(self.note.id) self.assertEqual(self.note.get_absolute_url(), url) - - def test_search_index(self): - """Search for a note within IndexDen""" - - index = SearchIndex() - - # Search for it - results = index.search('alpaca') - self.assertIn(self.note.id, results.ordered_ids) - - # Search for it, filtering by course - results = index.search('alpaca', self.note.course.id) - self.assertIn(self.note.id, results.ordered_ids) - - # Delete the note, see if it's removed from the index - self.note.delete() - results = index.search('alpaca') - self.assertNotIn(self.note.id, results.ordered_ids) - diff --git a/karmaworld/apps/users/tests.py b/karmaworld/apps/users/tests.py index d4e2b11..4d48c8b 100644 --- a/karmaworld/apps/users/tests.py +++ b/karmaworld/apps/users/tests.py @@ -12,7 +12,6 @@ from karmaworld.apps.courses.views import flag_course from karmaworld.apps.notes.models import Note from karmaworld.apps.courses.models import Course from karmaworld.apps.courses.models import School -from karmaworld.apps.notes.search import SearchIndex from karmaworld.apps.notes.views import thank_note, flag_note, downloaded_note from karmaworld.apps.users.models import NoteKarmaEvent, GenericKarmaEvent, CourseKarmaEvent, give_email_confirm_karma @@ -63,18 +62,6 @@ class TestUsers(TestCase): self.request2.META = {'HTTP_X_REQUESTED_WITH': 'XMLHttpRequest'} self.request2.session = SessionStore() - @classmethod - def setUpClass(cls): - index = SearchIndex(testing=True) - index.setup(testing=True) - - @classmethod - def tearDownClass(cls): - """Delete the test index that was automatically - created by notes/search.py""" - index = SearchIndex() - index.delete_index() - def test_thank_own_note_karma(self): """Make sure you don't get karma for thanking your own note""" thank_note(self.request1, self.note.pk) diff --git a/karmaworld/settings/common.py b/karmaworld/settings/common.py index eb8b18d..23c6f34 100644 --- a/karmaworld/settings/common.py +++ b/karmaworld/settings/common.py @@ -5,6 +5,7 @@ from datetime import timedelta +import sys from os.path import abspath, basename, dirname, join, normpath from sys import path @@ -370,3 +371,8 @@ TAGGIT_STOPWORDS = [u'a', u'an', u'and', u'be', u'from', u'of'] ########## END TAGGIT CONFIGURATION +########## TESTING CONFIGURATION +if 'test' in sys.argv: + TESTING = True +########## END TESTING CONFIGURATION + -- 2.25.1