From dcaf5748400e34405a2f3ba453d43126ee3983cc Mon Sep 17 00:00:00 2001 From: "Charles V. Dulac" Date: Sat, 19 Nov 2016 15:30:19 +0900 Subject: [PATCH] Accept string based primary keys [fixes #138] --- tests/xapian_tests/models.py | 17 +++++- tests/xapian_tests/search_indexes.py | 29 ++++++++++ tests/xapian_tests/tests/test_backend.py | 72 +++++++++++++++++++++++- tests/xapian_tests/tests/test_query.py | 4 ++ xapian_backend.py | 15 ++++- 5 files changed, 131 insertions(+), 6 deletions(-) diff --git a/tests/xapian_tests/models.py b/tests/xapian_tests/models.py index 496c6b4..407031d 100644 --- a/tests/xapian_tests/models.py +++ b/tests/xapian_tests/models.py @@ -15,7 +15,7 @@ class Document(models.Model): text = models.TextField() -class BlogEntry(models.Model): +class AbstractBlogEntry(models.Model): """ Same as tests.core.MockModel with a few extra fields for testing various sorting and ordering criteria. @@ -36,6 +36,21 @@ class BlogEntry(models.Model): float_number = models.FloatField() decimal_number = models.DecimalField(max_digits=4, decimal_places=2) + class Meta: + abstract = True + + +class BlogEntry(AbstractBlogEntry): + pass + + +class UUIDBlogEntry(AbstractBlogEntry): + """ + A blog entry with string based primary key instead of an integer. + Covers #138 + """ + uuid = models.CharField(primary_key=True, max_length=20) + class DjangoContentType(models.Model): content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) diff --git a/tests/xapian_tests/search_indexes.py b/tests/xapian_tests/search_indexes.py index a588290..19d5843 100644 --- a/tests/xapian_tests/search_indexes.py +++ b/tests/xapian_tests/search_indexes.py @@ -82,6 +82,35 @@ def prepare_empty(self, obj): return '' +class UUIDBlogSearchIndex(BlogSearchIndex): + + def get_model(self): + return models.UUIDBlogEntry + + def prepare_sites(self, obj): + return ['%d' % (i * int(obj.pk.split('-')[1])) for i in range(1, 4)] + + def prepare_tags(self, obj): + if obj.pk == 'uuid-1': + return ['a', 'b', 'c'] + elif obj.pk == 'uuid-2': + return ['ab', 'bc', 'cd'] + else: + return ['an', 'to', 'or'] + + def prepare_keys(self, obj): + return [i * int(obj.pk.split('-')[1]) for i in range(1, 4)] + + def prepare_titles(self, obj): + if obj.pk == 'uuid-1': + return ['object one title one', 'object one title two'] + elif obj.pk == 'uuid-2': + return ['object two title one', 'object two title two'] + else: + return ['object three title one', 'object three title two'] + + + class CompleteBlogEntryIndex(indexes.SearchIndex): text = indexes.CharField(model_attr='text', document=True) author = indexes.CharField(model_attr='author') diff --git a/tests/xapian_tests/tests/test_backend.py b/tests/xapian_tests/tests/test_backend.py index 1f8f353..367727a 100644 --- a/tests/xapian_tests/tests/test_backend.py +++ b/tests/xapian_tests/tests/test_backend.py @@ -18,8 +18,8 @@ from haystack.utils.loading import UnifiedIndex from ..search_indexes import XapianNGramIndex, XapianEdgeNGramIndex, \ - CompleteBlogEntryIndex, BlogSearchIndex, DjangoContentTypeIndex -from ..models import BlogEntry, AnotherMockModel, MockTag, DjangoContentType + CompleteBlogEntryIndex, BlogSearchIndex, DjangoContentTypeIndex, UUIDBlogSearchIndex +from ..models import BlogEntry, AnotherMockModel, MockTag, UUIDBlogEntry, DjangoContentType XAPIAN_VERSION = [int(x) for x in xapian.__version__.split('.')] @@ -682,6 +682,74 @@ def test_more_like_this_with_unindexed_model(self): self.assertRaises(InvalidIndexError, self.backend.more_like_this, mock) +class StringBasedPKModelBackendFeaturesTestCase(HaystackBackendTestCase, TestCase): + """ + Covers #138, Must not assume django_id is an int + """ + + def get_index(self): + return UUIDBlogSearchIndex() + + @staticmethod + def get_entry(i): + entry = UUIDBlogEntry() + entry.uuid = 'uuid-%s' % i + entry.author = 'david%s' % i + entry.url = 'http://example.com/%d/' % i + entry.boolean = bool(i % 2) + entry.number = i*5 + entry.float_number = i*5.0 + entry.decimal_number = Decimal('22.34') + entry.datetime = ( + datetime.datetime(2009, 2, 25, 1, 1, 1) - datetime.timedelta(seconds=i) + ) + entry.date = datetime.date(2009, 2, 23) + datetime.timedelta(days=i) + return entry + + def setUp(self): + super(StringBasedPKModelBackendFeaturesTestCase, self).setUp() + + self.sample_objs = [] + + for i in range(1, 4): + entry = self.get_entry(i) + self.sample_objs.append(entry) + + self.sample_objs[0].float_number = 834.0 + self.sample_objs[1].float_number = 35.5 + self.sample_objs[2].float_number = 972.0 + for obj in self.sample_objs: + obj.save() + + self.backend.update(self.index, UUIDBlogEntry.objects.all()) + + def test_update(self): + self.assertEqual(pks(self.backend.search(xapian.Query(''))['results']), + ['uuid-1', 'uuid-2', 'uuid-3']) + + def test_order_by_django_id(self): + """ + We need this test because ordering on more than + 10 entries was not correct at some point. + """ + self.sample_objs = [] + pk_list = [] + for i in range(101, 200): + entry = self.get_entry(i) + self.sample_objs.append(entry) + pk_list.append('uuid-%03d' % i) + + for obj in self.sample_objs: + obj.save() + + self.backend.clear() + self.backend.update(self.index, self.sample_objs) + + results = self.backend.search(xapian.Query(''), sort_by=['-django_id']) + self.assertEqual(pks(results['results']), list(reversed(pk_list))) + + + class IndexationNGramTestCase(HaystackBackendTestCase, TestCase): def get_index(self): return XapianNGramIndex() diff --git a/tests/xapian_tests/tests/test_query.py b/tests/xapian_tests/tests/test_query.py index 24d3e1e..2d7c6d4 100644 --- a/tests/xapian_tests/tests/test_query.py +++ b/tests/xapian_tests/tests/test_query.py @@ -335,6 +335,10 @@ def test_multiple_filter_types(self): 'zzzzzzzzzzzzzzzzzzzzzzzzz AND' ' (QQ000000000001 OR QQ000000000002 OR QQ000000000003))') + def test_filter_string_based_django_pk(self): + self.sq.add_filter(SQ(django_id='uuid-1')) + self.assertExpectedQuery(self.sq.build_query(), 'QQuuid-1') + def test_log_query(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) diff --git a/xapian_backend.py b/xapian_backend.py index 3d4f79f..4deb878 100755 --- a/xapian_backend.py +++ b/xapian_backend.py @@ -285,7 +285,7 @@ def update(self, index, iterable): term_generator.set_stemmer(xapian.Stem(self.language)) try: term_generator.set_stemming_strategy(self.stemming_strategy) - except AttributeError: + except AttributeError: # Versions before Xapian 1.2.11 do not support stemming strategies for TermGenerator pass if self.include_spelling is True: @@ -433,7 +433,11 @@ def add_datetime_to_document(termpos, prefix, term, weight): # `django_id` is an int and `django_ct` is text; # besides, they are indexed by their (unstemmed) value. if field['field_name'] == DJANGO_ID: - value = int(value) + try: + value = int(value) + except ValueError: + # Django_id is a string + field['type'] = 'text' value = _term_to_xapian_value(value, field['type']) document.add_term(TERM_PREFIXES[field['field_name']] + value, weight) @@ -1494,7 +1498,12 @@ def _term_query(self, term, field_name, field_type, stemmed=True): if field_name in (ID, DJANGO_ID, DJANGO_CT): # to ensure the value is serialized correctly. if field_name == DJANGO_ID: - term = int(term) + try: + term = int(term) + except ValueError: + # Django_id is a string + field_type = 'text' + term = _term_to_xapian_value(term, field_type) return xapian.Query('%s%s' % (TERM_PREFIXES[field_name], term))