From 3357b55fbfd0f4cfa225ecbbd9c4f809e058c733 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sat, 16 Jan 2010 15:35:01 +0000 Subject: [PATCH] Indexing on ListFields now works properly --- mongoengine/base.py | 9 +++++++-- mongoengine/fields.py | 3 +++ mongoengine/queryset.py | 37 ++++++++++++++++++++++++++++--------- tests/document.py | 7 ++++++- 4 files changed, 44 insertions(+), 12 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index cf97f875..62bf80b5 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -11,6 +11,9 @@ class BaseField(object): """A base class for fields in a MongoDB document. Instances of this class may be added to subclasses of `Document` to define a document's schema. """ + + # Fields may have _types inserted into indexes by default + _index_with_types = True def __init__(self, name=None, required=False, default=None, unique=False, unique_with=None, primary_key=False): @@ -172,8 +175,6 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # Apply document-defined meta options meta.update(attrs.get('meta', {})) - meta['indexes'] += base_indexes - # Only simple classes - direct subclasses of Document - may set # allow_inheritance to False if not simple_class and not meta['allow_inheritance']: @@ -186,6 +187,10 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): new_class = super_new(cls, name, bases, attrs) new_class.objects = QuerySetManager() + user_indexes = [QuerySet._build_index_spec(new_class, spec) + for spec in meta['indexes']] + base_indexes + new_class._meta['indexes'] = user_indexes + unique_indexes = [] for field_name, field in new_class._fields.items(): # Generate a list of indexes needed by uniqueness constraints diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 664edb53..13fa6689 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -142,6 +142,9 @@ class ListField(BaseField): of the field to be used as a list in the database. """ + # ListFields cannot be indexed with _types - MongoDB doesn't support this + _index_with_types = False + def __init__(self, field, **kwargs): if not isinstance(field, BaseField): raise ValidationError('Argument to ListField constructor must be ' diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 9c9cc8da..06c82cf9 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -126,14 +126,19 @@ class QuerySet(object): construct a multi-field index); keys may be prefixed with a **+** or a **-** to determine the index ordering """ + index_list = QuerySet._build_index_spec(self._document, key_or_list) + self._collection.ensure_index(index_list) + return self + + @classmethod + def _build_index_spec(cls, doc_cls, key_or_list): + """Build a PyMongo index spec from a MongoEngine index spec. + """ if isinstance(key_or_list, basestring): key_or_list = [key_or_list] index_list = [] - # If _types is being used, prepend it to every specified index - if self._document._meta.get('allow_inheritance'): - index_list.append(('_types', 1)) - + use_types = doc_cls._meta.get('allow_inheritance', True) for key in key_or_list: # Get direction from + or - direction = pymongo.ASCENDING @@ -141,11 +146,24 @@ class QuerySet(object): direction = pymongo.DESCENDING if key.startswith(("+", "-")): key = key[1:] - # Use real field name - key = QuerySet._translate_field_name(self._document, key) + + # Use real field name, do it manually because we need field + # objects for the next part (list field checking) + parts = key.split('.') + fields = QuerySet._lookup_field(doc_cls, parts) + parts = [field.name for field in fields] + key = '.'.join(parts) index_list.append((key, direction)) - self._collection.ensure_index(index_list) - return self + + # Check if a list field is being used, don't use _types if it is + if use_types and not all(f._index_with_types for f in fields): + use_types = False + + # If _types is being used, prepend it to every specified index + if doc_cls._meta.get('allow_inheritance') and use_types: + index_list.insert(0, ('_types', 1)) + + return index_list def __call__(self, *q_objs, **query): """Filter the selected documents by calling the @@ -177,7 +195,8 @@ class QuerySet(object): # Ensure document-defined indexes are created if self._document._meta['indexes']: for key_or_list in self._document._meta['indexes']: - self.ensure_index(key_or_list) + #self.ensure_index(key_or_list) + self._collection.ensure_index(key_or_list) # Ensure indexes created by uniqueness constraints for index in self._document._meta['unique_indexes']: diff --git a/tests/document.py b/tests/document.py index 2aa98811..317472ef 100644 --- a/tests/document.py +++ b/tests/document.py @@ -227,9 +227,11 @@ class DocumentTest(unittest.TestCase): class BlogPost(Document): date = DateTimeField(name='addDate', default=datetime.datetime.now) category = StringField() + tags = ListField(StringField()) meta = { 'indexes': [ '-date', + 'tags', ('category', '-date') ], } @@ -237,7 +239,8 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() info = BlogPost.objects._collection.index_information() - self.assertEqual(len(info), 4) # _id, types, '-date', ('cat', 'date') + # _id, types, '-date', 'tags', ('cat', 'date') + self.assertEqual(len(info), 5) # Indexes are lazy so use list() to perform query list(BlogPost.objects) @@ -245,6 +248,8 @@ class DocumentTest(unittest.TestCase): self.assertTrue([('_types', 1), ('category', 1), ('addDate', -1)] in info.values()) self.assertTrue([('_types', 1), ('addDate', -1)] in info.values()) + # tags is a list field so it shouldn't have _types in the index + self.assertTrue([('tags', 1)] in info.values()) class ExtendedBlogPost(BlogPost): title = StringField()