Indexing on ListFields now works properly

mongoengine/base.py

@@ -12,6 +12,9 @@ class BaseField(object):
     may be added to subclasses of `Document` to define a document's schema.
     """
 
+    # Fields may have _types inserted into indexes by default 
+    _index_with_types = True
+    
     def __init__(self, name=None, required=False, default=None, unique=False,
                  unique_with=None, primary_key=False):
         self.name = name if not primary_key else '_id'
@@ -172,8 +175,6 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
         # Apply document-defined meta options
         meta.update(attrs.get('meta', {}))
 
-        meta['indexes'] += base_indexes
-        
         # Only simple classes - direct subclasses of Document - may set
         # allow_inheritance to False
         if not simple_class and not meta['allow_inheritance']:
@@ -186,6 +187,10 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
         new_class = super_new(cls, name, bases, attrs)
         new_class.objects = QuerySetManager()
 
+        user_indexes = [QuerySet._build_index_spec(new_class, spec)
+                        for spec in meta['indexes']] + base_indexes
+        new_class._meta['indexes'] = user_indexes
+        
         unique_indexes = []
         for field_name, field in new_class._fields.items():
             # Generate a list of indexes needed by uniqueness constraints

mongoengine/fields.py

@@ -142,6 +142,9 @@ class ListField(BaseField):
     of the field to be used as a list in the database.
     """
 
+    # ListFields cannot be indexed with _types - MongoDB doesn't support this
+    _index_with_types = False
+
     def __init__(self, field, **kwargs):
         if not isinstance(field, BaseField):
             raise ValidationError('Argument to ListField constructor must be '

mongoengine/queryset.py

@@ -126,14 +126,19 @@ class QuerySet(object):
             construct a multi-field index); keys may be prefixed with a **+**
             or a **-** to determine the index ordering
         """
+        index_list = QuerySet._build_index_spec(self._document, key_or_list)
+        self._collection.ensure_index(index_list)
+        return self
+
+    @classmethod
+    def _build_index_spec(cls, doc_cls, key_or_list):
+        """Build a PyMongo index spec from a MongoEngine index spec.
+        """
         if isinstance(key_or_list, basestring):
             key_or_list = [key_or_list]
 
         index_list = []
-        # If _types is being used, prepend it to every specified index
+        use_types = doc_cls._meta.get('allow_inheritance', True)
-        if self._document._meta.get('allow_inheritance'):
-            index_list.append(('_types', 1))
-
         for key in key_or_list:
             # Get direction from + or -
             direction = pymongo.ASCENDING
@@ -141,11 +146,24 @@ class QuerySet(object):
                 direction = pymongo.DESCENDING
             if key.startswith(("+", "-")):
                     key = key[1:]
-            # Use real field name
+
-            key = QuerySet._translate_field_name(self._document, key)
+            # Use real field name, do it manually because we need field
+            # objects for the next part (list field checking)
+            parts = key.split('.')
+            fields = QuerySet._lookup_field(doc_cls, parts)
+            parts = [field.name for field in fields]
+            key = '.'.join(parts)
             index_list.append((key, direction))
-        self._collection.ensure_index(index_list)
+
-        return self
+            # Check if a list field is being used, don't use _types if it is
+            if use_types and not all(f._index_with_types for f in fields):
+                use_types = False
+
+        # If _types is being used, prepend it to every specified index
+        if doc_cls._meta.get('allow_inheritance') and use_types:
+            index_list.insert(0, ('_types', 1))
+
+        return index_list
 
     def __call__(self, *q_objs, **query):
         """Filter the selected documents by calling the 
@@ -177,7 +195,8 @@ class QuerySet(object):
             # Ensure document-defined indexes are created
             if self._document._meta['indexes']:
                 for key_or_list in self._document._meta['indexes']:
-                    self.ensure_index(key_or_list)
+                    #self.ensure_index(key_or_list)
+                    self._collection.ensure_index(key_or_list)
 
             # Ensure indexes created by uniqueness constraints
             for index in self._document._meta['unique_indexes']:

tests/document.py

@@ -227,9 +227,11 @@ class DocumentTest(unittest.TestCase):
         class BlogPost(Document):
             date = DateTimeField(name='addDate', default=datetime.datetime.now)
             category = StringField()
+            tags = ListField(StringField())
             meta = {
                 'indexes': [
                     '-date', 
+                    'tags',
                     ('category', '-date')
                 ],
             }
@@ -237,7 +239,8 @@ class DocumentTest(unittest.TestCase):
         BlogPost.drop_collection()
 
         info = BlogPost.objects._collection.index_information()
-        self.assertEqual(len(info), 4) # _id, types, '-date', ('cat', 'date')
+        # _id, types, '-date', 'tags', ('cat', 'date')
+        self.assertEqual(len(info), 5) 
 
         # Indexes are lazy so use list() to perform query
         list(BlogPost.objects)
@@ -245,6 +248,8 @@ class DocumentTest(unittest.TestCase):
         self.assertTrue([('_types', 1), ('category', 1), ('addDate', -1)] 
                         in info.values())
         self.assertTrue([('_types', 1), ('addDate', -1)] in info.values())
+        # tags is a list field so it shouldn't have _types in the index
+        self.assertTrue([('tags', 1)] in info.values())
         
         class ExtendedBlogPost(BlogPost):
             title = StringField()