Merge pull request #994 from MRigal/fix/cls-index-at-desired-position

Added hashed index, a bit more of geo-indexes, possibility to give _cls

docs/guide/defining-documents.rst

@@ -465,19 +465,26 @@ You can specify indexes on collections to make querying faster. This is done
 by creating a list of index specifications called :attr:`indexes` in the
 :attr:`~mongoengine.Document.meta` dictionary, where an index specification may
 either be a single field name, a tuple containing multiple field names, or a
-dictionary containing a full index definition. A direction may be specified on
+dictionary containing a full index definition.
-fields by prefixing the field name with a **+** (for ascending) or a **-** sign
+
-(for descending). Note that direction only matters on multi-field indexes.
+A direction may be specified on fields by prefixing the field name with a
-Text indexes may be specified by prefixing the field name with a **$**. ::
+**+** (for ascending) or a **-** sign (for descending). Note that direction
+only matters on multi-field indexes. Text indexes may be specified by prefixing
+the field name with a **$**. Hashed indexes may be specified by prefixing
+the field name with a **#**::
 
     class Page(Document):
+        category = IntField()
         title = StringField()
         rating = StringField()
         created = DateTimeField()
         meta = {
             'indexes': [
                 'title',
+                '$title',  # text index
+                '#title',  # hashed index
                 ('title', '-rating'),
+                ('category', '_cls'),
                 {
                     'fields': ['created'],
                     'expireAfterSeconds': 3600
@@ -532,11 +539,14 @@ There are a few top level defaults for all indexes that can be set::
 :attr:`index_background` (Optional)
     Set the default value for if an index should be indexed in the background
 
+:attr:`index_cls` (Optional)
+    A way to turn off a specific index for _cls.
+
 :attr:`index_drop_dups` (Optional)
     Set the default value for if an index should drop duplicates
 
-:attr:`index_cls` (Optional)
+.. note:: Since MongoDB 3.0 drop_dups is not supported anymore. Raises a Warning
-    A way to turn off a specific index for _cls.
+    and has no effect
 
 
 Compound Indexes and Indexing sub documents

mongoengine/base/document.py

@@ -782,7 +782,7 @@ class BaseDocument(object):
         allow_inheritance = cls._meta.get('allow_inheritance',
                                           ALLOW_INHERITANCE)
         include_cls = (allow_inheritance and not spec.get('sparse', False) and
-                       spec.get('cls',  True))
+                       spec.get('cls',  True) and '_cls' not in spec['fields'])
 
         # 733: don't include cls if index_cls is False unless there is an explicit cls with the index
         include_cls = include_cls and (spec.get('cls', False) or cls._meta.get('index_cls', True))
@@ -795,16 +795,25 @@ class BaseDocument(object):
 
             # ASCENDING from +
             # DESCENDING from -
-            # GEO2D from *
             # TEXT from $
+            # HASHED from #
+            # GEOSPHERE from (
+            # GEOHAYSTACK from )
+            # GEO2D from *
             direction = pymongo.ASCENDING
             if key.startswith("-"):
                 direction = pymongo.DESCENDING
-            elif key.startswith("*"):
-                direction = pymongo.GEO2D
             elif key.startswith("$"):
                 direction = pymongo.TEXT
-            if key.startswith(("+", "-", "*", "$")):
+            elif key.startswith("#"):
+                direction = pymongo.HASHED
+            elif key.startswith("("):
+                direction = pymongo.GEOSPHERE
+            elif key.startswith(")"):
+                direction = pymongo.GEOHAYSTACK
+            elif key.startswith("*"):
+                direction = pymongo.GEO2D
+            if key.startswith(("+", "-", "*", "$", "#", "(", ")")):
                 key = key[1:]
 
             # Use real field name, do it manually because we need field
@@ -827,7 +836,8 @@ class BaseDocument(object):
             index_list.append((key, direction))
 
         # Don't add cls to a geo index
-        if include_cls and direction is not pymongo.GEO2D:
+        if include_cls and direction not in (
+                pymongo.GEO2D, pymongo.GEOHAYSTACK, pymongo.GEOSPHERE):
             index_list.insert(0, ('_cls', 1))
 
         if index_list:

mongoengine/document.py

@@ -1,4 +1,4 @@
-
+import warnings
 import pymongo
 import re
 
@@ -17,6 +17,7 @@ from mongoengine.base import (
     get_document
 )
 from mongoengine.errors import InvalidQueryError, InvalidDocumentError
+from mongoengine.python_support import IS_PYMONGO_3
 from mongoengine.queryset import (OperationError, NotUniqueError,
                                   QuerySet, transform)
 from mongoengine.connection import get_db, DEFAULT_CONNECTION_NAME
@@ -635,23 +636,51 @@ class Document(BaseDocument):
         db = cls._get_db()
         db.drop_collection(cls._get_collection_name())
 
+    @classmethod
+    def create_index(cls, keys, background=False, **kwargs):
+        """Creates the given indexes if required.
+
+        :param keys: a single index key or a list of index keys (to
+            construct a multi-field index); keys may be prefixed with a **+**
+            or a **-** to determine the index ordering
+        :param background: Allows index creation in the background
+        """
+        index_spec = cls._build_index_spec(keys)
+        index_spec = index_spec.copy()
+        fields = index_spec.pop('fields')
+        drop_dups = kwargs.get('drop_dups', False)
+        if IS_PYMONGO_3 and drop_dups:
+            msg = "drop_dups is deprecated and is removed when using PyMongo 3+."
+            warnings.warn(msg, DeprecationWarning)
+        elif not IS_PYMONGO_3:
+            index_spec['drop_dups'] = drop_dups
+        index_spec['background'] = background
+        index_spec.update(kwargs)
+
+        if IS_PYMONGO_3:
+            return cls._get_collection().create_index(fields, **index_spec)
+        else:
+            return cls._get_collection().ensure_index(fields, **index_spec)
+
     @classmethod
     def ensure_index(cls, key_or_list, drop_dups=False, background=False,
                      **kwargs):
-        """Ensure that the given indexes are in place.
+        """Ensure that the given indexes are in place. Deprecated in favour
+        of create_index.
 
         :param key_or_list: a single index key or a list of index keys (to
             construct a multi-field index); keys may be prefixed with a **+**
             or a **-** to determine the index ordering
+        :param background: Allows index creation in the background
+        :param drop_dups: Was removed/ignored with MongoDB >2.7.5. The value
+            will be removed if PyMongo3+ is used
         """
-        index_spec = cls._build_index_spec(key_or_list)
+        if IS_PYMONGO_3 and drop_dups:
-        index_spec = index_spec.copy()
+            msg = "drop_dups is deprecated and is removed when using PyMongo 3+."
-        fields = index_spec.pop('fields')
+            warnings.warn(msg, DeprecationWarning)
-        index_spec['drop_dups'] = drop_dups
+        elif not IS_PYMONGO_3:
-        index_spec['background'] = background
+            kwargs.update({'drop_dups': drop_dups})
-        index_spec.update(kwargs)
+        return cls.create_index(key_or_list, background=background, **kwargs)
-
-        return cls._get_collection().ensure_index(fields, **index_spec)
 
     @classmethod
     def ensure_indexes(cls):
@@ -666,6 +695,9 @@ class Document(BaseDocument):
         drop_dups = cls._meta.get('index_drop_dups', False)
         index_opts = cls._meta.get('index_opts') or {}
         index_cls = cls._meta.get('index_cls', True)
+        if IS_PYMONGO_3 and drop_dups:
+            msg = "drop_dups is deprecated and is removed when using PyMongo 3+."
+            warnings.warn(msg, DeprecationWarning)
 
         collection = cls._get_collection()
         # 746: when connection is via mongos, the read preference is not necessarily an indication that
@@ -694,8 +726,11 @@ class Document(BaseDocument):
                 if 'cls' in opts:
                     del opts['cls']
 
-                collection.ensure_index(fields, background=background,
+                if IS_PYMONGO_3:
-                                        drop_dups=drop_dups, **opts)
+                    collection.create_index(fields, background=background, **opts)
+                else:
+                    collection.ensure_index(fields, background=background,
+                                            drop_dups=drop_dups, **opts)
 
         # If _cls is being used (for polymorphism), it needs an index,
         # only if another index doesn't begin with _cls
@@ -707,8 +742,12 @@ class Document(BaseDocument):
             if 'cls' in index_opts:
                 del index_opts['cls']
 
-            collection.ensure_index('_cls', background=background,
+            if IS_PYMONGO_3:
-                                    **index_opts)
+                collection.create_index('_cls', background=background,
+                                        **index_opts)
+            else:
+                collection.ensure_index('_cls', background=background,
+                                        **index_opts)
 
     @classmethod
     def list_indexes(cls, go_up=True, go_down=True):

tests/document/indexes.py

@@ -275,6 +275,60 @@ class IndexesTest(unittest.TestCase):
         info = [value['key'] for key, value in info.iteritems()]
         self.assertTrue([('current.location.point', '2d')] in info)
 
+    def test_explicit_geosphere_index(self):
+        """Ensure that geosphere indexes work when created via meta[indexes]
+        """
+        class Place(Document):
+            location = DictField()
+            meta = {
+                'allow_inheritance': True,
+                'indexes': [
+                    '(location.point',
+                ]
+            }
+
+        self.assertEqual([{'fields': [('location.point', '2dsphere')]}],
+                         Place._meta['index_specs'])
+
+        Place.ensure_indexes()
+        info = Place._get_collection().index_information()
+        info = [value['key'] for key, value in info.iteritems()]
+        self.assertTrue([('location.point', '2dsphere')] in info)
+
+    def test_explicit_geohaystack_index(self):
+        """Ensure that geohaystack indexes work when created via meta[indexes]
+        """
+        raise SkipTest('GeoHaystack index creation is not supported for now'
+                       'from meta, as it requires a bucketSize parameter.')
+
+        class Place(Document):
+            location = DictField()
+            name = StringField()
+            meta = {
+                'indexes': [
+                    (')location.point', 'name')
+                ]
+            }
+        self.assertEqual([{'fields': [('location.point', 'geoHaystack'), ('name', 1)]}],
+                         Place._meta['index_specs'])
+
+        Place.ensure_indexes()
+        info = Place._get_collection().index_information()
+        info = [value['key'] for key, value in info.iteritems()]
+        self.assertTrue([('location.point', 'geoHaystack')] in info)
+
+    def test_create_geohaystack_index(self):
+        """Ensure that geohaystack indexes can be created
+        """
+        class Place(Document):
+            location = DictField()
+            name = StringField()
+
+        Place.create_index({'fields': (')location.point', 'name')}, bucketSize=10)
+        info = Place._get_collection().index_information()
+        info = [value['key'] for key, value in info.iteritems()]
+        self.assertTrue([('location.point', 'geoHaystack'), ('name', 1)] in info)
+
     def test_dictionary_indexes(self):
         """Ensure that indexes are used when meta[indexes] contains
         dictionaries instead of lists.
@@ -822,6 +876,18 @@ class IndexesTest(unittest.TestCase):
         key = indexes["title_text"]["key"]
         self.assertTrue(('_fts', 'text') in key)
 
+    def test_hashed_indexes(self):
+
+        class Book(Document):
+            ref_id = StringField()
+            meta = {
+                "indexes": ["#ref_id"],
+            }
+
+        indexes = Book.objects._collection.index_information()
+        self.assertTrue("ref_id_hashed" in indexes)
+        self.assertTrue(('ref_id', 'hashed') in indexes["ref_id_hashed"]["key"])
+
     def test_indexes_after_database_drop(self):
         """
         Test to ensure that indexes are re-created on a collection even
@@ -909,6 +975,30 @@ class IndexesTest(unittest.TestCase):
             }
         })
 
+    def test_compound_index_underscore_cls_not_overwritten(self):
+        """
+        Test that the compound index doesn't get another _cls when it is specified
+        """
+        class TestDoc(Document):
+            shard_1 = StringField()
+            txt_1 = StringField()
+
+            meta = {
+                'collection': 'test',
+                'allow_inheritance': True,
+                'sparse': True,
+                'shard_key': 'shard_1',
+                'indexes': [
+                    ('shard_1', '_cls', 'txt_1'),
+                ]
+            }
+
+        TestDoc.drop_collection()
+        TestDoc.ensure_indexes()
+
+        index_info = TestDoc._get_collection().index_information()
+        self.assertTrue('shard_1_1__cls_1_txt_1_1' in index_info)
+
 
 if __name__ == '__main__':
     unittest.main()