Added hashed index, a bit more of geo-indexes, possibility to give _cls and docs

docs/guide/defining-documents.rst

@@ -465,19 +465,26 @@ You can specify indexes on collections to make querying faster. This is done
 by creating a list of index specifications called :attr:`indexes` in the
 :attr:`~mongoengine.Document.meta` dictionary, where an index specification may
 either be a single field name, a tuple containing multiple field names, or a
-dictionary containing a full index definition. A direction may be specified on
+dictionary containing a full index definition.
-fields by prefixing the field name with a **+** (for ascending) or a **-** sign
+
-(for descending). Note that direction only matters on multi-field indexes.
+A direction may be specified on fields by prefixing the field name with a
-Text indexes may be specified by prefixing the field name with a **$**. ::
+**+** (for ascending) or a **-** sign (for descending). Note that direction
+only matters on multi-field indexes. Text indexes may be specified by prefixing
+the field name with a **$**. Hashed indexes may be specified by prefixing
+the field name with a **#**::
 
     class Page(Document):
+        category = IntField()
         title = StringField()
         rating = StringField()
         created = DateTimeField()
         meta = {
             'indexes': [
                 'title',
+                '$title',  # text index
+                '#title',  # hashed index
                 ('title', '-rating'),
+                ('category', '_cls'),
                 {
                     'fields': ['created'],
                     'expireAfterSeconds': 3600
@@ -532,11 +539,14 @@ There are a few top level defaults for all indexes that can be set::
 :attr:`index_background` (Optional)
     Set the default value for if an index should be indexed in the background
 
+:attr:`index_cls` (Optional)
+    A way to turn off a specific index for _cls.
+
 :attr:`index_drop_dups` (Optional)
     Set the default value for if an index should drop duplicates
 
-:attr:`index_cls` (Optional)
+.. note:: Since MongoDB 3.0 drop_dups is not supported anymore. Raises a Warning
-    A way to turn off a specific index for _cls.
+    and has no effect
 
 
 Compound Indexes and Indexing sub documents

mongoengine/base/document.py

@@ -782,7 +782,7 @@ class BaseDocument(object):
         allow_inheritance = cls._meta.get('allow_inheritance',
                                           ALLOW_INHERITANCE)
         include_cls = (allow_inheritance and not spec.get('sparse', False) and
-                       spec.get('cls',  True))
+                       spec.get('cls',  True) and '_cls' not in spec['fields'])
 
         # 733: don't include cls if index_cls is False unless there is an explicit cls with the index
         include_cls = include_cls and (spec.get('cls', False) or cls._meta.get('index_cls', True))
@@ -795,16 +795,25 @@ class BaseDocument(object):
 
             # ASCENDING from +
             # DESCENDING from -
-            # GEO2D from *
             # TEXT from $
+            # HASHED from #
+            # GEOSPHERE from (
+            # GEOHAYSTACK from )
+            # GEO2D from *
             direction = pymongo.ASCENDING
             if key.startswith("-"):
                 direction = pymongo.DESCENDING
-            elif key.startswith("*"):
-                direction = pymongo.GEO2D
             elif key.startswith("$"):
                 direction = pymongo.TEXT
-            if key.startswith(("+", "-", "*", "$")):
+            elif key.startswith("#"):
+                direction = pymongo.HASHED
+            elif key.startswith("("):
+                direction = pymongo.GEOSPHERE
+            elif key.startswith(")"):
+                direction = pymongo.GEOHAYSTACK
+            elif key.startswith("*"):
+                direction = pymongo.GEO2D
+            if key.startswith(("+", "-", "*", "$", "#", "(", ")")):
                 key = key[1:]
 
             # Use real field name, do it manually because we need field
@@ -827,7 +836,8 @@ class BaseDocument(object):
             index_list.append((key, direction))
 
         # Don't add cls to a geo index
-        if include_cls and direction is not pymongo.GEO2D:
+        if include_cls and direction not in (
+                pymongo.GEO2D, pymongo.GEOHAYSTACK, pymongo.GEOSPHERE):
             index_list.insert(0, ('_cls', 1))
 
         if index_list:

mongoengine/document.py

@@ -637,14 +637,19 @@ class Document(BaseDocument):
         :param key_or_list: a single index key or a list of index keys (to
             construct a multi-field index); keys may be prefixed with a **+**
             or a **-** to determine the index ordering
+        :param background: Allows index creation in the background
+        :param drop_dups: Was removed with MongoDB 3. The value will be
+            removed if PyMongo3+ is used
         """
         index_spec = cls._build_index_spec(key_or_list)
         index_spec = index_spec.copy()
         fields = index_spec.pop('fields')
         index_spec['drop_dups'] = drop_dups
+        # TODO: raise warning if dropdups given and remove with PyMongo3+
         index_spec['background'] = background
         index_spec.update(kwargs)
 
+        # TODO: ensure_index is deprecated
         return cls._get_collection().ensure_index(fields, **index_spec)
 
     @classmethod
@@ -688,6 +693,7 @@ class Document(BaseDocument):
                 if 'cls' in opts:
                     del opts['cls']
 
+                # TODO: ensure_index is deprecated in PyMongo 3+ and drop_dups removed
                 collection.ensure_index(fields, background=background,
                                         drop_dups=drop_dups, **opts)
 
@@ -701,6 +707,7 @@ class Document(BaseDocument):
             if 'cls' in index_opts:
                 del index_opts['cls']
 
+            # TODO: ensure_index is deprecated in PyMongo 3+
             collection.ensure_index('_cls', background=background,
                                     **index_opts)
 

tests/document/indexes.py

@@ -275,6 +275,49 @@ class IndexesTest(unittest.TestCase):
         info = [value['key'] for key, value in info.iteritems()]
         self.assertTrue([('current.location.point', '2d')] in info)
 
+    def test_explicit_geosphere_index(self):
+        """Ensure that geosphere indexes work when created via meta[indexes]
+        """
+        class Place(Document):
+            location = DictField()
+            meta = {
+                'allow_inheritance': True,
+                'indexes': [
+                    '(location.point',
+                ]
+            }
+
+        self.assertEqual([{'fields': [('location.point', '2dsphere')]}],
+                         Place._meta['index_specs'])
+
+        Place.ensure_indexes()
+        info = Place._get_collection().index_information()
+        info = [value['key'] for key, value in info.iteritems()]
+        self.assertTrue([('location.point', '2dsphere')] in info)
+
+    def test_explicit_geohaystack_index(self):
+        """Ensure that geohaystack indexes work when created via meta[indexes]
+        """
+        raise SkipTest('GeoHaystack index creation seems broken on PyMongo'
+                       'side, as it requires a bucketSize parameter.')
+
+        class Place(Document):
+            location = DictField()
+            meta = {
+                'allow_inheritance': True,
+                'indexes': [
+                    ')location.point',
+                ]
+            }
+
+        self.assertEqual([{'fields': [('location.point', 'geoHaystack')]}],
+                         Place._meta['index_specs'])
+
+        Place.ensure_indexes()
+        info = Place._get_collection().index_information()
+        info = [value['key'] for key, value in info.iteritems()]
+        self.assertTrue([('location.point', 'geoHaystack')] in info)
+
     def test_dictionary_indexes(self):
         """Ensure that indexes are used when meta[indexes] contains
         dictionaries instead of lists.
@@ -822,6 +865,18 @@ class IndexesTest(unittest.TestCase):
         key = indexes["title_text"]["key"]
         self.assertTrue(('_fts', 'text') in key)
 
+    def test_hashed_indexes(self):
+
+        class Book(Document):
+            ref_id = StringField()
+            meta = {
+                "indexes": ["#ref_id"],
+            }
+
+        indexes = Book.objects._collection.index_information()
+        self.assertTrue("ref_id_hashed" in indexes)
+        self.assertTrue(('ref_id', 'hashed') in indexes["ref_id_hashed"]["key"])
+
     def test_indexes_after_database_drop(self):
         """
         Test to ensure that indexes are re-created on a collection even
@@ -909,6 +964,30 @@ class IndexesTest(unittest.TestCase):
             }
         })
 
+    def test_compound_index_underscore_cls_not_overwritten(self):
+        """
+        Test that the compound index doesn't get another _cls when it is specified
+        """
+        class TestDoc(Document):
+            shard_1 = StringField()
+            txt_1 = StringField()
+
+            meta = {
+                'collection': 'test',
+                'allow_inheritance': True,
+                'sparse': True,
+                'shard_key': 'shard_1',
+                'indexes': [
+                    ('shard_1', '_cls', 'txt_1'),
+                ]
+            }
+
+        TestDoc.drop_collection()
+        TestDoc.ensure_indexes()
+
+        index_info = TestDoc._get_collection().index_information()
+        self.assertTrue('shard_1_1__cls_1_txt_1_1' in index_info)
+
 
 if __name__ == '__main__':
     unittest.main()