From 283e92d55d1acc219c1dd1f8f0e374300b6bfdbe Mon Sep 17 00:00:00 2001 From: Matthieu Rigal Date: Thu, 7 May 2015 15:11:33 +0200 Subject: [PATCH] Added hashed index, a bit more of geo-indexes, possibility to give _cls and docs --- docs/guide/defining-documents.rst | 22 ++++++--- mongoengine/base/document.py | 22 ++++++--- mongoengine/document.py | 7 +++ tests/document/indexes.py | 79 +++++++++++++++++++++++++++++++ 4 files changed, 118 insertions(+), 12 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 56370c14..13519a28 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -465,19 +465,26 @@ You can specify indexes on collections to make querying faster. This is done by creating a list of index specifications called :attr:`indexes` in the :attr:`~mongoengine.Document.meta` dictionary, where an index specification may either be a single field name, a tuple containing multiple field names, or a -dictionary containing a full index definition. A direction may be specified on -fields by prefixing the field name with a **+** (for ascending) or a **-** sign -(for descending). Note that direction only matters on multi-field indexes. -Text indexes may be specified by prefixing the field name with a **$**. :: +dictionary containing a full index definition. + +A direction may be specified on fields by prefixing the field name with a +**+** (for ascending) or a **-** sign (for descending). Note that direction +only matters on multi-field indexes. Text indexes may be specified by prefixing +the field name with a **$**. Hashed indexes may be specified by prefixing +the field name with a **#**:: class Page(Document): + category = IntField() title = StringField() rating = StringField() created = DateTimeField() meta = { 'indexes': [ 'title', + '$title', # text index + '#title', # hashed index ('title', '-rating'), + ('category', '_cls'), { 'fields': ['created'], 'expireAfterSeconds': 3600 @@ -532,11 +539,14 @@ There are a few top level defaults for all indexes that can be set:: :attr:`index_background` (Optional) Set the default value for if an index should be indexed in the background +:attr:`index_cls` (Optional) + A way to turn off a specific index for _cls. + :attr:`index_drop_dups` (Optional) Set the default value for if an index should drop duplicates -:attr:`index_cls` (Optional) - A way to turn off a specific index for _cls. +.. note:: Since MongoDB 3.0 drop_dups is not supported anymore. Raises a Warning + and has no effect Compound Indexes and Indexing sub documents diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 3eba16ca..5707992a 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -782,7 +782,7 @@ class BaseDocument(object): allow_inheritance = cls._meta.get('allow_inheritance', ALLOW_INHERITANCE) include_cls = (allow_inheritance and not spec.get('sparse', False) and - spec.get('cls', True)) + spec.get('cls', True) and '_cls' not in spec['fields']) # 733: don't include cls if index_cls is False unless there is an explicit cls with the index include_cls = include_cls and (spec.get('cls', False) or cls._meta.get('index_cls', True)) @@ -795,16 +795,25 @@ class BaseDocument(object): # ASCENDING from + # DESCENDING from - - # GEO2D from * # TEXT from $ + # HASHED from # + # GEOSPHERE from ( + # GEOHAYSTACK from ) + # GEO2D from * direction = pymongo.ASCENDING if key.startswith("-"): direction = pymongo.DESCENDING - elif key.startswith("*"): - direction = pymongo.GEO2D elif key.startswith("$"): direction = pymongo.TEXT - if key.startswith(("+", "-", "*", "$")): + elif key.startswith("#"): + direction = pymongo.HASHED + elif key.startswith("("): + direction = pymongo.GEOSPHERE + elif key.startswith(")"): + direction = pymongo.GEOHAYSTACK + elif key.startswith("*"): + direction = pymongo.GEO2D + if key.startswith(("+", "-", "*", "$", "#", "(", ")")): key = key[1:] # Use real field name, do it manually because we need field @@ -827,7 +836,8 @@ class BaseDocument(object): index_list.append((key, direction)) # Don't add cls to a geo index - if include_cls and direction is not pymongo.GEO2D: + if include_cls and direction not in ( + pymongo.GEO2D, pymongo.GEOHAYSTACK, pymongo.GEOSPHERE): index_list.insert(0, ('_cls', 1)) if index_list: diff --git a/mongoengine/document.py b/mongoengine/document.py index 838feb81..3d919e7d 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -637,14 +637,19 @@ class Document(BaseDocument): :param key_or_list: a single index key or a list of index keys (to construct a multi-field index); keys may be prefixed with a **+** or a **-** to determine the index ordering + :param background: Allows index creation in the background + :param drop_dups: Was removed with MongoDB 3. The value will be + removed if PyMongo3+ is used """ index_spec = cls._build_index_spec(key_or_list) index_spec = index_spec.copy() fields = index_spec.pop('fields') index_spec['drop_dups'] = drop_dups + # TODO: raise warning if dropdups given and remove with PyMongo3+ index_spec['background'] = background index_spec.update(kwargs) + # TODO: ensure_index is deprecated return cls._get_collection().ensure_index(fields, **index_spec) @classmethod @@ -688,6 +693,7 @@ class Document(BaseDocument): if 'cls' in opts: del opts['cls'] + # TODO: ensure_index is deprecated in PyMongo 3+ and drop_dups removed collection.ensure_index(fields, background=background, drop_dups=drop_dups, **opts) @@ -701,6 +707,7 @@ class Document(BaseDocument): if 'cls' in index_opts: del index_opts['cls'] + # TODO: ensure_index is deprecated in PyMongo 3+ collection.ensure_index('_cls', background=background, **index_opts) diff --git a/tests/document/indexes.py b/tests/document/indexes.py index d43b22e5..cecb8bf2 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -275,6 +275,49 @@ class IndexesTest(unittest.TestCase): info = [value['key'] for key, value in info.iteritems()] self.assertTrue([('current.location.point', '2d')] in info) + def test_explicit_geosphere_index(self): + """Ensure that geosphere indexes work when created via meta[indexes] + """ + class Place(Document): + location = DictField() + meta = { + 'allow_inheritance': True, + 'indexes': [ + '(location.point', + ] + } + + self.assertEqual([{'fields': [('location.point', '2dsphere')]}], + Place._meta['index_specs']) + + Place.ensure_indexes() + info = Place._get_collection().index_information() + info = [value['key'] for key, value in info.iteritems()] + self.assertTrue([('location.point', '2dsphere')] in info) + + def test_explicit_geohaystack_index(self): + """Ensure that geohaystack indexes work when created via meta[indexes] + """ + raise SkipTest('GeoHaystack index creation seems broken on PyMongo' + 'side, as it requires a bucketSize parameter.') + + class Place(Document): + location = DictField() + meta = { + 'allow_inheritance': True, + 'indexes': [ + ')location.point', + ] + } + + self.assertEqual([{'fields': [('location.point', 'geoHaystack')]}], + Place._meta['index_specs']) + + Place.ensure_indexes() + info = Place._get_collection().index_information() + info = [value['key'] for key, value in info.iteritems()] + self.assertTrue([('location.point', 'geoHaystack')] in info) + def test_dictionary_indexes(self): """Ensure that indexes are used when meta[indexes] contains dictionaries instead of lists. @@ -822,6 +865,18 @@ class IndexesTest(unittest.TestCase): key = indexes["title_text"]["key"] self.assertTrue(('_fts', 'text') in key) + def test_hashed_indexes(self): + + class Book(Document): + ref_id = StringField() + meta = { + "indexes": ["#ref_id"], + } + + indexes = Book.objects._collection.index_information() + self.assertTrue("ref_id_hashed" in indexes) + self.assertTrue(('ref_id', 'hashed') in indexes["ref_id_hashed"]["key"]) + def test_indexes_after_database_drop(self): """ Test to ensure that indexes are re-created on a collection even @@ -909,6 +964,30 @@ class IndexesTest(unittest.TestCase): } }) + def test_compound_index_underscore_cls_not_overwritten(self): + """ + Test that the compound index doesn't get another _cls when it is specified + """ + class TestDoc(Document): + shard_1 = StringField() + txt_1 = StringField() + + meta = { + 'collection': 'test', + 'allow_inheritance': True, + 'sparse': True, + 'shard_key': 'shard_1', + 'indexes': [ + ('shard_1', '_cls', 'txt_1'), + ] + } + + TestDoc.drop_collection() + TestDoc.ensure_indexes() + + index_info = TestDoc._get_collection().index_information() + self.assertTrue('shard_1_1__cls_1_txt_1_1' in index_info) + if __name__ == '__main__': unittest.main()