pub/mongoengine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #994 from MRigal/fix/cls-index-at-desired-position

Browse Source 
Added hashed index, a bit more of geo-indexes, possibility to give _cls
This commit is contained in:
Matthieu Rigal 2015-06-11 14:20:01 +02:00
commit 548a552638
4 changed files with 175 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
docs/guide/defining-documents.rst
View File

@ -465,19 +465,26 @@ You can specify indexes on collections to make querying faster. This is done
by creating a list of index specifications called :attr:`indexes` in the
:attr:`~mongoengine.Document.meta` dictionary, where an index specification may
either be a single field name, a tuple containing multiple field names, or a
dictionary containing a full index definition. A direction may be specified on
fields by prefixing the field name with a **+** (for ascending) or a **-** sign
(for descending). Note that direction only matters on multi-field indexes.
Text indexes may be specified by prefixing the field name with a **$**. ::
dictionary containing a full index definition.
A direction may be specified on fields by prefixing the field name with a
**+** (for ascending) or a **-** sign (for descending). Note that direction
only matters on multi-field indexes. Text indexes may be specified by prefixing
the field name with a **$**. Hashed indexes may be specified by prefixing
the field name with a **#**::
class Page(Document):
category = IntField()
title = StringField()
rating = StringField()
created = DateTimeField()
meta = {
'indexes': [
'title',
'$title', # text index
'#title', # hashed index
('title', '-rating'),
('category', '_cls'),
{
'fields': ['created'],
'expireAfterSeconds': 3600
@ -532,11 +539,14 @@ There are a few top level defaults for all indexes that can be set::
:attr:`index_background` (Optional)
Set the default value for if an index should be indexed in the background
:attr:`index_cls` (Optional)
A way to turn off a specific index for _cls.
:attr:`index_drop_dups` (Optional)
Set the default value for if an index should drop duplicates
:attr:`index_cls` (Optional)
A way to turn off a specific index for _cls.
.. note:: Since MongoDB 3.0 drop_dups is not supported anymore. Raises a Warning
and has no effect
Compound Indexes and Indexing sub documents

22
mongoengine/base/document.py
View File

@ -782,7 +782,7 @@ class BaseDocument(object):
allow_inheritance = cls._meta.get('allow_inheritance',
ALLOW_INHERITANCE)
include_cls = (allow_inheritance and not spec.get('sparse', False) and
spec.get('cls', True))
spec.get('cls', True) and '_cls' not in spec['fields'])
# 733: don't include cls if index_cls is False unless there is an explicit cls with the index
include_cls = include_cls and (spec.get('cls', False) or cls._meta.get('index_cls', True))
@ -795,16 +795,25 @@ class BaseDocument(object):
# ASCENDING from +
# DESCENDING from -
# GEO2D from *
# TEXT from $
# HASHED from #
# GEOSPHERE from (
# GEOHAYSTACK from )
# GEO2D from *
direction = pymongo.ASCENDING
if key.startswith("-"):
direction = pymongo.DESCENDING
elif key.startswith("*"):
direction = pymongo.GEO2D
elif key.startswith("$"):
direction = pymongo.TEXT
if key.startswith(("+", "-", "*", "$")):
elif key.startswith("#"):
direction = pymongo.HASHED
elif key.startswith("("):
direction = pymongo.GEOSPHERE
elif key.startswith(")"):
direction = pymongo.GEOHAYSTACK
elif key.startswith("*"):
direction = pymongo.GEO2D
if key.startswith(("+", "-", "*", "$", "#", "(", ")")):
key = key[1:]
# Use real field name, do it manually because we need field
@ -827,7 +836,8 @@ class BaseDocument(object):
index_list.append((key, direction))
# Don't add cls to a geo index
if include_cls and direction is not pymongo.GEO2D:
if include_cls and direction not in (
pymongo.GEO2D, pymongo.GEOHAYSTACK, pymongo.GEOSPHERE):
index_list.insert(0, ('_cls', 1))
if index_list:

51
mongoengine/document.py
View File

@ -1,4 +1,4 @@
import warnings
import pymongo
import re
@ -17,6 +17,7 @@ from mongoengine.base import (
get_document
)
from mongoengine.errors import InvalidQueryError, InvalidDocumentError
from mongoengine.python_support import IS_PYMONGO_3
from mongoengine.queryset import (OperationError, NotUniqueError,
QuerySet, transform)
from mongoengine.connection import get_db, DEFAULT_CONNECTION_NAME
@ -636,23 +637,51 @@ class Document(BaseDocument):
db.drop_collection(cls._get_collection_name())
@classmethod
def ensure_index(cls, key_or_list, drop_dups=False, background=False,
**kwargs):
"""Ensure that the given indexes are in place.
def create_index(cls, keys, background=False, **kwargs):
"""Creates the given indexes if required.
:param key_or_list: a single index key or a list of index keys (to
:param keys: a single index key or a list of index keys (to
construct a multi-field index); keys may be prefixed with a **+**
or a **-** to determine the index ordering
:param background: Allows index creation in the background
"""
index_spec = cls._build_index_spec(key_or_list)
index_spec = cls._build_index_spec(keys)
index_spec = index_spec.copy()
fields = index_spec.pop('fields')
drop_dups = kwargs.get('drop_dups', False)
if IS_PYMONGO_3 and drop_dups:
msg = "drop_dups is deprecated and is removed when using PyMongo 3+."
warnings.warn(msg, DeprecationWarning)
elif not IS_PYMONGO_3:
index_spec['drop_dups'] = drop_dups
index_spec['background'] = background
index_spec.update(kwargs)
if IS_PYMONGO_3:
return cls._get_collection().create_index(fields, **index_spec)
else:
return cls._get_collection().ensure_index(fields, **index_spec)
@classmethod
def ensure_index(cls, key_or_list, drop_dups=False, background=False,
**kwargs):
"""Ensure that the given indexes are in place. Deprecated in favour
of create_index.
:param key_or_list: a single index key or a list of index keys (to
construct a multi-field index); keys may be prefixed with a **+**
or a **-** to determine the index ordering
:param background: Allows index creation in the background
:param drop_dups: Was removed/ignored with MongoDB >2.7.5. The value
will be removed if PyMongo3+ is used
"""
if IS_PYMONGO_3 and drop_dups:
msg = "drop_dups is deprecated and is removed when using PyMongo 3+."
warnings.warn(msg, DeprecationWarning)
elif not IS_PYMONGO_3:
kwargs.update({'drop_dups': drop_dups})
return cls.create_index(key_or_list, background=background, **kwargs)
@classmethod
def ensure_indexes(cls):
"""Checks the document meta data and ensures all the indexes exist.
@ -666,6 +695,9 @@ class Document(BaseDocument):
drop_dups = cls._meta.get('index_drop_dups', False)
index_opts = cls._meta.get('index_opts') or {}
index_cls = cls._meta.get('index_cls', True)
if IS_PYMONGO_3 and drop_dups:
msg = "drop_dups is deprecated and is removed when using PyMongo 3+."
warnings.warn(msg, DeprecationWarning)
collection = cls._get_collection()
# 746: when connection is via mongos, the read preference is not necessarily an indication that
@ -694,6 +726,9 @@ class Document(BaseDocument):
if 'cls' in opts:
del opts['cls']
if IS_PYMONGO_3:
collection.create_index(fields, background=background, **opts)
else:
collection.ensure_index(fields, background=background,
drop_dups=drop_dups, **opts)
@ -707,6 +742,10 @@ class Document(BaseDocument):
if 'cls' in index_opts:
del index_opts['cls']
if IS_PYMONGO_3:
collection.create_index('_cls', background=background,
**index_opts)
else:
collection.ensure_index('_cls', background=background,
**index_opts)

90
tests/document/indexes.py
View File

@ -275,6 +275,60 @@ class IndexesTest(unittest.TestCase):
info = [value['key'] for key, value in info.iteritems()]
self.assertTrue([('current.location.point', '2d')] in info)
def test_explicit_geosphere_index(self):
"""Ensure that geosphere indexes work when created via meta[indexes]
"""
class Place(Document):
location = DictField()
meta = {
'allow_inheritance': True,
'indexes': [
'(location.point',
]
}
self.assertEqual([{'fields': [('location.point', '2dsphere')]}],
Place._meta['index_specs'])
Place.ensure_indexes()
info = Place._get_collection().index_information()
info = [value['key'] for key, value in info.iteritems()]
self.assertTrue([('location.point', '2dsphere')] in info)
def test_explicit_geohaystack_index(self):
"""Ensure that geohaystack indexes work when created via meta[indexes]
"""
raise SkipTest('GeoHaystack index creation is not supported for now'
'from meta, as it requires a bucketSize parameter.')
class Place(Document):
location = DictField()
name = StringField()
meta = {
'indexes': [
(')location.point', 'name')
]
}
self.assertEqual([{'fields': [('location.point', 'geoHaystack'), ('name', 1)]}],
Place._meta['index_specs'])
Place.ensure_indexes()
info = Place._get_collection().index_information()
info = [value['key'] for key, value in info.iteritems()]
self.assertTrue([('location.point', 'geoHaystack')] in info)
def test_create_geohaystack_index(self):
"""Ensure that geohaystack indexes can be created
"""
class Place(Document):
location = DictField()
name = StringField()
Place.create_index({'fields': (')location.point', 'name')}, bucketSize=10)
info = Place._get_collection().index_information()
info = [value['key'] for key, value in info.iteritems()]
self.assertTrue([('location.point', 'geoHaystack'), ('name', 1)] in info)
def test_dictionary_indexes(self):
"""Ensure that indexes are used when meta[indexes] contains
dictionaries instead of lists.
@ -822,6 +876,18 @@ class IndexesTest(unittest.TestCase):
key = indexes["title_text"]["key"]
self.assertTrue(('_fts', 'text') in key)
def test_hashed_indexes(self):
class Book(Document):
ref_id = StringField()
meta = {
"indexes": ["#ref_id"],
}
indexes = Book.objects._collection.index_information()
self.assertTrue("ref_id_hashed" in indexes)
self.assertTrue(('ref_id', 'hashed') in indexes["ref_id_hashed"]["key"])
def test_indexes_after_database_drop(self):
"""
Test to ensure that indexes are re-created on a collection even
@ -909,6 +975,30 @@ class IndexesTest(unittest.TestCase):
}
})
def test_compound_index_underscore_cls_not_overwritten(self):
"""
Test that the compound index doesn't get another _cls when it is specified
"""
class TestDoc(Document):
shard_1 = StringField()
txt_1 = StringField()
meta = {
'collection': 'test',
'allow_inheritance': True,
'sparse': True,
'shard_key': 'shard_1',
'indexes': [
('shard_1', '_cls', 'txt_1'),
]
}
TestDoc.drop_collection()
TestDoc.ensure_indexes()
index_info = TestDoc._get_collection().index_information()
self.assertTrue('shard_1_1__cls_1_txt_1_1' in index_info)
if __name__ == '__main__':
unittest.main()