pub/mongoengine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into master

Browse Source
This commit is contained in:
erdenezul
2018-05-21 16:22:07 +08:00
committed by GitHub
parent 9f02f71c52 0fffaccdf4
commit 7fb1c9dd35
34 changed files with 1480 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
mongoengine/__init__.py
View File

@@ -23,7 +23,7 @@ __all__ = (list(document.__all__) + list(fields.__all__) +
list(signals.__all__) + list(errors.__all__))
VERSION = (0, 14, 0)
VERSION = (0, 15, 0)
def get_version():

2
mongoengine/base/__init__.py
View File

@@ -15,7 +15,7 @@ __all__ = (
'UPDATE_OPERATORS', '_document_registry', 'get_document',
# datastructures
'BaseDict', 'BaseList', 'EmbeddedDocumentList',
'BaseDict', 'BaseList', 'EmbeddedDocumentList', 'LazyReference',
# document
'BaseDocument',

7
mongoengine/base/common.py
View File

@@ -3,9 +3,10 @@ from mongoengine.errors import NotRegistered
__all__ = ('UPDATE_OPERATORS', 'get_document', '_document_registry')
UPDATE_OPERATORS = set(['set', 'unset', 'inc', 'dec', 'pop', 'push',
'push_all', 'pull', 'pull_all', 'add_to_set',
'set_on_insert', 'min', 'max', 'rename'])
UPDATE_OPERATORS = set(['set', 'unset', 'inc', 'dec', 'mul',
'pop', 'push', 'push_all', 'pull',
'pull_all', 'add_to_set', 'set_on_insert',
'min', 'max', 'rename'])
_document_registry = {}

72
mongoengine/base/datastructures.py
View File

@@ -1,12 +1,13 @@
import itertools
import weakref
from bson import DBRef
import six
from mongoengine.common import _import_class
from mongoengine.errors import DoesNotExist, MultipleObjectsReturned
__all__ = ('BaseDict', 'BaseList', 'EmbeddedDocumentList')
__all__ = ('BaseDict', 'BaseList', 'EmbeddedDocumentList', 'LazyReference')
class BaseDict(dict):
@@ -127,7 +128,7 @@ class BaseList(list):
return value
def __iter__(self):
for i in xrange(self.__len__()):
for i in six.moves.range(self.__len__()):
yield self[i]
def __setitem__(self, key, value, *args, **kwargs):
@@ -350,7 +351,8 @@ class EmbeddedDocumentList(BaseList):
def update(self, **update):
"""
Updates the embedded documents with the given update values.
Updates the embedded documents with the given replacement values. This
function does not support mongoDB update operators such as ``inc__``.
.. note::
The embedded document changes are not automatically saved
@@ -447,40 +449,40 @@ class StrictDict(object):
return cls._classes[allowed_keys]
class SemiStrictDict(StrictDict):
__slots__ = ('_extras', )
_classes = {}
class LazyReference(DBRef):
__slots__ = ('_cached_doc', 'passthrough', 'document_type')
def __getattr__(self, attr):
try:
super(SemiStrictDict, self).__getattr__(attr)
except AttributeError:
try:
return self.__getattribute__('_extras')[attr]
except KeyError as e:
raise AttributeError(e)
def fetch(self, force=False):
if not self._cached_doc or force:
self._cached_doc = self.document_type.objects.get(pk=self.pk)
if not self._cached_doc:
raise DoesNotExist('Trying to dereference unknown document %s' % (self))
return self._cached_doc
def __setattr__(self, attr, value):
try:
super(SemiStrictDict, self).__setattr__(attr, value)
except AttributeError:
try:
self._extras[attr] = value
except AttributeError:
self._extras = {attr: value}
@property
def pk(self):
return self.id
def __delattr__(self, attr):
try:
super(SemiStrictDict, self).__delattr__(attr)
except AttributeError:
try:
del self._extras[attr]
except KeyError as e:
raise AttributeError(e)
def __init__(self, document_type, pk, cached_doc=None, passthrough=False):
self.document_type = document_type
self._cached_doc = cached_doc
self.passthrough = passthrough
super(LazyReference, self).__init__(self.document_type._get_collection_name(), pk)
def __iter__(self):
def __getitem__(self, name):
if not self.passthrough:
raise KeyError()
document = self.fetch()
return document[name]
def __getattr__(self, name):
if not object.__getattribute__(self, 'passthrough'):
raise AttributeError()
document = self.fetch()
try:
extras_iter = iter(self.__getattribute__('_extras'))
except AttributeError:
extras_iter = ()
return itertools.chain(super(SemiStrictDict, self).__iter__(), extras_iter)
return document[name]
except KeyError:
raise AttributeError()
def __repr__(self):
return "<LazyReference(%s, %r)>" % (self.document_type, self.pk)

14
mongoengine/base/document.py
View File

@@ -13,13 +13,14 @@ from mongoengine import signals
from mongoengine.base.common import get_document
from mongoengine.base.datastructures import (BaseDict, BaseList,
EmbeddedDocumentList,
SemiStrictDict, StrictDict)
LazyReference,
StrictDict)
from mongoengine.base.fields import ComplexBaseField
from mongoengine.common import _import_class
from mongoengine.errors import (FieldDoesNotExist, InvalidDocumentError,
LookUpError, OperationError, ValidationError)
__all__ = ('BaseDocument',)
__all__ = ('BaseDocument', 'NON_FIELD_ERRORS')
NON_FIELD_ERRORS = '__all__'
@@ -79,8 +80,7 @@ class BaseDocument(object):
if self.STRICT and not self._dynamic:
self._data = StrictDict.create(allowed_keys=self._fields_ordered)()
else:
self._data = SemiStrictDict.create(
allowed_keys=self._fields_ordered)()
self._data = {}
self._dynamic_fields = SON()
@@ -147,7 +147,7 @@ class BaseDocument(object):
if not hasattr(self, name) and not name.startswith('_'):
DynamicField = _import_class('DynamicField')
field = DynamicField(db_field=name)
field = DynamicField(db_field=name, null=True)
field.name = name
self._dynamic_fields[name] = field
self._fields_ordered += (name,)
@@ -337,7 +337,7 @@ class BaseDocument(object):
value = field.generate()
self._data[field_name] = value
if value is not None:
if (value is not None) or (field.null):
if use_db_field:
data[field.db_field] = value
else:
@@ -489,7 +489,7 @@ class BaseDocument(object):
else:
data = getattr(data, part, None)
if hasattr(data, '_changed_fields'):
if not isinstance(data, LazyReference) and hasattr(data, '_changed_fields'):
if getattr(data, '_is_document', False):
continue

16
mongoengine/connection.py
View File

@@ -28,7 +28,7 @@ _connections = {}
_dbs = {}
def register_connection(alias, name=None, host=None, port=None,
def register_connection(alias, db=None, name=None, host=None, port=None,
read_preference=READ_PREFERENCE,
username=None, password=None,
authentication_source=None,
@@ -39,6 +39,7 @@ def register_connection(alias, name=None, host=None, port=None,
:param alias: the name that will be used to refer to this connection
throughout MongoEngine
:param name: the name of the specific database to use
:param db: the name of the database to use, for compatibility with connect
:param host: the host name of the :program:`mongod` instance to connect to
:param port: the port that the :program:`mongod` instance is running on
:param read_preference: The read preference for the collection
@@ -58,7 +59,7 @@ def register_connection(alias, name=None, host=None, port=None,
.. versionchanged:: 0.10.6 - added mongomock support
"""
conn_settings = {
'name': name or 'test',
'name': name or db or 'test',
'host': host or 'localhost',
'port': port or 27017,
'read_preference': read_preference,
@@ -146,13 +147,14 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False):
raise MongoEngineConnectionError(msg)
def _clean_settings(settings_dict):
irrelevant_fields = set([
'name', 'username', 'password', 'authentication_source',
'authentication_mechanism'
])
# set literal more efficient than calling set function
irrelevant_fields_set = {
'name', 'username', 'password',
'authentication_source', 'authentication_mechanism'
}
return {
k: v for k, v in settings_dict.items()
if k not in irrelevant_fields
if k not in irrelevant_fields_set
}
# Retrieve a copy of the connection settings associated with the requested

11
mongoengine/context_managers.py
View File

@@ -1,9 +1,11 @@
from contextlib import contextmanager
from pymongo.write_concern import WriteConcern
from mongoengine.common import _import_class
from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db
__all__ = ('switch_db', 'switch_collection', 'no_dereference',
'no_sub_classes', 'query_counter')
'no_sub_classes', 'query_counter', 'set_write_concern')
class switch_db(object):
@@ -215,3 +217,10 @@ class query_counter(object):
count = self.db.system.profile.find(ignore_query).count() - self.counter
self.counter += 1
return count
@contextmanager
def set_write_concern(collection, write_concerns):
combined_concerns = dict(collection.write_concern.document.items())
combined_concerns.update(write_concerns)
yield collection.with_options(write_concern=WriteConcern(**combined_concerns))

11
mongoengine/dereference.py
View File

@@ -3,6 +3,7 @@ import six
from mongoengine.base import (BaseDict, BaseList, EmbeddedDocumentList,
TopLevelDocumentMetaclass, get_document)
from mongoengine.base.datastructures import LazyReference
from mongoengine.connection import get_db
from mongoengine.document import Document, EmbeddedDocument
from mongoengine.fields import DictField, ListField, MapField, ReferenceField
@@ -99,7 +100,10 @@ class DeReference(object):
if isinstance(item, (Document, EmbeddedDocument)):
for field_name, field in item._fields.iteritems():
v = item._data.get(field_name, None)
if isinstance(v, DBRef):
if isinstance(v, LazyReference):
# LazyReference inherits DBRef but should not be dereferenced here !
continue
elif isinstance(v, DBRef):
reference_map.setdefault(field.document_type, set()).add(v.id)
elif isinstance(v, (dict, SON)) and '_ref' in v:
reference_map.setdefault(get_document(v['_cls']), set()).add(v['_ref'].id)
@@ -110,6 +114,9 @@ class DeReference(object):
if isinstance(field_cls, (Document, TopLevelDocumentMetaclass)):
key = field_cls
reference_map.setdefault(key, set()).update(refs)
elif isinstance(item, LazyReference):
# LazyReference inherits DBRef but should not be dereferenced here !
continue
elif isinstance(item, DBRef):
reference_map.setdefault(item.collection, set()).add(item.id)
elif isinstance(item, (dict, SON)) and '_ref' in item:
@@ -230,7 +237,7 @@ class DeReference(object):
elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:
item_name = '%s.%s' % (name, k) if name else name
data[k] = self._attach_objects(v, depth - 1, instance=instance, name=item_name)
elif hasattr(v, 'id'):
elif isinstance(v, DBRef) and hasattr(v, 'id'):
data[k] = self.object_map.get((v.collection, v.id), v)
if instance and name:

23
mongoengine/document.py
View File

@@ -280,6 +280,9 @@ class Document(BaseDocument):
elif query[id_field] != self.pk:
raise InvalidQueryError('Invalid document modify query: it must modify only this document.')
# Need to add shard key to query, or you get an error
query.update(self._object_key)
updated = self._qs(**query).modify(new=True, **update)
if updated is None:
return False
@@ -576,7 +579,7 @@ class Document(BaseDocument):
"""Delete the :class:`~mongoengine.Document` from the database. This
will only take effect if the document has been previously saved.
:parm signal_kwargs: (optional) kwargs dictionary to be passed to
:param signal_kwargs: (optional) kwargs dictionary to be passed to
the signal calls.
:param write_concern: Extra keyword arguments are passed down which
will be used as options for the resultant
@@ -702,7 +705,6 @@ class Document(BaseDocument):
obj = obj[0]
else:
raise self.DoesNotExist('Document does not exist')
for field in obj._data:
if not fields or field in fields:
try:
@@ -718,7 +720,9 @@ class Document(BaseDocument):
# i.e. obj.update(unset__field=1) followed by obj.reload()
delattr(self, field)
self._changed_fields = obj._changed_fields
self._changed_fields = list(
set(self._changed_fields) - set(fields)
) if fields else obj._changed_fields
self._created = False
return self
@@ -964,8 +968,16 @@ class Document(BaseDocument):
"""
required = cls.list_indexes()
existing = [info['key']
for info in cls._get_collection().index_information().values()]
existing = []
for info in cls._get_collection().index_information().values():
if '_fts' in info['key'][0]:
index_type = info['key'][0][1]
text_index_fields = info.get('weights').keys()
existing.append(
[(key, index_type) for key in text_index_fields])
else:
existing.append(info['key'])
missing = [index for index in required if index not in existing]
extra = [index for index in existing if index not in required]
@@ -1010,6 +1022,7 @@ class DynamicDocument(Document):
field_name = args[0]
if field_name in self._dynamic_fields:
setattr(self, field_name, None)
self._dynamic_fields[field_name].null = False
else:
super(DynamicDocument, self).__delattr__(*args, **kwargs)

282
mongoengine/fields.py
View File

@@ -26,7 +26,9 @@ except ImportError:
Int64 = long
from mongoengine.base import (BaseDocument, BaseField, ComplexBaseField,
GeoJsonBaseField, ObjectIdField, get_document)
GeoJsonBaseField, LazyReference, ObjectIdField,
get_document)
from mongoengine.common import _import_class
from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db
from mongoengine.document import Document, EmbeddedDocument
from mongoengine.errors import DoesNotExist, InvalidQueryError, ValidationError
@@ -46,6 +48,7 @@ __all__ = (
'GenericEmbeddedDocumentField', 'DynamicField', 'ListField',
'SortedListField', 'EmbeddedDocumentListField', 'DictField',
'MapField', 'ReferenceField', 'CachedReferenceField',
'LazyReferenceField', 'GenericLazyReferenceField',
'GenericReferenceField', 'BinaryField', 'GridFSError', 'GridFSProxy',
'FileField', 'ImageGridFsProxy', 'ImproperlyConfigured', 'ImageField',
'GeoPointField', 'PointField', 'LineStringField', 'PolygonField',
@@ -611,6 +614,7 @@ class EmbeddedDocumentField(BaseField):
"""
def __init__(self, document_type, **kwargs):
# XXX ValidationError raised outside of the "validate" method.
if not (
isinstance(document_type, six.string_types) or
issubclass(document_type, EmbeddedDocument)
@@ -686,16 +690,28 @@ class GenericEmbeddedDocumentField(BaseField):
return value
def validate(self, value, clean=True):
if self.choices and isinstance(value, SON):
for choice in self.choices:
if value['_cls'] == choice._class_name:
return True
if not isinstance(value, EmbeddedDocument):
self.error('Invalid embedded document instance provided to an '
'GenericEmbeddedDocumentField')
value.validate(clean=clean)
def lookup_member(self, member_name):
if self.choices:
for choice in self.choices:
field = choice._fields.get(member_name)
if field:
return field
return None
def to_mongo(self, document, use_db_field=True, fields=None):
if document is None:
return None
data = document.to_mongo(use_db_field, fields)
if '_cls' not in data:
data['_cls'] = document._class_name
@@ -779,6 +795,17 @@ class ListField(ComplexBaseField):
kwargs.setdefault('default', lambda: [])
super(ListField, self).__init__(**kwargs)
def __get__(self, instance, owner):
if instance is None:
# Document class being used rather than a document object
return self
value = instance._data.get(self.name)
LazyReferenceField = _import_class('LazyReferenceField')
GenericLazyReferenceField = _import_class('GenericLazyReferenceField')
if isinstance(self.field, (LazyReferenceField, GenericLazyReferenceField)) and value:
instance._data[self.name] = [self.field.build_lazyref(x) for x in value]
return super(ListField, self).__get__(instance, owner)
def validate(self, value):
"""Make sure that a list of valid fields is being used."""
if (not isinstance(value, (list, tuple, QuerySet)) or
@@ -893,8 +920,11 @@ class DictField(ComplexBaseField):
self.field = field
self._auto_dereference = False
self.basecls = basecls or BaseField
# XXX ValidationError raised outside of the "validate" method.
if not issubclass(self.basecls, BaseField):
self.error('DictField only accepts dict values')
kwargs.setdefault('default', lambda: {})
super(DictField, self).__init__(*args, **kwargs)
@@ -943,6 +973,7 @@ class MapField(DictField):
"""
def __init__(self, field=None, *args, **kwargs):
# XXX ValidationError raised outside of the "validate" method.
if not isinstance(field, BaseField):
self.error('Argument to MapField constructor must be a valid '
'field')
@@ -953,6 +984,15 @@ class ReferenceField(BaseField):
"""A reference to a document that will be automatically dereferenced on
access (lazily).
Note this means you will get a database I/O access everytime you access
this field. This is necessary because the field returns a :class:`~mongoengine.Document`
which precise type can depend of the value of the `_cls` field present in the
document in database.
In short, using this type of field can lead to poor performances (especially
if you access this field only to retrieve it `pk` field which is already
known before dereference). To solve this you should consider using the
:class:`~mongoengine.fields.LazyReferenceField`.
Use the `reverse_delete_rule` to handle what should happen if the document
the field is referencing is deleted. EmbeddedDocuments, DictFields and
MapFields does not support reverse_delete_rule and an `InvalidDocumentError`
@@ -993,6 +1033,7 @@ class ReferenceField(BaseField):
A reference to an abstract document type is always stored as a
:class:`~pymongo.dbref.DBRef`, regardless of the value of `dbref`.
"""
# XXX ValidationError raised outside of the "validate" method.
if (
not isinstance(document_type, six.string_types) and
not issubclass(document_type, Document)
@@ -1047,6 +1088,8 @@ class ReferenceField(BaseField):
if isinstance(document, Document):
# We need the id from the saved object to create the DBRef
id_ = document.pk
# XXX ValidationError raised outside of the "validate" method.
if id_ is None:
self.error('You can only reference documents once they have'
' been saved to the database')
@@ -1086,19 +1129,21 @@ class ReferenceField(BaseField):
return self.to_mongo(value)
def validate(self, value):
if not isinstance(value, (self.document_type, DBRef, ObjectId)):
self.error('A ReferenceField only accepts DBRef, ObjectId or documents')
if not isinstance(value, (self.document_type, LazyReference, DBRef, ObjectId)):
self.error('A ReferenceField only accepts DBRef, LazyReference, ObjectId or documents')
if isinstance(value, Document) and value.id is None:
self.error('You can only reference documents once they have been '
'saved to the database')
if self.document_type._meta.get('abstract') and \
not isinstance(value, self.document_type):
if (
self.document_type._meta.get('abstract') and
not isinstance(value, self.document_type)
):
self.error(
'%s is not an instance of abstract reference type %s' % (
self.document_type._class_name)
self.document_type._class_name
)
)
def lookup_member(self, member_name):
@@ -1121,6 +1166,7 @@ class CachedReferenceField(BaseField):
if fields is None:
fields = []
# XXX ValidationError raised outside of the "validate" method.
if (
not isinstance(document_type, six.string_types) and
not issubclass(document_type, Document)
@@ -1195,6 +1241,7 @@ class CachedReferenceField(BaseField):
id_field_name = self.document_type._meta['id_field']
id_field = self.document_type._fields[id_field_name]
# XXX ValidationError raised outside of the "validate" method.
if isinstance(document, Document):
# We need the id from the saved object to create the DBRef
id_ = document.pk
@@ -1203,7 +1250,6 @@ class CachedReferenceField(BaseField):
' been saved to the database')
else:
self.error('Only accept a document object')
# TODO: should raise here or will fail next statement
value = SON((
('_id', id_field.to_mongo(id_)),
@@ -1221,16 +1267,20 @@ class CachedReferenceField(BaseField):
if value is None:
return None
# XXX ValidationError raised outside of the "validate" method.
if isinstance(value, Document):
if value.pk is None:
self.error('You can only reference documents once they have'
' been saved to the database')
return {'_id': value.pk}
value_dict = {'_id': value.pk}
for field in self.fields:
value_dict.update({field: value[field]})
return value_dict
raise NotImplementedError
def validate(self, value):
if not isinstance(value, self.document_type):
self.error('A CachedReferenceField only accepts documents')
@@ -1263,6 +1313,12 @@ class GenericReferenceField(BaseField):
"""A reference to *any* :class:`~mongoengine.document.Document` subclass
that will be automatically dereferenced on access (lazily).
Note this field works the same way as :class:`~mongoengine.document.ReferenceField`,
doing database I/O access the first time it is accessed (even if it's to access
it ``pk`` or ``id`` field).
To solve this you should consider using the
:class:`~mongoengine.fields.GenericLazyReferenceField`.
.. note ::
* Any documents used as a generic reference must be registered in the
document registry. Importing the model will automatically register
@@ -1285,6 +1341,8 @@ class GenericReferenceField(BaseField):
elif isinstance(choice, type) and issubclass(choice, Document):
self.choices.append(choice._class_name)
else:
# XXX ValidationError raised outside of the "validate"
# method.
self.error('Invalid choices provided: must be a list of'
'Document subclasses and/or six.string_typess')
@@ -1348,6 +1406,7 @@ class GenericReferenceField(BaseField):
# We need the id from the saved object to create the DBRef
id_ = document.id
if id_ is None:
# XXX ValidationError raised outside of the "validate" method.
self.error('You can only reference documents once they have'
' been saved to the database')
else:
@@ -1465,6 +1524,9 @@ class GridFSProxy(object):
else:
return False
def __ne__(self, other):
return not self == other
@property
def fs(self):
if not self._fs:
@@ -2138,3 +2200,201 @@ class MultiPolygonField(GeoJsonBaseField):
.. versionadded:: 0.9
"""
_type = 'MultiPolygon'
class LazyReferenceField(BaseField):
"""A really lazy reference to a document.
Unlike the :class:`~mongoengine.fields.ReferenceField` it will
**not** be automatically (lazily) dereferenced on access.
Instead, access will return a :class:`~mongoengine.base.LazyReference` class
instance, allowing access to `pk` or manual dereference by using
``fetch()`` method.
.. versionadded:: 0.15
"""
def __init__(self, document_type, passthrough=False, dbref=False,
reverse_delete_rule=DO_NOTHING, **kwargs):
"""Initialises the Reference Field.
:param dbref: Store the reference as :class:`~pymongo.dbref.DBRef`
or as the :class:`~pymongo.objectid.ObjectId`.id .
:param reverse_delete_rule: Determines what to do when the referring
object is deleted
:param passthrough: When trying to access unknown fields, the
:class:`~mongoengine.base.datastructure.LazyReference` instance will
automatically call `fetch()` and try to retrive the field on the fetched
document. Note this only work getting field (not setting or deleting).
"""
# XXX ValidationError raised outside of the "validate" method.
if (
not isinstance(document_type, six.string_types) and
not issubclass(document_type, Document)
):
self.error('Argument to LazyReferenceField constructor must be a '
'document class or a string')
self.dbref = dbref
self.passthrough = passthrough
self.document_type_obj = document_type
self.reverse_delete_rule = reverse_delete_rule
super(LazyReferenceField, self).__init__(**kwargs)
@property
def document_type(self):
if isinstance(self.document_type_obj, six.string_types):
if self.document_type_obj == RECURSIVE_REFERENCE_CONSTANT:
self.document_type_obj = self.owner_document
else:
self.document_type_obj = get_document(self.document_type_obj)
return self.document_type_obj
def build_lazyref(self, value):
if isinstance(value, LazyReference):
if value.passthrough != self.passthrough:
value = LazyReference(value.document_type, value.pk, passthrough=self.passthrough)
elif value is not None:
if isinstance(value, self.document_type):
value = LazyReference(self.document_type, value.pk, passthrough=self.passthrough)
elif isinstance(value, DBRef):
value = LazyReference(self.document_type, value.id, passthrough=self.passthrough)
else:
# value is the primary key of the referenced document
value = LazyReference(self.document_type, value, passthrough=self.passthrough)
return value
def __get__(self, instance, owner):
"""Descriptor to allow lazy dereferencing."""
if instance is None:
# Document class being used rather than a document object
return self
value = self.build_lazyref(instance._data.get(self.name))
if value:
instance._data[self.name] = value
return super(LazyReferenceField, self).__get__(instance, owner)
def to_mongo(self, value):
if isinstance(value, LazyReference):
pk = value.pk
elif isinstance(value, self.document_type):
pk = value.pk
elif isinstance(value, DBRef):
pk = value.id
else:
# value is the primary key of the referenced document
pk = value
id_field_name = self.document_type._meta['id_field']
id_field = self.document_type._fields[id_field_name]
pk = id_field.to_mongo(pk)
if self.dbref:
return DBRef(self.document_type._get_collection_name(), pk)
else:
return pk
def validate(self, value):
if isinstance(value, LazyReference):
if value.collection != self.document_type._get_collection_name():
self.error('Reference must be on a `%s` document.' % self.document_type)
pk = value.pk
elif isinstance(value, self.document_type):
pk = value.pk
elif isinstance(value, DBRef):
# TODO: check collection ?
collection = self.document_type._get_collection_name()
if value.collection != collection:
self.error("DBRef on bad collection (must be on `%s`)" % collection)
pk = value.id
else:
# value is the primary key of the referenced document
id_field_name = self.document_type._meta['id_field']
id_field = getattr(self.document_type, id_field_name)
pk = value
try:
id_field.validate(pk)
except ValidationError:
self.error(
"value should be `{0}` document, LazyReference or DBRef on `{0}` "
"or `{0}`'s primary key (i.e. `{1}`)".format(
self.document_type.__name__, type(id_field).__name__))
if pk is None:
self.error('You can only reference documents once they have been '
'saved to the database')
def prepare_query_value(self, op, value):
if value is None:
return None
super(LazyReferenceField, self).prepare_query_value(op, value)
return self.to_mongo(value)
def lookup_member(self, member_name):
return self.document_type._fields.get(member_name)
class GenericLazyReferenceField(GenericReferenceField):
"""A reference to *any* :class:`~mongoengine.document.Document` subclass.
Unlike the :class:`~mongoengine.fields.GenericReferenceField` it will
**not** be automatically (lazily) dereferenced on access.
Instead, access will return a :class:`~mongoengine.base.LazyReference` class
instance, allowing access to `pk` or manual dereference by using
``fetch()`` method.
.. note ::
* Any documents used as a generic reference must be registered in the
document registry. Importing the model will automatically register
it.
* You can use the choices param to limit the acceptable Document types
.. versionadded:: 0.15
"""
def __init__(self, *args, **kwargs):
self.passthrough = kwargs.pop('passthrough', False)
super(GenericLazyReferenceField, self).__init__(*args, **kwargs)
def _validate_choices(self, value):
if isinstance(value, LazyReference):
value = value.document_type._class_name
super(GenericLazyReferenceField, self)._validate_choices(value)
def build_lazyref(self, value):
if isinstance(value, LazyReference):
if value.passthrough != self.passthrough:
value = LazyReference(value.document_type, value.pk, passthrough=self.passthrough)
elif value is not None:
if isinstance(value, (dict, SON)):
value = LazyReference(get_document(value['_cls']), value['_ref'].id, passthrough=self.passthrough)
elif isinstance(value, Document):
value = LazyReference(type(value), value.pk, passthrough=self.passthrough)
return value
def __get__(self, instance, owner):
if instance is None:
return self
value = self.build_lazyref(instance._data.get(self.name))
if value:
instance._data[self.name] = value
return super(GenericLazyReferenceField, self).__get__(instance, owner)
def validate(self, value):
if isinstance(value, LazyReference) and value.pk is None:
self.error('You can only reference documents once they have been'
' saved to the database')
return super(GenericLazyReferenceField, self).validate(value)
def to_mongo(self, document):
if document is None:
return None
if isinstance(document, LazyReference):
return SON((
('_cls', document.document_type._class_name),
('_ref', DBRef(document.document_type._get_collection_name(), document.pk))
))
else:
return super(GenericLazyReferenceField, self).to_mongo(document)

41
mongoengine/queryset/base.py
View File

@@ -18,7 +18,7 @@ from mongoengine import signals
from mongoengine.base import get_document
from mongoengine.common import _import_class
from mongoengine.connection import get_db
from mongoengine.context_managers import switch_db
from mongoengine.context_managers import set_write_concern, switch_db
from mongoengine.errors import (InvalidQueryError, LookUpError,
NotUniqueError, OperationError)
from mongoengine.python_support import IS_PYMONGO_3
@@ -350,11 +350,24 @@ class BaseQuerySet(object):
documents=docs, **signal_kwargs)
raw = [doc.to_mongo() for doc in docs]
with set_write_concern(self._collection, write_concern) as collection:
insert_func = collection.insert_many
if return_one:
raw = raw[0]
insert_func = collection.insert_one
try:
ids = self._collection.insert(raw, **write_concern)
inserted_result = insert_func(raw)
ids = return_one and [inserted_result.inserted_id] or inserted_result.inserted_ids
except pymongo.errors.DuplicateKeyError as err:
message = 'Could not save document (%s)'
raise NotUniqueError(message % six.text_type(err))
except pymongo.errors.BulkWriteError as err:
# inserting documents that already have an _id field will
# give huge performance debt or raise
message = u'Document must not have _id value before bulk write (%s)'
raise NotUniqueError(message % six.text_type(err))
except pymongo.errors.OperationFailure as err:
message = 'Could not save document (%s)'
if re.match('^E1100[01] duplicate key', six.text_type(err)):
@@ -368,7 +381,6 @@ class BaseQuerySet(object):
signals.post_bulk_insert.send(
self._document, documents=docs, loaded=False, **signal_kwargs)
return return_one and ids[0] or ids
documents = self.in_bulk(ids)
results = []
for obj_id in ids:
@@ -486,8 +498,9 @@ class BaseQuerySet(object):
``save(..., write_concern={w: 2, fsync: True}, ...)`` will
wait until at least two servers have recorded the write and
will force an fsync on the primary server.
:param full_result: Return the full result rather than just the number
updated.
:param full_result: Return the full result dictionary rather than just the number
updated, e.g. return
``{'n': 2, 'nModified': 2, 'ok': 1.0, 'updatedExisting': True}``.
:param update: Django-style update keyword arguments
.. versionadded:: 0.2
@@ -510,12 +523,15 @@ class BaseQuerySet(object):
else:
update['$set'] = {'_cls': queryset._document._class_name}
try:
result = queryset._collection.update(query, update, multi=multi,
upsert=upsert, **write_concern)
with set_write_concern(queryset._collection, write_concern) as collection:
update_func = collection.update_one
if multi:
update_func = collection.update_many
result = update_func(query, update, upsert=upsert)
if full_result:
return result
elif result:
return result['n']
elif result.raw_result:
return result.raw_result['n']
except pymongo.errors.DuplicateKeyError as err:
raise NotUniqueError(u'Update failed (%s)' % six.text_type(err))
except pymongo.errors.OperationFailure as err:
@@ -544,10 +560,10 @@ class BaseQuerySet(object):
write_concern=write_concern,
full_result=True, **update)
if atomic_update['updatedExisting']:
if atomic_update.raw_result['updatedExisting']:
document = self.get()
else:
document = self._document.objects.with_id(atomic_update['upserted'])
document = self._document.objects.with_id(atomic_update.upserted_id)
return document
def update_one(self, upsert=False, write_concern=None, **update):
@@ -1578,6 +1594,9 @@ class BaseQuerySet(object):
if self._batch_size is not None:
self._cursor_obj.batch_size(self._batch_size)
if self._comment is not None:
self._cursor_obj.comment(self._comment)
return self._cursor_obj
def __deepcopy__(self, memo):

8
mongoengine/queryset/queryset.py
View File

@@ -1,3 +1,5 @@
import six
from mongoengine.errors import OperationError
from mongoengine.queryset.base import (BaseQuerySet, CASCADE, DENY, DO_NOTHING,
NULLIFY, PULL)
@@ -90,7 +92,7 @@ class QuerySet(BaseQuerySet):
# Raise StopIteration if we already established there were no more
# docs in the db cursor.
if not self._has_more:
raise StopIteration
return
# Otherwise, populate more of the cache and repeat.
if len(self._result_cache) <= pos:
@@ -112,7 +114,7 @@ class QuerySet(BaseQuerySet):
# Pull in ITER_CHUNK_SIZE docs from the database and store them in
# the result cache.
try:
for _ in xrange(ITER_CHUNK_SIZE):
for _ in six.moves.range(ITER_CHUNK_SIZE):
self._result_cache.append(self.next())
except StopIteration:
# Getting this exception means there are no more docs in the
@@ -166,7 +168,7 @@ class QuerySetNoCache(BaseQuerySet):
return '.. queryset mid-iteration ..'
data = []
for _ in xrange(REPR_OUTPUT_SIZE + 1):
for _ in six.moves.range(REPR_OUTPUT_SIZE + 1):
try:
data.append(self.next())
except StopIteration:

76
mongoengine/queryset/transform.py
View File

@@ -101,21 +101,8 @@ def query(_doc_cls=None, **kwargs):
value = value['_id']
elif op in ('in', 'nin', 'all', 'near') and not isinstance(value, dict):
# Raise an error if the in/nin/all/near param is not iterable. We need a
# special check for BaseDocument, because - although it's iterable - using
# it as such in the context of this method is most definitely a mistake.
BaseDocument = _import_class('BaseDocument')
if isinstance(value, BaseDocument):
raise TypeError("When using the `in`, `nin`, `all`, or "
"`near`-operators you can\'t use a "
"`Document`, you must wrap your object "
"in a list (object -> [object]).")
elif not hasattr(value, '__iter__'):
raise TypeError("The `in`, `nin`, `all`, or "
"`near`-operators must be applied to an "
"iterable (e.g. a list).")
else:
value = [field.prepare_query_value(op, v) for v in value]
# Raise an error if the in/nin/all/near param is not iterable.
value = _prepare_query_for_iterable(field, op, value)
# If we're querying a GenericReferenceField, we need to alter the
# key depending on the value:
@@ -284,7 +271,15 @@ def update(_doc_cls=None, **update):
if isinstance(field, GeoJsonBaseField):
value = field.to_mongo(value)
if op in (None, 'set', 'push', 'pull'):
if op == 'pull':
if field.required or value is not None:
if match == 'in' and not isinstance(value, dict):
value = _prepare_query_for_iterable(field, op, value)
else:
value = field.prepare_query_value(op, value)
elif op == 'push' and isinstance(value, (list, tuple, set)):
value = [field.prepare_query_value(op, v) for v in value]
elif op in (None, 'set', 'push'):
if field.required or value is not None:
value = field.prepare_query_value(op, value)
elif op in ('pushAll', 'pullAll'):
@@ -319,11 +314,17 @@ def update(_doc_cls=None, **update):
field_classes = [c.__class__ for c in cleaned_fields]
field_classes.reverse()
ListField = _import_class('ListField')
if ListField in field_classes:
# Join all fields via dot notation to the last ListField
EmbeddedDocumentListField = _import_class('EmbeddedDocumentListField')
if ListField in field_classes or EmbeddedDocumentListField in field_classes:
# Join all fields via dot notation to the last ListField or EmbeddedDocumentListField
# Then process as normal
if ListField in field_classes:
_check_field = ListField
else:
_check_field = EmbeddedDocumentListField
last_listField = len(
cleaned_fields) - field_classes.index(ListField)
cleaned_fields) - field_classes.index(_check_field)
key = '.'.join(parts[:last_listField])
parts = parts[last_listField:]
parts.insert(0, key)
@@ -333,10 +334,26 @@ def update(_doc_cls=None, **update):
value = {key: value}
elif op == 'addToSet' and isinstance(value, list):
value = {key: {'$each': value}}
elif op in ('push', 'pushAll'):
if parts[-1].isdigit():
key = parts[0]
position = int(parts[-1])
# $position expects an iterable. If pushing a single value,
# wrap it in a list.
if not isinstance(value, (set, tuple, list)):
value = [value]
value = {key: {'$each': value, '$position': position}}
else:
if op == 'pushAll':
op = 'push' # convert to non-deprecated keyword
if not isinstance(value, (set, tuple, list)):
value = [value]
value = {key: {'$each': value}}
else:
value = {key: value}
else:
value = {key: value}
key = '$' + op
if key not in mongo_update:
mongo_update[key] = value
elif key in mongo_update and isinstance(mongo_update[key], dict):
@@ -425,3 +442,22 @@ def _infer_geometry(value):
raise InvalidQueryError('Invalid $geometry data. Can be either a '
'dictionary or (nested) lists of coordinate(s)')
def _prepare_query_for_iterable(field, op, value):
# We need a special check for BaseDocument, because - although it's iterable - using
# it as such in the context of this method is most definitely a mistake.
BaseDocument = _import_class('BaseDocument')
if isinstance(value, BaseDocument):
raise TypeError("When using the `in`, `nin`, `all`, or "
"`near`-operators you can\'t use a "
"`Document`, you must wrap your object "
"in a list (object -> [object]).")
if not hasattr(value, '__iter__'):
raise TypeError("The `in`, `nin`, `all`, or "
"`near`-operators must be applied to an "
"iterable (e.g. a list).")
return [field.prepare_query_value(op, v) for v in value]