Merge branch 'master' into limit_behaviour

mongoengine/__init__.py

@@ -23,7 +23,7 @@ __all__ = (list(document.__all__) + list(fields.__all__) +
            list(signals.__all__) + list(errors.__all__))
 
 
-VERSION = (0, 14, 0)
+VERSION = (0, 15, 3)
 
 
 def get_version():

mongoengine/base/__init__.py

@@ -15,7 +15,7 @@ __all__ = (
     'UPDATE_OPERATORS', '_document_registry', 'get_document',
 
     # datastructures
-    'BaseDict', 'BaseList', 'EmbeddedDocumentList',
+    'BaseDict', 'BaseList', 'EmbeddedDocumentList', 'LazyReference',
 
     # document
     'BaseDocument',

mongoengine/base/common.py

@@ -3,9 +3,10 @@ from mongoengine.errors import NotRegistered
 __all__ = ('UPDATE_OPERATORS', 'get_document', '_document_registry')
 
 
-UPDATE_OPERATORS = set(['set', 'unset', 'inc', 'dec', 'pop', 'push',
-                        'push_all', 'pull', 'pull_all', 'add_to_set',
-                        'set_on_insert', 'min', 'max', 'rename'])
+UPDATE_OPERATORS = set(['set', 'unset', 'inc', 'dec', 'mul',
+                        'pop', 'push', 'push_all', 'pull',
+                        'pull_all', 'add_to_set', 'set_on_insert',
+                        'min', 'max', 'rename'])
 
 
 _document_registry = {}

mongoengine/base/datastructures.py

@@ -1,12 +1,13 @@
 import itertools
 import weakref
 
+from bson import DBRef
 import six
 
 from mongoengine.common import _import_class
 from mongoengine.errors import DoesNotExist, MultipleObjectsReturned
 
-__all__ = ('BaseDict', 'BaseList', 'EmbeddedDocumentList')
+__all__ = ('BaseDict', 'BaseList', 'EmbeddedDocumentList', 'LazyReference')
 
 
 class BaseDict(dict):
@@ -127,8 +128,8 @@ class BaseList(list):
         return value
 
     def __iter__(self):
-        for i in xrange(self.__len__()):
-            yield self[i]
+        for v in super(BaseList, self).__iter__():
+            yield v
 
     def __setitem__(self, key, value, *args, **kwargs):
         if isinstance(key, slice):
@@ -137,7 +138,7 @@ class BaseList(list):
             self._mark_as_changed(key)
         return super(BaseList, self).__setitem__(key, value)
 
-    def __delitem__(self, key, *args, **kwargs):
+    def __delitem__(self, key):
         self._mark_as_changed()
         return super(BaseList, self).__delitem__(key)
 
@@ -186,7 +187,7 @@ class BaseList(list):
         self._mark_as_changed()
         return super(BaseList, self).remove(*args, **kwargs)
 
-    def reverse(self, *args, **kwargs):
+    def reverse(self):
         self._mark_as_changed()
         return super(BaseList, self).reverse()
 
@@ -233,6 +234,9 @@ class EmbeddedDocumentList(BaseList):
         Filters the list by only including embedded documents with the
         given keyword arguments.
 
+        This method only supports simple comparison (e.g: .filter(name='John Doe'))
+        and does not support operators like __gte, __lte, __icontains like queryset.filter does
+
         :param kwargs: The keyword arguments corresponding to the fields to
          filter on. *Multiple arguments are treated as if they are ANDed
          together.*
@@ -350,7 +354,8 @@ class EmbeddedDocumentList(BaseList):
 
     def update(self, **update):
         """
-        Updates the embedded documents with the given update values.
+        Updates the embedded documents with the given replacement values. This
+        function does not support mongoDB update operators such as ``inc__``.
 
         .. note::
             The embedded document changes are not automatically saved
@@ -447,40 +452,40 @@ class StrictDict(object):
         return cls._classes[allowed_keys]
 
 
-class SemiStrictDict(StrictDict):
-    __slots__ = ('_extras', )
-    _classes = {}
+class LazyReference(DBRef):
+    __slots__ = ('_cached_doc', 'passthrough', 'document_type')
 
-    def __getattr__(self, attr):
-        try:
-            super(SemiStrictDict, self).__getattr__(attr)
-        except AttributeError:
-            try:
-                return self.__getattribute__('_extras')[attr]
-            except KeyError as e:
-                raise AttributeError(e)
+    def fetch(self, force=False):
+        if not self._cached_doc or force:
+            self._cached_doc = self.document_type.objects.get(pk=self.pk)
+            if not self._cached_doc:
+                raise DoesNotExist('Trying to dereference unknown document %s' % (self))
+        return self._cached_doc
 
-    def __setattr__(self, attr, value):
-        try:
-            super(SemiStrictDict, self).__setattr__(attr, value)
-        except AttributeError:
-            try:
-                self._extras[attr] = value
-            except AttributeError:
-                self._extras = {attr: value}
+    @property
+    def pk(self):
+        return self.id
 
-    def __delattr__(self, attr):
-        try:
-            super(SemiStrictDict, self).__delattr__(attr)
-        except AttributeError:
-            try:
-                del self._extras[attr]
-            except KeyError as e:
-                raise AttributeError(e)
+    def __init__(self, document_type, pk, cached_doc=None, passthrough=False):
+        self.document_type = document_type
+        self._cached_doc = cached_doc
+        self.passthrough = passthrough
+        super(LazyReference, self).__init__(self.document_type._get_collection_name(), pk)
 
-    def __iter__(self):
+    def __getitem__(self, name):
+        if not self.passthrough:
+            raise KeyError()
+        document = self.fetch()
+        return document[name]
+
+    def __getattr__(self, name):
+        if not object.__getattribute__(self, 'passthrough'):
+            raise AttributeError()
+        document = self.fetch()
         try:
-            extras_iter = iter(self.__getattribute__('_extras'))
-        except AttributeError:
-            extras_iter = ()
-        return itertools.chain(super(SemiStrictDict, self).__iter__(), extras_iter)
+            return document[name]
+        except KeyError:
+            raise AttributeError()
+
+    def __repr__(self):
+        return "<LazyReference(%s, %r)>" % (self.document_type, self.pk)

mongoengine/base/document.py

@@ -13,13 +13,14 @@ from mongoengine import signals
 from mongoengine.base.common import get_document
 from mongoengine.base.datastructures import (BaseDict, BaseList,
                                              EmbeddedDocumentList,
-                                             SemiStrictDict, StrictDict)
+                                             LazyReference,
+                                             StrictDict)
 from mongoengine.base.fields import ComplexBaseField
 from mongoengine.common import _import_class
 from mongoengine.errors import (FieldDoesNotExist, InvalidDocumentError,
                                 LookUpError, OperationError, ValidationError)
 
-__all__ = ('BaseDocument',)
+__all__ = ('BaseDocument', 'NON_FIELD_ERRORS')
 
 NON_FIELD_ERRORS = '__all__'
 
@@ -79,8 +80,7 @@ class BaseDocument(object):
         if self.STRICT and not self._dynamic:
             self._data = StrictDict.create(allowed_keys=self._fields_ordered)()
         else:
-            self._data = SemiStrictDict.create(
-                allowed_keys=self._fields_ordered)()
+            self._data = {}
 
         self._dynamic_fields = SON()
 
@@ -100,13 +100,11 @@ class BaseDocument(object):
             for key, value in values.iteritems():
                 if key in self._fields or key == '_id':
                     setattr(self, key, value)
-                elif self._dynamic:
+                else:
                     dynamic_data[key] = value
         else:
             FileField = _import_class('FileField')
             for key, value in values.iteritems():
-                if key == '__auto_convert':
-                    continue
                 key = self._reverse_db_field_map.get(key, key)
                 if key in self._fields or key in ('id', 'pk', '_cls'):
                     if __auto_convert and value is not None:
@@ -147,7 +145,7 @@ class BaseDocument(object):
 
             if not hasattr(self, name) and not name.startswith('_'):
                 DynamicField = _import_class('DynamicField')
-                field = DynamicField(db_field=name)
+                field = DynamicField(db_field=name, null=True)
                 field.name = name
                 self._dynamic_fields[name] = field
                 self._fields_ordered += (name,)
@@ -337,7 +335,7 @@ class BaseDocument(object):
                 value = field.generate()
                 self._data[field_name] = value
 
-            if value is not None:
+            if (value is not None) or (field.null):
                 if use_db_field:
                     data[field.db_field] = value
                 else:
@@ -406,7 +404,15 @@ class BaseDocument(object):
 
     @classmethod
     def from_json(cls, json_data, created=False):
-        """Converts json data to an unsaved document instance"""
+        """Converts json data to a Document instance
+
+        :param json_data: The json data to load into the Document
+        :param created: If True, the document will be considered as a brand new document
+                        If False and an id is provided, it will consider that the data being
+                        loaded corresponds to what's already in the database (This has an impact of subsequent call to .save())
+                        If False and no id is provided, it will consider the data as a new document
+                        (default ``False``)
+        """
         return cls._from_son(json_util.loads(json_data), created=created)
 
     def __expand_dynamic_values(self, name, value):
@@ -489,7 +495,7 @@ class BaseDocument(object):
                 else:
                     data = getattr(data, part, None)
 
-                if hasattr(data, '_changed_fields'):
+                if not isinstance(data, LazyReference) and hasattr(data, '_changed_fields'):
                     if getattr(data, '_is_document', False):
                         continue
 
@@ -1080,5 +1086,11 @@ class BaseDocument(object):
         """Return the display value for a choice field"""
         value = getattr(self, field.name)
         if field.choices and isinstance(field.choices[0], (list, tuple)):
-            return dict(field.choices).get(value, value)
+            if value is None:
+                return None
+            sep = getattr(field, 'display_sep', ' ')
+            values = value if field.__class__.__name__ in ('ListField', 'SortedListField') else [value]
+            return sep.join([
+                dict(field.choices).get(val, val)
+                for val in values or []])
         return value

mongoengine/base/fields.py

@@ -55,7 +55,7 @@ class BaseField(object):
             field.  Generally this is deprecated in favour of the
             `FIELD.validate` method
         :param choices: (optional) The valid choices
-        :param null: (optional) Is the field value can be null. If no and there is a default value
+        :param null: (optional) If the field value can be null. If no and there is a default value
             then the default value is set
         :param sparse: (optional) `sparse=True` combined with `unique=True` and `required=False`
             means that uniqueness won't be enforced for `None` values
@@ -130,7 +130,6 @@ class BaseField(object):
     def __set__(self, instance, value):
         """Descriptor for assigning a value to a field in a document.
         """
-
         # If setting to None and there is a default
         # Then set the value to the default value
         if value is None:
@@ -213,8 +212,10 @@ class BaseField(object):
                     )
                 )
         # Choices which are types other than Documents
-        elif value not in choice_list:
-            self.error('Value must be one of %s' % six.text_type(choice_list))
+        else:
+            values = value if isinstance(value, (list, tuple)) else [value]
+            if len(set(values) - set(choice_list)):
+                self.error('Value must be one of %s' % six.text_type(choice_list))
 
     def _validate(self, value, **kwargs):
         # Check the Choices Constraint

mongoengine/base/utils.py

@@ -0,0 +1,22 @@
+import re
+
+
+class LazyRegexCompiler(object):
+    """Descriptor to allow lazy compilation of regex"""
+
+    def __init__(self, pattern, flags=0):
+        self._pattern = pattern
+        self._flags = flags
+        self._compiled_regex = None
+
+    @property
+    def compiled_regex(self):
+        if self._compiled_regex is None:
+            self._compiled_regex = re.compile(self._pattern, self._flags)
+        return self._compiled_regex
+
+    def __get__(self, obj, objtype):
+        return self.compiled_regex
+
+    def __set__(self, instance, value):
+        raise AttributeError("Can not set attribute LazyRegexCompiler")

mongoengine/connection.py

@@ -28,7 +28,7 @@ _connections = {}
 _dbs = {}
 
 
-def register_connection(alias, name=None, host=None, port=None,
+def register_connection(alias, db=None, name=None, host=None, port=None,
                         read_preference=READ_PREFERENCE,
                         username=None, password=None,
                         authentication_source=None,
@@ -39,6 +39,7 @@ def register_connection(alias, name=None, host=None, port=None,
     :param alias: the name that will be used to refer to this connection
         throughout MongoEngine
     :param name: the name of the specific database to use
+    :param db: the name of the database to use, for compatibility with connect
     :param host: the host name of the :program:`mongod` instance to connect to
     :param port: the port that the :program:`mongod` instance is running on
     :param read_preference: The read preference for the collection
@@ -58,7 +59,7 @@ def register_connection(alias, name=None, host=None, port=None,
     .. versionchanged:: 0.10.6 - added mongomock support
     """
     conn_settings = {
-        'name': name or 'test',
+        'name': name or db or 'test',
         'host': host or 'localhost',
         'port': port or 27017,
         'read_preference': read_preference,
@@ -103,6 +104,18 @@ def register_connection(alias, name=None, host=None, port=None,
                 conn_settings['authentication_source'] = uri_options['authsource']
             if 'authmechanism' in uri_options:
                 conn_settings['authentication_mechanism'] = uri_options['authmechanism']
+            if IS_PYMONGO_3 and 'readpreference' in uri_options:
+                read_preferences = (
+                    ReadPreference.NEAREST,
+                    ReadPreference.PRIMARY,
+                    ReadPreference.PRIMARY_PREFERRED,
+                    ReadPreference.SECONDARY,
+                    ReadPreference.SECONDARY_PREFERRED)
+                read_pf_mode = uri_options['readpreference'].lower()
+                for preference in read_preferences:
+                    if preference.name.lower() == read_pf_mode:
+                        conn_settings['read_preference'] = preference
+                        break
         else:
             resolved_hosts.append(entity)
     conn_settings['host'] = resolved_hosts

mongoengine/context_managers.py

@@ -1,9 +1,11 @@
+from contextlib import contextmanager
+from pymongo.write_concern import WriteConcern
 from mongoengine.common import _import_class
 from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db
 
 
 __all__ = ('switch_db', 'switch_collection', 'no_dereference',
-           'no_sub_classes', 'query_counter')
+           'no_sub_classes', 'query_counter', 'set_write_concern')
 
 
 class switch_db(object):
@@ -143,66 +145,83 @@ class no_sub_classes(object):
         :param cls: the class to turn querying sub classes on
         """
         self.cls = cls
+        self.cls_initial_subclasses = None
 
     def __enter__(self):
         """Change the objects default and _auto_dereference values."""
-        self.cls._all_subclasses = self.cls._subclasses
-        self.cls._subclasses = (self.cls,)
+        self.cls_initial_subclasses = self.cls._subclasses
+        self.cls._subclasses = (self.cls._class_name,)
         return self.cls
 
     def __exit__(self, t, value, traceback):
         """Reset the default and _auto_dereference values."""
-        self.cls._subclasses = self.cls._all_subclasses
-        delattr(self.cls, '_all_subclasses')
-        return self.cls
+        self.cls._subclasses = self.cls_initial_subclasses
 
 
 class query_counter(object):
-    """Query_counter context manager to get the number of queries."""
+    """Query_counter context manager to get the number of queries.
+    This works by updating the `profiling_level` of the database so that all queries get logged,
+    resetting the db.system.profile collection at the beginnig of the context and counting the new entries.
+
+    This was designed for debugging purpose. In fact it is a global counter so queries issued by other threads/processes
+    can interfere with it
+
+    Be aware that:
+    - Iterating over large amount of documents (>101) makes pymongo issue `getmore` queries to fetch the next batch of
+        documents (https://docs.mongodb.com/manual/tutorial/iterate-a-cursor/#cursor-batches)
+    - Some queries are ignored by default by the counter (killcursors, db.system.indexes)
+    """
 
     def __init__(self):
-        """Construct the query_counter."""
-        self.counter = 0
+        """Construct the query_counter
+        """
         self.db = get_db()
+        self.initial_profiling_level = None
+        self._ctx_query_counter = 0             # number of queries issued by the context
 
-    def __enter__(self):
-        """On every with block we need to drop the profile collection."""
+        self._ignored_query = {
+            'ns':
+                {'$ne': '%s.system.indexes' % self.db.name},
+            'op':
+                {'$ne': 'killcursors'}
+        }
+
+    def _turn_on_profiling(self):
+        self.initial_profiling_level = self.db.profiling_level()
         self.db.set_profiling_level(0)
         self.db.system.profile.drop()
         self.db.set_profiling_level(2)
+
+    def _resets_profiling(self):
+        self.db.set_profiling_level(self.initial_profiling_level)
+
+    def __enter__(self):
+        self._turn_on_profiling()
         return self
 
     def __exit__(self, t, value, traceback):
-        """Reset the profiling level."""
-        self.db.set_profiling_level(0)
+        self._resets_profiling()
 
     def __eq__(self, value):
-        """== Compare querycounter."""
         counter = self._get_count()
         return value == counter
 
     def __ne__(self, value):
-        """!= Compare querycounter."""
         return not self.__eq__(value)
 
     def __lt__(self, value):
-        """< Compare querycounter."""
         return self._get_count() < value
 
     def __le__(self, value):
-        """<= Compare querycounter."""
         return self._get_count() <= value
 
     def __gt__(self, value):
-        """> Compare querycounter."""
         return self._get_count() > value
 
     def __ge__(self, value):
-        """>= Compare querycounter."""
         return self._get_count() >= value
 
     def __int__(self):
-        """int representation."""
         return self._get_count()
 
     def __repr__(self):
@@ -210,8 +229,17 @@ class query_counter(object):
         return u"%s" % self._get_count()
 
     def _get_count(self):
-        """Get the number of queries."""
-        ignore_query = {'ns': {'$ne': '%s.system.indexes' % self.db.name}}
-        count = self.db.system.profile.find(ignore_query).count() - self.counter
-        self.counter += 1
+        """Get the number of queries by counting the current number of entries in db.system.profile
+        and substracting the queries issued by this context. In fact everytime this is called, 1 query is
+        issued so we need to balance that
+        """
+        count = self.db.system.profile.find(self._ignored_query).count() - self._ctx_query_counter
+        self._ctx_query_counter += 1    # Account for the query we just issued to gather the information
         return count
+
+
+@contextmanager
+def set_write_concern(collection, write_concerns):
+    combined_concerns = dict(collection.write_concern.document.items())
+    combined_concerns.update(write_concerns)
+    yield collection.with_options(write_concern=WriteConcern(**combined_concerns))

mongoengine/dereference.py

@@ -3,6 +3,7 @@ import six
 
 from mongoengine.base import (BaseDict, BaseList, EmbeddedDocumentList,
                               TopLevelDocumentMetaclass, get_document)
+from mongoengine.base.datastructures import LazyReference
 from mongoengine.connection import get_db
 from mongoengine.document import Document, EmbeddedDocument
 from mongoengine.fields import DictField, ListField, MapField, ReferenceField
@@ -99,7 +100,10 @@ class DeReference(object):
             if isinstance(item, (Document, EmbeddedDocument)):
                 for field_name, field in item._fields.iteritems():
                     v = item._data.get(field_name, None)
-                    if isinstance(v, DBRef):
+                    if isinstance(v, LazyReference):
+                        # LazyReference inherits DBRef but should not be dereferenced here !
+                        continue
+                    elif isinstance(v, DBRef):
                         reference_map.setdefault(field.document_type, set()).add(v.id)
                     elif isinstance(v, (dict, SON)) and '_ref' in v:
                         reference_map.setdefault(get_document(v['_cls']), set()).add(v['_ref'].id)
@@ -110,6 +114,9 @@ class DeReference(object):
                             if isinstance(field_cls, (Document, TopLevelDocumentMetaclass)):
                                 key = field_cls
                             reference_map.setdefault(key, set()).update(refs)
+            elif isinstance(item, LazyReference):
+                # LazyReference inherits DBRef but should not be dereferenced here !
+                continue
             elif isinstance(item, DBRef):
                 reference_map.setdefault(item.collection, set()).add(item.id)
             elif isinstance(item, (dict, SON)) and '_ref' in item:
@@ -126,7 +133,12 @@ class DeReference(object):
         """
         object_map = {}
         for collection, dbrefs in self.reference_map.iteritems():
-            if hasattr(collection, 'objects'):  # We have a document class for the refs
+
+            # we use getattr instead of hasattr because as hasattr swallows any exception under python2
+            # so it could hide nasty things without raising exceptions (cfr bug #1688))
+            ref_document_cls_exists = (getattr(collection, 'objects', None) is not None)
+
+            if ref_document_cls_exists:
                 col_name = collection._get_collection_name()
                 refs = [dbref for dbref in dbrefs
                         if (col_name, dbref) not in object_map]
@@ -230,7 +242,7 @@ class DeReference(object):
             elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:
                 item_name = '%s.%s' % (name, k) if name else name
                 data[k] = self._attach_objects(v, depth - 1, instance=instance, name=item_name)
-            elif hasattr(v, 'id'):
+            elif isinstance(v, DBRef) and hasattr(v, 'id'):
                 data[k] = self.object_map.get((v.collection, v.id), v)
 
         if instance and name:

mongoengine/document.py

@@ -195,7 +195,10 @@ class Document(BaseDocument):
 
             # Ensure indexes on the collection unless auto_create_index was
             # set to False.
-            if cls._meta.get('auto_create_index', True):
+            # Also there is no need to ensure indexes on slave.
+            db = cls._get_db()
+            if cls._meta.get('auto_create_index', True) and\
+                    db.client.is_primary:
                 cls.ensure_indexes()
 
         return cls._collection
@@ -280,6 +283,9 @@ class Document(BaseDocument):
         elif query[id_field] != self.pk:
             raise InvalidQueryError('Invalid document modify query: it must modify only this document.')
 
+        # Need to add shard key to query, or you get an error
+        query.update(self._object_key)
+
         updated = self._qs(**query).modify(new=True, **update)
         if updated is None:
             return False
@@ -576,12 +582,11 @@ class Document(BaseDocument):
         """Delete the :class:`~mongoengine.Document` from the database. This
         will only take effect if the document has been previously saved.
 
-        :parm signal_kwargs: (optional) kwargs dictionary to be passed to
+        :param signal_kwargs: (optional) kwargs dictionary to be passed to
             the signal calls.
         :param write_concern: Extra keyword arguments are passed down which
-            will be used as options for the resultant
-            ``getLastError`` command.  For example,
-            ``save(..., write_concern={w: 2, fsync: True}, ...)`` will
+            will be used as options for the resultant ``getLastError`` command.
+            For example, ``save(..., w: 2, fsync: True)`` will
             wait until at least two servers have recorded the write and
             will force an fsync on the primary server.
 
@@ -702,7 +707,6 @@ class Document(BaseDocument):
             obj = obj[0]
         else:
             raise self.DoesNotExist('Document does not exist')
-
         for field in obj._data:
             if not fields or field in fields:
                 try:
@@ -710,7 +714,7 @@ class Document(BaseDocument):
                 except (KeyError, AttributeError):
                     try:
                         # If field is a special field, e.g. items is stored as _reserved_items,
-                        # an KeyError is thrown. So try to retrieve the field from _data
+                        # a KeyError is thrown. So try to retrieve the field from _data
                         setattr(self, field, self._reload(field, obj._data.get(field)))
                     except KeyError:
                         # If field is removed from the database while the object
@@ -718,7 +722,9 @@ class Document(BaseDocument):
                         # i.e. obj.update(unset__field=1) followed by obj.reload()
                         delattr(self, field)
 
-        self._changed_fields = obj._changed_fields
+        self._changed_fields = list(
+            set(self._changed_fields) - set(fields)
+        ) if fields else obj._changed_fields
         self._created = False
         return self
 
@@ -964,8 +970,16 @@ class Document(BaseDocument):
         """
 
         required = cls.list_indexes()
-        existing = [info['key']
-                    for info in cls._get_collection().index_information().values()]
+
+        existing = []
+        for info in cls._get_collection().index_information().values():
+            if '_fts' in info['key'][0]:
+                index_type = info['key'][0][1]
+                text_index_fields = info.get('weights').keys()
+                existing.append(
+                    [(key, index_type) for key in text_index_fields])
+            else:
+                existing.append(info['key'])
         missing = [index for index in required if index not in existing]
         extra = [index for index in existing if index not in required]
 
@@ -985,7 +999,7 @@ class Document(BaseDocument):
 class DynamicDocument(Document):
     """A Dynamic Document class allowing flexible, expandable and uncontrolled
     schemas.  As a :class:`~mongoengine.Document` subclass, acts in the same
-    way as an ordinary document but has expando style properties.  Any data
+    way as an ordinary document but has expanded style properties.  Any data
     passed or set against the :class:`~mongoengine.DynamicDocument` that is
     not a field is automatically converted into a
     :class:`~mongoengine.fields.DynamicField` and data can be attributed to that
@@ -1010,6 +1024,7 @@ class DynamicDocument(Document):
         field_name = args[0]
         if field_name in self._dynamic_fields:
             setattr(self, field_name, None)
+            self._dynamic_fields[field_name].null = False
         else:
             super(DynamicDocument, self).__delattr__(*args, **kwargs)
 

mongoengine/fields.py

@@ -5,7 +5,6 @@ import re
 import socket
 import time
 import uuid
-import warnings
 from operator import itemgetter
 
 from bson import Binary, DBRef, ObjectId, SON
@@ -26,7 +25,10 @@ except ImportError:
     Int64 = long
 
 from mongoengine.base import (BaseDocument, BaseField, ComplexBaseField,
-                              GeoJsonBaseField, ObjectIdField, get_document)
+                              GeoJsonBaseField, LazyReference, ObjectIdField,
+                              get_document)
+from mongoengine.base.utils import LazyRegexCompiler
+from mongoengine.common import _import_class
 from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db
 from mongoengine.document import Document, EmbeddedDocument
 from mongoengine.errors import DoesNotExist, InvalidQueryError, ValidationError
@@ -41,11 +43,12 @@ except ImportError:
 
 __all__ = (
     'StringField', 'URLField', 'EmailField', 'IntField', 'LongField',
-    'FloatField', 'DecimalField', 'BooleanField', 'DateTimeField',
+    'FloatField', 'DecimalField', 'BooleanField', 'DateTimeField', 'DateField',
     'ComplexDateTimeField', 'EmbeddedDocumentField', 'ObjectIdField',
     'GenericEmbeddedDocumentField', 'DynamicField', 'ListField',
     'SortedListField', 'EmbeddedDocumentListField', 'DictField',
     'MapField', 'ReferenceField', 'CachedReferenceField',
+    'LazyReferenceField', 'GenericLazyReferenceField',
     'GenericReferenceField', 'BinaryField', 'GridFSError', 'GridFSProxy',
     'FileField', 'ImageGridFsProxy', 'ImproperlyConfigured', 'ImageField',
     'GeoPointField', 'PointField', 'LineStringField', 'PolygonField',
@@ -120,7 +123,7 @@ class URLField(StringField):
     .. versionadded:: 0.3
     """
 
-    _URL_REGEX = re.compile(
+    _URL_REGEX = LazyRegexCompiler(
         r'^(?:[a-z0-9\.\-]*)://'  # scheme is validated separately
         r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}(?<!-)\.?)|'  # domain...
         r'localhost|'  # localhost...
@@ -154,7 +157,7 @@ class EmailField(StringField):
 
     .. versionadded:: 0.4
     """
-    USER_REGEX = re.compile(
+    USER_REGEX = LazyRegexCompiler(
         # `dot-atom` defined in RFC 5322 Section 3.2.3.
         r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*\Z"
         # `quoted-string` defined in RFC 5322 Section 3.2.4.
@@ -162,7 +165,7 @@ class EmailField(StringField):
         re.IGNORECASE
     )
 
-    UTF8_USER_REGEX = re.compile(
+    UTF8_USER_REGEX = LazyRegexCompiler(
         six.u(
             # RFC 6531 Section 3.3 extends `atext` (used by dot-atom) to
             # include `UTF8-non-ascii`.
@@ -172,7 +175,7 @@ class EmailField(StringField):
         ), re.IGNORECASE | re.UNICODE
     )
 
-    DOMAIN_REGEX = re.compile(
+    DOMAIN_REGEX = LazyRegexCompiler(
         r'((?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+)(?:[A-Z0-9-]{2,63}(?<!-))\Z',
         re.IGNORECASE
     )
@@ -361,7 +364,8 @@ class FloatField(BaseField):
 
 
 class DecimalField(BaseField):
-    """Fixed-point decimal number field.
+    """Fixed-point decimal number field. Stores the value as a float by default unless `force_string` is used.
+    If using floats, beware of Decimal to float conversion (potential precision loss)
 
     .. versionchanged:: 0.8
     .. versionadded:: 0.3
@@ -372,7 +376,9 @@ class DecimalField(BaseField):
         """
         :param min_value: Validation rule for the minimum acceptable value.
         :param max_value: Validation rule for the maximum acceptable value.
-        :param force_string: Store as a string.
+        :param force_string: Store the value as a string (instead of a float).
+         Be aware that this affects query sorting and operation like lte, gte (as string comparison is applied)
+         and some query operator won't work (e.g: inc, dec)
         :param precision: Number of decimal places to store.
         :param rounding: The rounding rule from the python decimal library:
 
@@ -459,6 +465,8 @@ class DateTimeField(BaseField):
     installed you can utilise it to convert varying types of date formats into valid
     python datetime objects.
 
+    Note: To default the field to the current datetime, use: DateTimeField(default=datetime.utcnow)
+
     Note: Microseconds are rounded to the nearest millisecond.
       Pre UTC microsecond support is effectively broken.
       Use :class:`~mongoengine.fields.ComplexDateTimeField` if you
@@ -522,6 +530,22 @@ class DateTimeField(BaseField):
         return super(DateTimeField, self).prepare_query_value(op, self.to_mongo(value))
 
 
+class DateField(DateTimeField):
+    def to_mongo(self, value):
+        value = super(DateField, self).to_mongo(value)
+        # drop hours, minutes, seconds
+        if isinstance(value, datetime.datetime):
+            value = datetime.datetime(value.year, value.month, value.day)
+        return value
+
+    def to_python(self, value):
+        value = super(DateField, self).to_python(value)
+        # convert datetime to date
+        if isinstance(value, datetime.datetime):
+            value = datetime.date(value.year, value.month, value.day)
+        return value
+
+
 class ComplexDateTimeField(StringField):
     """
     ComplexDateTimeField handles microseconds exactly instead of rounding
@@ -538,11 +562,15 @@ class ComplexDateTimeField(StringField):
     The `,` as the separator can be easily modified by passing the `separator`
     keyword when initializing the field.
 
+    Note: To default the field to the current datetime, use: DateTimeField(default=datetime.utcnow)
+
     .. versionadded:: 0.5
     """
 
     def __init__(self, separator=',', **kwargs):
-        self.names = ['year', 'month', 'day', 'hour', 'minute', 'second', 'microsecond']
+        """
+        :param separator: Allows to customize the separator used for storage (default ``,``)
+        """
         self.separator = separator
         self.format = separator.join(['%Y', '%m', '%d', '%H', '%M', '%S', '%f'])
         super(ComplexDateTimeField, self).__init__(**kwargs)
@@ -573,16 +601,20 @@ class ComplexDateTimeField(StringField):
         return datetime.datetime(*values)
 
     def __get__(self, instance, owner):
+        if instance is None:
+            return self
+
         data = super(ComplexDateTimeField, self).__get__(instance, owner)
-        if data is None:
-            return None if self.null else datetime.datetime.now()
-        if isinstance(data, datetime.datetime):
+
+        if isinstance(data, datetime.datetime) or data is None:
             return data
         return self._convert_from_string(data)
 
     def __set__(self, instance, value):
-        value = self._convert_from_datetime(value) if value else value
-        return super(ComplexDateTimeField, self).__set__(instance, value)
+        super(ComplexDateTimeField, self).__set__(instance, value)
+        value = instance._data[self.name]
+        if value is not None:
+            instance._data[self.name] = self._convert_from_datetime(value)
 
     def validate(self, value):
         value = self.to_python(value)
@@ -611,6 +643,7 @@ class EmbeddedDocumentField(BaseField):
     """
 
     def __init__(self, document_type, **kwargs):
+        # XXX ValidationError raised outside of the "validate" method.
         if not (
             isinstance(document_type, six.string_types) or
             issubclass(document_type, EmbeddedDocument)
@@ -625,9 +658,17 @@ class EmbeddedDocumentField(BaseField):
     def document_type(self):
         if isinstance(self.document_type_obj, six.string_types):
             if self.document_type_obj == RECURSIVE_REFERENCE_CONSTANT:
-                self.document_type_obj = self.owner_document
+                resolved_document_type = self.owner_document
             else:
-                self.document_type_obj = get_document(self.document_type_obj)
+                resolved_document_type = get_document(self.document_type_obj)
+
+            if not issubclass(resolved_document_type, EmbeddedDocument):
+                # Due to the late resolution of the document_type
+                # There is a chance that it won't be an EmbeddedDocument (#1661)
+                self.error('Invalid embedded document class provided to an '
+                           'EmbeddedDocumentField')
+            self.document_type_obj = resolved_document_type
+
         return self.document_type_obj
 
     def to_python(self, value):
@@ -686,16 +727,28 @@ class GenericEmbeddedDocumentField(BaseField):
         return value
 
     def validate(self, value, clean=True):
+        if self.choices and isinstance(value, SON):
+            for choice in self.choices:
+                if value['_cls'] == choice._class_name:
+                    return True
+
         if not isinstance(value, EmbeddedDocument):
             self.error('Invalid embedded document instance provided to an '
                        'GenericEmbeddedDocumentField')
 
         value.validate(clean=clean)
 
+    def lookup_member(self, member_name):
+        if self.choices:
+            for choice in self.choices:
+                field = choice._fields.get(member_name)
+                if field:
+                    return field
+        return None
+
     def to_mongo(self, document, use_db_field=True, fields=None):
         if document is None:
             return None
-
         data = document.to_mongo(use_db_field, fields)
         if '_cls' not in data:
             data['_cls'] = document._class_name
@@ -779,6 +832,17 @@ class ListField(ComplexBaseField):
         kwargs.setdefault('default', lambda: [])
         super(ListField, self).__init__(**kwargs)
 
+    def __get__(self, instance, owner):
+        if instance is None:
+            # Document class being used rather than a document object
+            return self
+        value = instance._data.get(self.name)
+        LazyReferenceField = _import_class('LazyReferenceField')
+        GenericLazyReferenceField = _import_class('GenericLazyReferenceField')
+        if isinstance(self.field, (LazyReferenceField, GenericLazyReferenceField)) and value:
+            instance._data[self.name] = [self.field.build_lazyref(x) for x in value]
+        return super(ListField, self).__get__(instance, owner)
+
     def validate(self, value):
         """Make sure that a list of valid fields is being used."""
         if (not isinstance(value, (list, tuple, QuerySet)) or
@@ -889,12 +953,10 @@ class DictField(ComplexBaseField):
     .. versionchanged:: 0.5 - Can now handle complex / varying types of data
     """
 
-    def __init__(self, basecls=None, field=None, *args, **kwargs):
+    def __init__(self, field=None, *args, **kwargs):
         self.field = field
         self._auto_dereference = False
-        self.basecls = basecls or BaseField
-        if not issubclass(self.basecls, BaseField):
-            self.error('DictField only accepts dict values')
+
         kwargs.setdefault('default', lambda: {})
         super(DictField, self).__init__(*args, **kwargs)
 
@@ -913,7 +975,7 @@ class DictField(ComplexBaseField):
         super(DictField, self).validate(value)
 
     def lookup_member(self, member_name):
-        return DictField(basecls=self.basecls, db_field=member_name)
+        return DictField(db_field=member_name)
 
     def prepare_query_value(self, op, value):
         match_operators = ['contains', 'icontains', 'startswith',
@@ -943,6 +1005,7 @@ class MapField(DictField):
     """
 
     def __init__(self, field=None, *args, **kwargs):
+        # XXX ValidationError raised outside of the "validate" method.
         if not isinstance(field, BaseField):
             self.error('Argument to MapField constructor must be a valid '
                        'field')
@@ -953,6 +1016,15 @@ class ReferenceField(BaseField):
     """A reference to a document that will be automatically dereferenced on
     access (lazily).
 
+    Note this means you will get a database I/O access everytime you access
+    this field. This is necessary because the field returns a :class:`~mongoengine.Document`
+    which precise type can depend of the value of the `_cls` field present in the
+    document in database.
+    In short, using this type of field can lead to poor performances (especially
+    if you access this field only to retrieve it `pk` field which is already
+    known before dereference). To solve this you should consider using the
+    :class:`~mongoengine.fields.LazyReferenceField`.
+
     Use the `reverse_delete_rule` to handle what should happen if the document
     the field is referencing is deleted.  EmbeddedDocuments, DictFields and
     MapFields does not support reverse_delete_rule and an `InvalidDocumentError`
@@ -971,11 +1043,13 @@ class ReferenceField(BaseField):
 
     .. code-block:: python
 
-        class Bar(Document):
-            content = StringField()
-            foo = ReferenceField('Foo')
+        class Org(Document):
+            owner = ReferenceField('User')
 
-        Foo.register_delete_rule(Bar, 'foo', NULLIFY)
+        class User(Document):
+            org = ReferenceField('Org', reverse_delete_rule=CASCADE)
+
+        User.register_delete_rule(Org, 'owner', DENY)
 
     .. versionchanged:: 0.5 added `reverse_delete_rule`
     """
@@ -993,6 +1067,7 @@ class ReferenceField(BaseField):
             A reference to an abstract document type is always stored as a
             :class:`~pymongo.dbref.DBRef`, regardless of the value of `dbref`.
         """
+        # XXX ValidationError raised outside of the "validate" method.
         if (
             not isinstance(document_type, six.string_types) and
             not issubclass(document_type, Document)
@@ -1047,6 +1122,8 @@ class ReferenceField(BaseField):
         if isinstance(document, Document):
             # We need the id from the saved object to create the DBRef
             id_ = document.pk
+
+            # XXX ValidationError raised outside of the "validate" method.
             if id_ is None:
                 self.error('You can only reference documents once they have'
                            ' been saved to the database')
@@ -1086,19 +1163,20 @@ class ReferenceField(BaseField):
         return self.to_mongo(value)
 
     def validate(self, value):
-
-        if not isinstance(value, (self.document_type, DBRef, ObjectId)):
-            self.error('A ReferenceField only accepts DBRef, ObjectId or documents')
+        if not isinstance(value, (self.document_type, LazyReference, DBRef, ObjectId)):
+            self.error('A ReferenceField only accepts DBRef, LazyReference, ObjectId or documents')
 
         if isinstance(value, Document) and value.id is None:
             self.error('You can only reference documents once they have been '
                        'saved to the database')
 
-        if self.document_type._meta.get('abstract') and \
-                not isinstance(value, self.document_type):
+        if (
+            self.document_type._meta.get('abstract') and
+            not isinstance(value, self.document_type)
+        ):
             self.error(
                 '%s is not an instance of abstract reference type %s' % (
-                    self.document_type._class_name)
+                    value, self.document_type._class_name)
             )
 
     def lookup_member(self, member_name):
@@ -1121,6 +1199,7 @@ class CachedReferenceField(BaseField):
         if fields is None:
             fields = []
 
+        # XXX ValidationError raised outside of the "validate" method.
         if (
             not isinstance(document_type, six.string_types) and
             not issubclass(document_type, Document)
@@ -1195,6 +1274,7 @@ class CachedReferenceField(BaseField):
         id_field_name = self.document_type._meta['id_field']
         id_field = self.document_type._fields[id_field_name]
 
+        # XXX ValidationError raised outside of the "validate" method.
         if isinstance(document, Document):
             # We need the id from the saved object to create the DBRef
             id_ = document.pk
@@ -1203,7 +1283,6 @@ class CachedReferenceField(BaseField):
                            ' been saved to the database')
         else:
             self.error('Only accept a document object')
-            # TODO: should raise here or will fail next statement
 
         value = SON((
             ('_id', id_field.to_mongo(id_)),
@@ -1221,16 +1300,20 @@ class CachedReferenceField(BaseField):
         if value is None:
             return None
 
+        # XXX ValidationError raised outside of the "validate" method.
         if isinstance(value, Document):
             if value.pk is None:
                 self.error('You can only reference documents once they have'
                            ' been saved to the database')
-            return {'_id': value.pk}
+            value_dict = {'_id': value.pk}
+            for field in self.fields:
+                value_dict.update({field: value[field]})
+
+            return value_dict
 
         raise NotImplementedError
 
     def validate(self, value):
-
         if not isinstance(value, self.document_type):
             self.error('A CachedReferenceField only accepts documents')
 
@@ -1263,6 +1346,12 @@ class GenericReferenceField(BaseField):
     """A reference to *any* :class:`~mongoengine.document.Document` subclass
     that will be automatically dereferenced on access (lazily).
 
+    Note this field works the same way as :class:`~mongoengine.document.ReferenceField`,
+    doing database I/O access the first time it is accessed (even if it's to access
+    it ``pk`` or ``id`` field).
+    To solve this you should consider using the
+    :class:`~mongoengine.fields.GenericLazyReferenceField`.
+
     .. note ::
         * Any documents used as a generic reference must be registered in the
           document registry.  Importing the model will automatically register
@@ -1285,6 +1374,8 @@ class GenericReferenceField(BaseField):
                 elif isinstance(choice, type) and issubclass(choice, Document):
                     self.choices.append(choice._class_name)
                 else:
+                    # XXX ValidationError raised outside of the "validate"
+                    # method.
                     self.error('Invalid choices provided: must be a list of'
                                'Document subclasses and/or six.string_typess')
 
@@ -1348,6 +1439,7 @@ class GenericReferenceField(BaseField):
             # We need the id from the saved object to create the DBRef
             id_ = document.id
             if id_ is None:
+                # XXX ValidationError raised outside of the "validate" method.
                 self.error('You can only reference documents once they have'
                            ' been saved to the database')
         else:
@@ -1453,9 +1545,9 @@ class GridFSProxy(object):
         return '<%s: %s>' % (self.__class__.__name__, self.grid_id)
 
     def __str__(self):
-        name = getattr(
-            self.get(), 'filename', self.grid_id) if self.get() else '(no file)'
-        return '<%s: %s>' % (self.__class__.__name__, name)
+        gridout = self.get()
+        filename = getattr(gridout, 'filename') if gridout else '<no file>'
+        return '<%s: %s (%s)>' % (self.__class__.__name__, filename, self.grid_id)
 
     def __eq__(self, other):
         if isinstance(other, GridFSProxy):
@@ -1465,6 +1557,9 @@ class GridFSProxy(object):
         else:
             return False
 
+    def __ne__(self, other):
+        return not self == other
+
     @property
     def fs(self):
         if not self._fs:
@@ -2138,3 +2233,201 @@ class MultiPolygonField(GeoJsonBaseField):
     .. versionadded:: 0.9
     """
     _type = 'MultiPolygon'
+
+
+class LazyReferenceField(BaseField):
+    """A really lazy reference to a document.
+    Unlike the :class:`~mongoengine.fields.ReferenceField` it will
+    **not** be automatically (lazily) dereferenced on access.
+    Instead, access will return a :class:`~mongoengine.base.LazyReference` class
+    instance, allowing access to `pk` or manual dereference by using
+    ``fetch()`` method.
+
+    .. versionadded:: 0.15
+    """
+
+    def __init__(self, document_type, passthrough=False, dbref=False,
+                 reverse_delete_rule=DO_NOTHING, **kwargs):
+        """Initialises the Reference Field.
+
+        :param dbref:  Store the reference as :class:`~pymongo.dbref.DBRef`
+          or as the :class:`~pymongo.objectid.ObjectId`.id .
+        :param reverse_delete_rule: Determines what to do when the referring
+          object is deleted
+        :param passthrough: When trying to access unknown fields, the
+        :class:`~mongoengine.base.datastructure.LazyReference` instance will
+        automatically call `fetch()` and try to retrive the field on the fetched
+        document. Note this only work getting field (not setting or deleting).
+        """
+        # XXX ValidationError raised outside of the "validate" method.
+        if (
+            not isinstance(document_type, six.string_types) and
+            not issubclass(document_type, Document)
+        ):
+            self.error('Argument to LazyReferenceField constructor must be a '
+                       'document class or a string')
+
+        self.dbref = dbref
+        self.passthrough = passthrough
+        self.document_type_obj = document_type
+        self.reverse_delete_rule = reverse_delete_rule
+        super(LazyReferenceField, self).__init__(**kwargs)
+
+    @property
+    def document_type(self):
+        if isinstance(self.document_type_obj, six.string_types):
+            if self.document_type_obj == RECURSIVE_REFERENCE_CONSTANT:
+                self.document_type_obj = self.owner_document
+            else:
+                self.document_type_obj = get_document(self.document_type_obj)
+        return self.document_type_obj
+
+    def build_lazyref(self, value):
+        if isinstance(value, LazyReference):
+            if value.passthrough != self.passthrough:
+                value = LazyReference(value.document_type, value.pk, passthrough=self.passthrough)
+        elif value is not None:
+            if isinstance(value, self.document_type):
+                value = LazyReference(self.document_type, value.pk, passthrough=self.passthrough)
+            elif isinstance(value, DBRef):
+                value = LazyReference(self.document_type, value.id, passthrough=self.passthrough)
+            else:
+                # value is the primary key of the referenced document
+                value = LazyReference(self.document_type, value, passthrough=self.passthrough)
+        return value
+
+    def __get__(self, instance, owner):
+        """Descriptor to allow lazy dereferencing."""
+        if instance is None:
+            # Document class being used rather than a document object
+            return self
+
+        value = self.build_lazyref(instance._data.get(self.name))
+        if value:
+            instance._data[self.name] = value
+
+        return super(LazyReferenceField, self).__get__(instance, owner)
+
+    def to_mongo(self, value):
+        if isinstance(value, LazyReference):
+            pk = value.pk
+        elif isinstance(value, self.document_type):
+            pk = value.pk
+        elif isinstance(value, DBRef):
+            pk = value.id
+        else:
+            # value is the primary key of the referenced document
+            pk = value
+        id_field_name = self.document_type._meta['id_field']
+        id_field = self.document_type._fields[id_field_name]
+        pk = id_field.to_mongo(pk)
+        if self.dbref:
+            return DBRef(self.document_type._get_collection_name(), pk)
+        else:
+            return pk
+
+    def validate(self, value):
+        if isinstance(value, LazyReference):
+            if value.collection != self.document_type._get_collection_name():
+                self.error('Reference must be on a `%s` document.' % self.document_type)
+            pk = value.pk
+        elif isinstance(value, self.document_type):
+            pk = value.pk
+        elif isinstance(value, DBRef):
+            # TODO: check collection ?
+            collection = self.document_type._get_collection_name()
+            if value.collection != collection:
+                self.error("DBRef on bad collection (must be on `%s`)" % collection)
+            pk = value.id
+        else:
+            # value is the primary key of the referenced document
+            id_field_name = self.document_type._meta['id_field']
+            id_field = getattr(self.document_type, id_field_name)
+            pk = value
+            try:
+                id_field.validate(pk)
+            except ValidationError:
+                self.error(
+                    "value should be `{0}` document, LazyReference or DBRef on `{0}` "
+                    "or `{0}`'s primary key (i.e. `{1}`)".format(
+                        self.document_type.__name__, type(id_field).__name__))
+
+        if pk is None:
+            self.error('You can only reference documents once they have been '
+                       'saved to the database')
+
+    def prepare_query_value(self, op, value):
+        if value is None:
+            return None
+        super(LazyReferenceField, self).prepare_query_value(op, value)
+        return self.to_mongo(value)
+
+    def lookup_member(self, member_name):
+        return self.document_type._fields.get(member_name)
+
+
+class GenericLazyReferenceField(GenericReferenceField):
+    """A reference to *any* :class:`~mongoengine.document.Document` subclass.
+    Unlike the :class:`~mongoengine.fields.GenericReferenceField` it will
+    **not** be automatically (lazily) dereferenced on access.
+    Instead, access will return a :class:`~mongoengine.base.LazyReference` class
+    instance, allowing access to `pk` or manual dereference by using
+    ``fetch()`` method.
+
+    .. note ::
+        * Any documents used as a generic reference must be registered in the
+          document registry.  Importing the model will automatically register
+          it.
+
+        * You can use the choices param to limit the acceptable Document types
+
+    .. versionadded:: 0.15
+    """
+
+    def __init__(self, *args, **kwargs):
+        self.passthrough = kwargs.pop('passthrough', False)
+        super(GenericLazyReferenceField, self).__init__(*args, **kwargs)
+
+    def _validate_choices(self, value):
+        if isinstance(value, LazyReference):
+            value = value.document_type._class_name
+        super(GenericLazyReferenceField, self)._validate_choices(value)
+
+    def build_lazyref(self, value):
+        if isinstance(value, LazyReference):
+            if value.passthrough != self.passthrough:
+                value = LazyReference(value.document_type, value.pk, passthrough=self.passthrough)
+        elif value is not None:
+            if isinstance(value, (dict, SON)):
+                value = LazyReference(get_document(value['_cls']), value['_ref'].id, passthrough=self.passthrough)
+            elif isinstance(value, Document):
+                value = LazyReference(type(value), value.pk, passthrough=self.passthrough)
+        return value
+
+    def __get__(self, instance, owner):
+        if instance is None:
+            return self
+
+        value = self.build_lazyref(instance._data.get(self.name))
+        if value:
+            instance._data[self.name] = value
+
+        return super(GenericLazyReferenceField, self).__get__(instance, owner)
+
+    def validate(self, value):
+        if isinstance(value, LazyReference) and value.pk is None:
+            self.error('You can only reference documents once they have been'
+                       ' saved to the database')
+        return super(GenericLazyReferenceField, self).validate(value)
+
+    def to_mongo(self, document):
+        if document is None:
+            return None
+
+        if isinstance(document, LazyReference):
+            return SON((
+                ('_cls', document.document_type._class_name),
+                ('_ref', DBRef(document.document_type._get_collection_name(), document.pk))
+            ))
+        else:
+            return super(GenericLazyReferenceField, self).to_mongo(document)

mongoengine/python_support.py

@@ -6,11 +6,7 @@ import pymongo
 import six
 
 
-if pymongo.version_tuple[0] < 3:
-    IS_PYMONGO_3 = False
-else:
-    IS_PYMONGO_3 = True
-
+IS_PYMONGO_3 = pymongo.version_tuple[0] >= 3
 
 # six.BytesIO resolves to StringIO.StringIO in Py2 and io.BytesIO in Py3.
 StringIO = six.BytesIO

mongoengine/queryset/base.py

@@ -18,7 +18,7 @@ from mongoengine import signals
 from mongoengine.base import get_document
 from mongoengine.common import _import_class
 from mongoengine.connection import get_db
-from mongoengine.context_managers import switch_db
+from mongoengine.context_managers import set_write_concern, switch_db
 from mongoengine.errors import (InvalidQueryError, LookUpError,
                                 NotUniqueError, OperationError)
 from mongoengine.python_support import IS_PYMONGO_3
@@ -350,11 +350,24 @@ class BaseQuerySet(object):
                                      documents=docs, **signal_kwargs)
 
         raw = [doc.to_mongo() for doc in docs]
+
+        with set_write_concern(self._collection, write_concern) as collection:
+            insert_func = collection.insert_many
+            if return_one:
+                raw = raw[0]
+                insert_func = collection.insert_one
+
         try:
-            ids = self._collection.insert(raw, **write_concern)
+            inserted_result = insert_func(raw)
+            ids = return_one and [inserted_result.inserted_id] or inserted_result.inserted_ids
         except pymongo.errors.DuplicateKeyError as err:
             message = 'Could not save document (%s)'
             raise NotUniqueError(message % six.text_type(err))
+        except pymongo.errors.BulkWriteError as err:
+            # inserting documents that already have an _id field will
+            # give huge performance debt or raise
+            message = u'Document must not have _id value before bulk write (%s)'
+            raise NotUniqueError(message % six.text_type(err))
         except pymongo.errors.OperationFailure as err:
             message = 'Could not save document (%s)'
             if re.match('^E1100[01] duplicate key', six.text_type(err)):
@@ -368,7 +381,6 @@ class BaseQuerySet(object):
             signals.post_bulk_insert.send(
                 self._document, documents=docs, loaded=False, **signal_kwargs)
             return return_one and ids[0] or ids
-
         documents = self.in_bulk(ids)
         results = []
         for obj_id in ids:
@@ -486,8 +498,9 @@ class BaseQuerySet(object):
             ``save(..., write_concern={w: 2, fsync: True}, ...)`` will
             wait until at least two servers have recorded the write and
             will force an fsync on the primary server.
-        :param full_result: Return the full result rather than just the number
-            updated.
+        :param full_result: Return the full result dictionary rather than just the number
+            updated, e.g. return
+            ``{'n': 2, 'nModified': 2, 'ok': 1.0, 'updatedExisting': True}``.
         :param update: Django-style update keyword arguments
 
         .. versionadded:: 0.2
@@ -510,12 +523,15 @@ class BaseQuerySet(object):
             else:
                 update['$set'] = {'_cls': queryset._document._class_name}
         try:
-            result = queryset._collection.update(query, update, multi=multi,
-                                                 upsert=upsert, **write_concern)
+            with set_write_concern(queryset._collection, write_concern) as collection:
+                update_func = collection.update_one
+                if multi:
+                    update_func = collection.update_many
+                result = update_func(query, update, upsert=upsert)
             if full_result:
                 return result
-            elif result:
-                return result['n']
+            elif result.raw_result:
+                return result.raw_result['n']
         except pymongo.errors.DuplicateKeyError as err:
             raise NotUniqueError(u'Update failed (%s)' % six.text_type(err))
         except pymongo.errors.OperationFailure as err:
@@ -544,10 +560,10 @@ class BaseQuerySet(object):
                                     write_concern=write_concern,
                                     full_result=True, **update)
 
-        if atomic_update['updatedExisting']:
+        if atomic_update.raw_result['updatedExisting']:
             document = self.get()
         else:
-            document = self._document.objects.with_id(atomic_update['upserted'])
+            document = self._document.objects.with_id(atomic_update.upserted_id)
         return document
 
     def update_one(self, upsert=False, write_concern=None, **update):
@@ -1183,6 +1199,10 @@ class BaseQuerySet(object):
 
         pipeline = initial_pipeline + list(pipeline)
 
+        if IS_PYMONGO_3 and self._read_preference is not None:
+            return self._collection.with_options(read_preference=self._read_preference) \
+                       .aggregate(pipeline, cursor={}, **kwargs)
+
         return self._collection.aggregate(pipeline, cursor={}, **kwargs)
 
     # JS functionality
@@ -1579,6 +1599,9 @@ class BaseQuerySet(object):
         if self._batch_size is not None:
             self._cursor_obj.batch_size(self._batch_size)
 
+        if self._comment is not None:
+            self._cursor_obj.comment(self._comment)
+
         return self._cursor_obj
 
     def __deepcopy__(self, memo):

mongoengine/queryset/queryset.py

@@ -1,3 +1,5 @@
+import six
+
 from mongoengine.errors import OperationError
 from mongoengine.queryset.base import (BaseQuerySet, CASCADE, DENY, DO_NOTHING,
                                        NULLIFY, PULL)
@@ -87,10 +89,10 @@ class QuerySet(BaseQuerySet):
                 yield self._result_cache[pos]
                 pos += 1
 
-            # Raise StopIteration if we already established there were no more
+            # return if we already established there were no more
             # docs in the db cursor.
             if not self._has_more:
-                raise StopIteration
+                return
 
             # Otherwise, populate more of the cache and repeat.
             if len(self._result_cache) <= pos:
@@ -112,7 +114,7 @@ class QuerySet(BaseQuerySet):
         # Pull in ITER_CHUNK_SIZE docs from the database and store them in
         # the result cache.
         try:
-            for _ in xrange(ITER_CHUNK_SIZE):
+            for _ in six.moves.range(ITER_CHUNK_SIZE):
                 self._result_cache.append(self.next())
         except StopIteration:
             # Getting this exception means there are no more docs in the
@@ -166,7 +168,7 @@ class QuerySetNoCache(BaseQuerySet):
             return '.. queryset mid-iteration ..'
 
         data = []
-        for _ in xrange(REPR_OUTPUT_SIZE + 1):
+        for _ in six.moves.range(REPR_OUTPUT_SIZE + 1):
             try:
                 data.append(self.next())
             except StopIteration:

mongoengine/queryset/transform.py

@@ -101,21 +101,8 @@ def query(_doc_cls=None, **kwargs):
                         value = value['_id']
 
             elif op in ('in', 'nin', 'all', 'near') and not isinstance(value, dict):
-                # Raise an error if the in/nin/all/near param is not iterable. We need a
-                # special check for BaseDocument, because - although it's iterable - using
-                # it as such in the context of this method is most definitely a mistake.
-                BaseDocument = _import_class('BaseDocument')
-                if isinstance(value, BaseDocument):
-                    raise TypeError("When using the `in`, `nin`, `all`, or "
-                                    "`near`-operators you can\'t use a "
-                                    "`Document`, you must wrap your object "
-                                    "in a list (object -> [object]).")
-                elif not hasattr(value, '__iter__'):
-                    raise TypeError("The `in`, `nin`, `all`, or "
-                                    "`near`-operators must be applied to an "
-                                    "iterable (e.g. a list).")
-                else:
-                    value = [field.prepare_query_value(op, v) for v in value]
+                # Raise an error if the in/nin/all/near param is not iterable.
+                value = _prepare_query_for_iterable(field, op, value)
 
             # If we're querying a GenericReferenceField, we need to alter the
             # key depending on the value:
@@ -160,7 +147,7 @@ def query(_doc_cls=None, **kwargs):
         if op is None or key not in mongo_query:
             mongo_query[key] = value
         elif key in mongo_query:
-            if isinstance(mongo_query[key], dict):
+            if isinstance(mongo_query[key], dict) and isinstance(value, dict):
                 mongo_query[key].update(value)
                 # $max/minDistance needs to come last - convert to SON
                 value_dict = mongo_query[key]
@@ -214,30 +201,37 @@ def update(_doc_cls=None, **update):
     format.
     """
     mongo_update = {}
+
     for key, value in update.items():
         if key == '__raw__':
             mongo_update.update(value)
             continue
+
         parts = key.split('__')
+
         # if there is no operator, default to 'set'
         if len(parts) < 3 and parts[0] not in UPDATE_OPERATORS:
             parts.insert(0, 'set')
+
         # Check for an operator and transform to mongo-style if there is
         op = None
         if parts[0] in UPDATE_OPERATORS:
             op = parts.pop(0)
             # Convert Pythonic names to Mongo equivalents
-            if op in ('push_all', 'pull_all'):
-                op = op.replace('_all', 'All')
-            elif op == 'dec':
+            operator_map = {
+                'push_all': 'pushAll',
+                'pull_all': 'pullAll',
+                'dec': 'inc',
+                'add_to_set': 'addToSet',
+                'set_on_insert': 'setOnInsert'
+            }
+            if op == 'dec':
                 # Support decrement by flipping a positive value's sign
                 # and using 'inc'
-                op = 'inc'
                 value = -value
-            elif op == 'add_to_set':
-                op = 'addToSet'
-            elif op == 'set_on_insert':
-                op = 'setOnInsert'
+            # If the operator doesn't found from operator map, the op value
+            # will stay unchanged
+            op = operator_map.get(op, op)
 
         match = None
         if parts[-1] in COMPARISON_OPERATORS:
@@ -284,9 +278,15 @@ def update(_doc_cls=None, **update):
             if isinstance(field, GeoJsonBaseField):
                 value = field.to_mongo(value)
 
-            if op == 'push' and isinstance(value, (list, tuple, set)):
+            if op == 'pull':
+                if field.required or value is not None:
+                    if match == 'in' and not isinstance(value, dict):
+                        value = _prepare_query_for_iterable(field, op, value)
+                    else:
+                        value = field.prepare_query_value(op, value)
+            elif op == 'push' and isinstance(value, (list, tuple, set)):
                 value = [field.prepare_query_value(op, v) for v in value]
-            elif op in (None, 'set', 'push', 'pull'):
+            elif op in (None, 'set', 'push'):
                 if field.required or value is not None:
                     value = field.prepare_query_value(op, value)
             elif op in ('pushAll', 'pullAll'):
@@ -298,6 +298,8 @@ def update(_doc_cls=None, **update):
                     value = field.prepare_query_value(op, value)
             elif op == 'unset':
                 value = 1
+            elif op == 'inc':
+                value = field.prepare_query_value(op, value)
 
         if match:
             match = '$' + match
@@ -321,11 +323,17 @@ def update(_doc_cls=None, **update):
             field_classes = [c.__class__ for c in cleaned_fields]
             field_classes.reverse()
             ListField = _import_class('ListField')
-            if ListField in field_classes:
-                # Join all fields via dot notation to the last ListField
+            EmbeddedDocumentListField = _import_class('EmbeddedDocumentListField')
+            if ListField in field_classes or EmbeddedDocumentListField in field_classes:
+                # Join all fields via dot notation to the last ListField or EmbeddedDocumentListField
                 # Then process as normal
+                if ListField in field_classes:
+                    _check_field = ListField
+                else:
+                    _check_field = EmbeddedDocumentListField
+
                 last_listField = len(
-                    cleaned_fields) - field_classes.index(ListField)
+                    cleaned_fields) - field_classes.index(_check_field)
                 key = '.'.join(parts[:last_listField])
                 parts = parts[last_listField:]
                 parts.insert(0, key)
@@ -335,7 +343,7 @@ def update(_doc_cls=None, **update):
                 value = {key: value}
         elif op == 'addToSet' and isinstance(value, list):
             value = {key: {'$each': value}}
-        elif op == 'push':
+        elif op in ('push', 'pushAll'):
             if parts[-1].isdigit():
                 key = parts[0]
                 position = int(parts[-1])
@@ -344,10 +352,14 @@ def update(_doc_cls=None, **update):
                 if not isinstance(value, (set, tuple, list)):
                     value = [value]
                 value = {key: {'$each': value, '$position': position}}
-            elif isinstance(value, list):
-                value = {key: {'$each': value}}
             else:
-                value = {key: value}
+                if op == 'pushAll':
+                    op = 'push'  # convert to non-deprecated keyword
+                    if not isinstance(value, (set, tuple, list)):
+                        value = [value]
+                    value = {key: {'$each': value}}
+                else:
+                    value = {key: value}
         else:
             value = {key: value}
         key = '$' + op
@@ -439,3 +451,22 @@ def _infer_geometry(value):
 
     raise InvalidQueryError('Invalid $geometry data. Can be either a '
                             'dictionary or (nested) lists of coordinate(s)')
+
+
+def _prepare_query_for_iterable(field, op, value):
+    # We need a special check for BaseDocument, because - although it's iterable - using
+    # it as such in the context of this method is most definitely a mistake.
+    BaseDocument = _import_class('BaseDocument')
+
+    if isinstance(value, BaseDocument):
+        raise TypeError("When using the `in`, `nin`, `all`, or "
+                        "`near`-operators you can\'t use a "
+                        "`Document`, you must wrap your object "
+                        "in a list (object -> [object]).")
+
+    if not hasattr(value, '__iter__'):
+        raise TypeError("The `in`, `nin`, `all`, or "
+                        "`near`-operators must be applied to an "
+                        "iterable (e.g. a list).")
+
+    return [field.prepare_query_value(op, v) for v in value]