feat: update code collection check

added a feature to save object data in order

docs/changelog.rst

@@ -5,11 +5,27 @@ Changelog
 Development
 ===========
 - (Fill this out as you fix issues and develop your features).
-- Fixed using sets in field choices #1481
+
+Changes in 0.13.0
+=================
+- POTENTIAL BREAKING CHANGE: Added Unicode support to the `EmailField`, see
+  docs/upgrade.rst for details.
+
+Changes in 0.12.0
+=================
 - POTENTIAL BREAKING CHANGE: Fixed limit/skip/hint/batch_size chaining #1476
 - POTENTIAL BREAKING CHANGE: Changed a public `QuerySet.clone_into` method to a private `QuerySet._clone_into` #1476
+- Fixed the way `Document.objects.create` works with duplicate IDs #1485
 - Fixed connecting to a replica set with PyMongo 2.x #1436
+- Fixed using sets in field choices #1481
+- Fixed deleting items from a `ListField` #1318
 - Fixed an obscure error message when filtering by `field__in=non_iterable`. #1237
+- Fixed behavior of a `dec` update operator #1450
+- Added a `rename` update operator #1454
+- Added validation for the `db_field` parameter #1448
+- Fixed the error message displayed when querying an `EmbeddedDocumentField` by an invalid value #1440
+- Fixed the error message displayed when validating unicode URLs #1486
+- Raise an error when trying to save an abstract document #1449
 
 Changes in 0.11.0
 =================

docs/guide/querying.rst

@@ -340,14 +340,19 @@ Javascript code that is executed on the database server.
 
 Counting results
 ----------------
-Just as with limiting and skipping results, there is a method on
-:class:`~mongoengine.queryset.QuerySet` objects --
-:meth:`~mongoengine.queryset.QuerySet.count`, but there is also a more Pythonic
-way of achieving this::
+Just as with limiting and skipping results, there is a method on a
+:class:`~mongoengine.queryset.QuerySet` object --
+:meth:`~mongoengine.queryset.QuerySet.count`::
 
-    num_users = len(User.objects)
+    num_users = User.objects.count()
 
-Even if len() is the Pythonic way of counting results, keep in mind that if you concerned about performance, :meth:`~mongoengine.queryset.QuerySet.count` is the way to go since it only execute a server side count query, while len() retrieves the results, places them in cache, and finally counts them. If we compare the performance of the two operations, len() is much slower than :meth:`~mongoengine.queryset.QuerySet.count`.
+You could technically use ``len(User.objects)`` to get the same result, but it
+would be significantly slower than :meth:`~mongoengine.queryset.QuerySet.count`.
+When you execute a server-side count query, you let MongoDB do the heavy
+lifting and you receive a single integer over the wire. Meanwhile, len()
+retrieves all the results, places them in a local cache, and finally counts
+them. If we compare the performance of the two operations, len() is much slower
+than :meth:`~mongoengine.queryset.QuerySet.count`.
 
 Further aggregation
 -------------------

docs/upgrade.rst

@@ -6,6 +6,20 @@ Development
 ***********
 (Fill this out whenever you introduce breaking changes to MongoEngine)
 
+0.13.0
+******
+This release adds Unicode support to the `EmailField` and changes its
+structure significantly. Previously, email addresses containing Unicode
+characters didn't work at all. Starting with v0.13.0, domains with Unicode
+characters are supported out of the box, meaning some emails that previously
+didn't pass validation now do. Make sure the rest of your application can
+accept such email addresses. Additionally, if you subclassed the `EmailField`
+in your application and overrode `EmailField.EMAIL_REGEX`, you will have to
+adjust your code to override `EmailField.USER_REGEX`, `EmailField.DOMAIN_REGEX`,
+and potentially `EmailField.UTF8_USER_REGEX`.
+
+0.12.0
+******
 This release includes various fixes for the `BaseQuerySet` methods and how they
 are chained together. Since version 0.10.1 applying limit/skip/hint/batch_size
 to an already-existing queryset wouldn't modify the underlying PyMongo cursor.

mongoengine/__init__.py

@@ -23,7 +23,7 @@ __all__ = (list(document.__all__) + list(fields.__all__) +
            list(signals.__all__) + list(errors.__all__))
 
 
-VERSION = (0, 11, 0)
+VERSION = (0, 13, 0)
 
 
 def get_version():

mongoengine/base/document.py

@@ -1,6 +1,6 @@
 import copy
 import numbers
-from collections import Hashable
+from collections.abc import Hashable
 from functools import partial
 
 from bson import ObjectId, json_util
@@ -684,8 +684,13 @@ class BaseDocument(object):
         # class if unavailable
         class_name = son.get('_cls', cls._class_name)
 
-        # Convert SON to a dict, making sure each key is a string
-        data = {str(key): value for key, value in son.iteritems()}
+        # Convert SON to a data dict, making sure each key is a string and
+        # corresponds to the right db field.
+        data = {}
+        for key, value in son.iteritems():
+            key = str(key)
+            key = cls._db_field_map.get(key, key)
+            data[key] = value
 
         # Return correct subclass for document type
         if class_name != cls._class_name:

mongoengine/dereference.py

@@ -1,3 +1,4 @@
+from collections import OrderedDict
 from bson import DBRef, SON
 import six
 
@@ -201,6 +202,10 @@ class DeReference(object):
             as_tuple = isinstance(items, tuple)
             iterator = enumerate(items)
             data = []
+        elif isinstance(items, OrderedDict):
+            is_list = False
+            iterator = items.iteritems()
+            data = OrderedDict()
         else:
             is_list = False
             iterator = items.iteritems()

mongoengine/document.py

@@ -808,7 +808,7 @@ class Document(BaseDocument):
         collection = cls._get_collection()
         # 746: when connection is via mongos, the read preference is not necessarily an indication that
         # this code runs on a secondary
-        if not collection.is_mongos and collection.read_preference > 1:
+        if collection.is_mongos is not None and collection.read_preference.mode > 1:
             return
 
         # determine if an index which we are creating includes

mongoengine/fields.py

@@ -2,9 +2,11 @@ import datetime
 import decimal
 import itertools
 import re
+import socket
 import time
 import uuid
 import warnings
+from collections.abc import Mapping
 from operator import itemgetter
 
 from bson import Binary, DBRef, ObjectId, SON
@@ -153,21 +155,105 @@ class EmailField(StringField):
 
     .. versionadded:: 0.4
     """
-
-    EMAIL_REGEX = re.compile(
-        # dot-atom
-        r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*"
-        # quoted-string
-        r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-011\013\014\016-\177])*"'
-        # domain (max length of an ICAAN TLD is 22 characters)
-        r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}|[A-Z0-9-]{2,}(?<!-))$', re.IGNORECASE
+    USER_REGEX = re.compile(
+        # `dot-atom` defined in RFC 5322 Section 3.2.3.
+        r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*\Z"
+        # `quoted-string` defined in RFC 5322 Section 3.2.4.
+        r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-\011\013\014\016-\177])*"\Z)',
+        re.IGNORECASE
     )
 
+    UTF8_USER_REGEX = re.compile(
+        six.u(
+            # RFC 6531 Section 3.3 extends `atext` (used by dot-atom) to
+            # include `UTF8-non-ascii`.
+            r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z\u0080-\U0010FFFF]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z\u0080-\U0010FFFF]+)*\Z"
+            # `quoted-string`
+            r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-\011\013\014\016-\177])*"\Z)'
+        ), re.IGNORECASE | re.UNICODE
+    )
+
+    DOMAIN_REGEX = re.compile(
+        r'((?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+)(?:[A-Z0-9-]{2,63}(?<!-))\Z',
+        re.IGNORECASE
+    )
+
+    error_msg = u'Invalid email address: %s'
+
+    def __init__(self, domain_whitelist=None, allow_utf8_user=False,
+                 allow_ip_domain=False, *args, **kwargs):
+        """Initialize the EmailField.
+
+        Args:
+            domain_whitelist (list) - list of otherwise invalid domain
+                                      names which you'd like to support.
+            allow_utf8_user (bool) - if True, the user part of the email
+                                     address can contain UTF8 characters.
+                                     False by default.
+            allow_ip_domain (bool) - if True, the domain part of the email
+                                     can be a valid IPv4 or IPv6 address.
+        """
+        self.domain_whitelist = domain_whitelist or []
+        self.allow_utf8_user = allow_utf8_user
+        self.allow_ip_domain = allow_ip_domain
+        super(EmailField, self).__init__(*args, **kwargs)
+
+    def validate_user_part(self, user_part):
+        """Validate the user part of the email address. Return True if
+        valid and False otherwise.
+        """
+        if self.allow_utf8_user:
+            return self.UTF8_USER_REGEX.match(user_part)
+        return self.USER_REGEX.match(user_part)
+
+    def validate_domain_part(self, domain_part):
+        """Validate the domain part of the email address. Return True if
+        valid and False otherwise.
+        """
+        # Skip domain validation if it's in the whitelist.
+        if domain_part in self.domain_whitelist:
+            return True
+
+        if self.DOMAIN_REGEX.match(domain_part):
+            return True
+
+        # Validate IPv4/IPv6, e.g. user@[192.168.0.1]
+        if (
+            self.allow_ip_domain and
+            domain_part[0] == '[' and
+            domain_part[-1] == ']'
+        ):
+            for addr_family in (socket.AF_INET, socket.AF_INET6):
+                try:
+                    socket.inet_pton(addr_family, domain_part[1:-1])
+                    return True
+                except (socket.error, UnicodeEncodeError):
+                    pass
+
+        return False
+
     def validate(self, value):
-        if not EmailField.EMAIL_REGEX.match(value):
-            self.error('Invalid email address: %s' % value)
         super(EmailField, self).validate(value)
 
+        if '@' not in value:
+            self.error(self.error_msg % value)
+
+        user_part, domain_part = value.rsplit('@', 1)
+
+        # Validate the user part.
+        if not self.validate_user_part(user_part):
+            self.error(self.error_msg % value)
+
+        # Validate the domain and, if invalid, see if it's IDN-encoded.
+        if not self.validate_domain_part(domain_part):
+            try:
+                domain_part = domain_part.encode('idna').decode('ascii')
+            except UnicodeError:
+                self.error(self.error_msg % value)
+            else:
+                if not self.validate_domain_part(domain_part):
+                    self.error(self.error_msg % value)
+
 
 class IntField(BaseField):
     """32-bit integer field."""
@@ -619,6 +705,14 @@ class DynamicField(BaseField):
 
     Used by :class:`~mongoengine.DynamicDocument` to handle dynamic data"""
 
+    def __init__(self, container_class=dict, *args, **kwargs):
+        self._container_cls = container_class
+        if not issubclass(self._container_cls, Mapping):
+            self.error('The class that is specified in `container_class` parameter '
+                       'must be a subclass of `dict`.')
+
+        super(DynamicField, self).__init__(*args, **kwargs)
+
     def to_mongo(self, value, use_db_field=True, fields=None):
         """Convert a Python type to a MongoDB compatible type.
         """
@@ -644,7 +738,7 @@ class DynamicField(BaseField):
             is_list = True
             value = {k: v for k, v in enumerate(value)}
 
-        data = {}
+        data = self._container_cls()
         for k, v in value.iteritems():
             data[k] = self.to_mongo(v, use_db_field, fields)
 
@@ -998,8 +1092,8 @@ class ReferenceField(BaseField):
 
     def validate(self, value):
 
-        if not isinstance(value, (self.document_type, DBRef)):
-            self.error('A ReferenceField only accepts DBRef or documents')
+        if not isinstance(value, (self.document_type, DBRef, ObjectId)):
+            self.error('A ReferenceField only accepts DBRef, ObjectId or documents')
 
         if isinstance(value, Document) and value.id is None:
             self.error('You can only reference documents once they have been '

mongoengine/queryset/base.py

@@ -158,44 +158,49 @@ class BaseQuerySet(object):
         # self._cursor
 
     def __getitem__(self, key):
-        """Support skip and limit using getitem and slicing syntax."""
+        """Return a document instance corresponding to a given index if
+        the key is an integer. If the key is a slice, translate its
+        bounds into a skip and a limit, and return a cloned queryset
+        with that skip/limit applied. For example:
+
+        >>> User.objects[0]
+        <User: User object>
+        >>> User.objects[1:3]
+        [<User: User object>, <User: User object>]
+        """
         queryset = self.clone()
 
-        # Slice provided
+        # Handle a slice
         if isinstance(key, slice):
-            try:
-                queryset._cursor_obj = queryset._cursor[key]
-                queryset._skip, queryset._limit = key.start, key.stop
-                if key.start and key.stop:
-                    queryset._limit = key.stop - key.start
-            except IndexError as err:
-                # PyMongo raises an error if key.start == key.stop, catch it,
-                # bin it, kill it.
-                start = key.start or 0
-                if start >= 0 and key.stop >= 0 and key.step is None:
-                    if start == key.stop:
-                        queryset.limit(0)
-                        queryset._skip = key.start
-                        queryset._limit = key.stop - start
-                        return queryset
-                raise err
+            queryset._cursor_obj = queryset._cursor[key]
+            queryset._skip, queryset._limit = key.start, key.stop
+            if key.start and key.stop:
+                queryset._limit = key.stop - key.start
+
             # Allow further QuerySet modifications to be performed
             return queryset
-        # Integer index provided
+
+        # Handle an index
         elif isinstance(key, int):
             if queryset._scalar:
                 return queryset._get_scalar(
-                    queryset._document._from_son(queryset._cursor[key],
-                                                 _auto_dereference=self._auto_dereference,
-                                                 only_fields=self.only_fields))
+                    queryset._document._from_son(
+                        queryset._cursor[key],
+                        _auto_dereference=self._auto_dereference,
+                        only_fields=self.only_fields
+                    )
+                )
 
             if queryset._as_pymongo:
                 return queryset._get_as_pymongo(queryset._cursor[key])
-            return queryset._document._from_son(queryset._cursor[key],
-                                                _auto_dereference=self._auto_dereference,
-                                                only_fields=self.only_fields)
 
-        raise AttributeError
+            return queryset._document._from_son(
+                queryset._cursor[key],
+                _auto_dereference=self._auto_dereference,
+                only_fields=self.only_fields
+            )
+
+        raise AttributeError('Provide a slice or an integer index')
 
     def __iter__(self):
         raise NotImplementedError

tests/document/instance.py

@@ -28,8 +28,6 @@ TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__),
 __all__ = ("InstanceTest",)
 
 
-
-
 class InstanceTest(unittest.TestCase):
 
     def setUp(self):
@@ -72,8 +70,7 @@ class InstanceTest(unittest.TestCase):
             self.assertEqual(field._instance, instance)
 
     def test_capped_collection(self):
-        """Ensure that capped collections work properly.
-        """
+        """Ensure that capped collections work properly."""
         class Log(Document):
             date = DateTimeField(default=datetime.now)
             meta = {
@@ -181,8 +178,7 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual('<Article: привет мир>', repr(doc))
 
     def test_repr_none(self):
-        """Ensure None values handled correctly
-        """
+        """Ensure None values are handled correctly."""
         class Article(Document):
             title = StringField()
 
@@ -190,25 +186,23 @@ class InstanceTest(unittest.TestCase):
                 return None
 
         doc = Article(title=u'привет мир')
-
         self.assertEqual('<Article: None>', repr(doc))
 
     def test_queryset_resurrects_dropped_collection(self):
         self.Person.drop_collection()
-
         self.assertEqual([], list(self.Person.objects()))
 
+        # Ensure works correctly with inhertited classes
         class Actor(self.Person):
             pass
 
-        # Ensure works correctly with inhertited classes
         Actor.objects()
         self.Person.drop_collection()
         self.assertEqual([], list(Actor.objects()))
 
     def test_polymorphic_references(self):
-        """Ensure that the correct subclasses are returned from a query when
-        using references / generic references
+        """Ensure that the correct subclasses are returned from a query
+        when using references / generic references
         """
         class Animal(Document):
             meta = {'allow_inheritance': True}
@@ -258,9 +252,6 @@ class InstanceTest(unittest.TestCase):
         classes = [a.__class__ for a in Zoo.objects.first().animals]
         self.assertEqual(classes, [Animal, Fish, Mammal, Dog, Human])
 
-        Zoo.drop_collection()
-        Animal.drop_collection()
-
     def test_reference_inheritance(self):
         class Stats(Document):
             created = DateTimeField(default=datetime.now)
@@ -287,8 +278,7 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(list_stats, CompareStats.objects.first().stats)
 
     def test_db_field_load(self):
-        """Ensure we load data correctly
-        """
+        """Ensure we load data correctly from the right db field."""
         class Person(Document):
             name = StringField(required=True)
             _rank = StringField(required=False, db_field="rank")
@@ -307,8 +297,7 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(Person.objects.get(name="Fred").rank, "Private")
 
     def test_db_embedded_doc_field_load(self):
-        """Ensure we load embedded document data correctly
-        """
+        """Ensure we load embedded document data correctly."""
         class Rank(EmbeddedDocument):
             title = StringField(required=True)
 
@@ -333,8 +322,7 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(Person.objects.get(name="Fred").rank, "Private")
 
     def test_custom_id_field(self):
-        """Ensure that documents may be created with custom primary keys.
-        """
+        """Ensure that documents may be created with custom primary keys."""
         class User(Document):
             username = StringField(primary_key=True)
             name = StringField()
@@ -382,10 +370,7 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(user_son['_id'], 'mongo')
         self.assertTrue('username' not in user_son['_id'])
 
-        User.drop_collection()
-
     def test_document_not_registered(self):
-
         class Place(Document):
             name = StringField()
 
@@ -407,7 +392,6 @@ class InstanceTest(unittest.TestCase):
             list(Place.objects.all())
 
     def test_document_registry_regressions(self):
-
         class Location(Document):
             name = StringField()
             meta = {'allow_inheritance': True}
@@ -421,18 +405,16 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(Area, get_document("Location.Area"))
 
     def test_creation(self):
-        """Ensure that document may be created using keyword arguments.
-        """
+        """Ensure that document may be created using keyword arguments."""
         person = self.Person(name="Test User", age=30)
         self.assertEqual(person.name, "Test User")
         self.assertEqual(person.age, 30)
 
     def test_to_dbref(self):
-        """Ensure that you can get a dbref of a document"""
+        """Ensure that you can get a dbref of a document."""
         person = self.Person(name="Test User", age=30)
         self.assertRaises(OperationError, person.to_dbref)
         person.save()
-
         person.to_dbref()
 
     def test_save_abstract_document(self):
@@ -445,8 +427,7 @@ class InstanceTest(unittest.TestCase):
             Doc(name='aaa').save()
 
     def test_reload(self):
-        """Ensure that attributes may be reloaded.
-        """
+        """Ensure that attributes may be reloaded."""
         person = self.Person(name="Test User", age=20)
         person.save()
 
@@ -479,7 +460,6 @@ class InstanceTest(unittest.TestCase):
         doc = Animal(superphylum='Deuterostomia')
         doc.save()
         doc.reload()
-        Animal.drop_collection()
 
     def test_reload_sharded_nested(self):
         class SuperPhylum(EmbeddedDocument):
@@ -493,11 +473,9 @@ class InstanceTest(unittest.TestCase):
         doc = Animal(superphylum=SuperPhylum(name='Deuterostomia'))
         doc.save()
         doc.reload()
-        Animal.drop_collection()
 
     def test_reload_referencing(self):
-        """Ensures reloading updates weakrefs correctly
-        """
+        """Ensures reloading updates weakrefs correctly."""
         class Embedded(EmbeddedDocument):
             dict_field = DictField()
             list_field = ListField()
@@ -569,8 +547,7 @@ class InstanceTest(unittest.TestCase):
             self.assertFalse("Threw wrong exception")
 
     def test_reload_of_non_strict_with_special_field_name(self):
-        """Ensures reloading works for documents with meta strict == False
-        """
+        """Ensures reloading works for documents with meta strict == False."""
         class Post(Document):
             meta = {
                 'strict': False
@@ -591,8 +568,7 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(post.items, ["more lorem", "even more ipsum"])
 
     def test_dictionary_access(self):
-        """Ensure that dictionary-style field access works properly.
-        """
+        """Ensure that dictionary-style field access works properly."""
         person = self.Person(name='Test User', age=30, job=self.Job())
         self.assertEqual(person['name'], 'Test User')
 
@@ -634,8 +610,7 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(sub_doc.to_mongo().keys(), ['id'])
 
     def test_embedded_document(self):
-        """Ensure that embedded documents are set up correctly.
-        """
+        """Ensure that embedded documents are set up correctly."""
         class Comment(EmbeddedDocument):
             content = StringField()
 
@@ -643,8 +618,7 @@ class InstanceTest(unittest.TestCase):
         self.assertFalse('id' in Comment._fields)
 
     def test_embedded_document_instance(self):
-        """Ensure that embedded documents can reference parent instance
-        """
+        """Ensure that embedded documents can reference parent instance."""
         class Embedded(EmbeddedDocument):
             string = StringField()
 
@@ -652,6 +626,7 @@ class InstanceTest(unittest.TestCase):
             embedded_field = EmbeddedDocumentField(Embedded)
 
         Doc.drop_collection()
+
         doc = Doc(embedded_field=Embedded(string="Hi"))
         self.assertHasInstance(doc.embedded_field, doc)
 
@@ -661,7 +636,8 @@ class InstanceTest(unittest.TestCase):
 
     def test_embedded_document_complex_instance(self):
         """Ensure that embedded documents in complex fields can reference
-        parent instance"""
+        parent instance.
+        """
         class Embedded(EmbeddedDocument):
             string = StringField()
 
@@ -677,8 +653,7 @@ class InstanceTest(unittest.TestCase):
         self.assertHasInstance(doc.embedded_field[0], doc)
 
     def test_embedded_document_complex_instance_no_use_db_field(self):
-        """Ensure that use_db_field is propagated to list of Emb Docs
-        """
+        """Ensure that use_db_field is propagated to list of Emb Docs."""
         class Embedded(EmbeddedDocument):
             string = StringField(db_field='s')
 
@@ -690,7 +665,6 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(d['embedded_field'], [{'string': 'Hi'}])
 
     def test_instance_is_set_on_setattr(self):
-
         class Email(EmbeddedDocument):
             email = EmailField()
 
@@ -698,6 +672,7 @@ class InstanceTest(unittest.TestCase):
             email = EmbeddedDocumentField(Email)
 
         Account.drop_collection()
+
         acc = Account()
         acc.email = Email(email='test@example.com')
         self.assertHasInstance(acc._data["email"], acc)
@@ -707,7 +682,6 @@ class InstanceTest(unittest.TestCase):
         self.assertHasInstance(acc1._data["email"], acc1)
 
     def test_instance_is_set_on_setattr_on_embedded_document_list(self):
-
         class Email(EmbeddedDocument):
             email = EmailField()
 
@@ -853,32 +827,28 @@ class InstanceTest(unittest.TestCase):
         self.assertDbEqual([dict(other_doc.to_mongo()), dict(doc.to_mongo())])
 
     def test_save(self):
-        """Ensure that a document may be saved in the database.
-        """
+        """Ensure that a document may be saved in the database."""
+
         # Create person object and save it to the database
         person = self.Person(name='Test User', age=30)
         person.save()
+
         # Ensure that the object is in the database
         collection = self.db[self.Person._get_collection_name()]
         person_obj = collection.find_one({'name': 'Test User'})
         self.assertEqual(person_obj['name'], 'Test User')
         self.assertEqual(person_obj['age'], 30)
         self.assertEqual(person_obj['_id'], person.id)
-        # Test skipping validation on save
 
+        # Test skipping validation on save
         class Recipient(Document):
             email = EmailField(required=True)
 
-        recipient = Recipient(email='root@localhost')
+        recipient = Recipient(email='not-an-email')
         self.assertRaises(ValidationError, recipient.save)
-
-        try:
-            recipient.save(validate=False)
-        except ValidationError:
-            self.fail()
+        recipient.save(validate=False)
 
     def test_save_to_a_value_that_equates_to_false(self):
-
         class Thing(EmbeddedDocument):
             count = IntField()
 
@@ -898,7 +868,6 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(user.thing.count, 0)
 
     def test_save_max_recursion_not_hit(self):
-
         class Person(Document):
             name = StringField()
             parent = ReferenceField('self')
@@ -924,7 +893,6 @@ class InstanceTest(unittest.TestCase):
         p0.save()
 
     def test_save_max_recursion_not_hit_with_file_field(self):
-
         class Foo(Document):
             name = StringField()
             picture = FileField()
@@ -948,7 +916,6 @@ class InstanceTest(unittest.TestCase):
             self.assertEqual(b.picture, b.bar.picture, b.bar.bar.picture)
 
     def test_save_cascades(self):
-
         class Person(Document):
             name = StringField()
             parent = ReferenceField('self')
@@ -971,7 +938,6 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(p1.name, p.parent.name)
 
     def test_save_cascade_kwargs(self):
-
         class Person(Document):
             name = StringField()
             parent = ReferenceField('self')
@@ -992,7 +958,6 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(p1.name, p2.parent.name)
 
     def test_save_cascade_meta_false(self):
-
         class Person(Document):
             name = StringField()
             parent = ReferenceField('self')
@@ -1021,7 +986,6 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(p1.name, p.parent.name)
 
     def test_save_cascade_meta_true(self):
-
         class Person(Document):
             name = StringField()
             parent = ReferenceField('self')
@@ -1046,7 +1010,6 @@ class InstanceTest(unittest.TestCase):
         self.assertNotEqual(p1.name, p.parent.name)
 
     def test_save_cascades_generically(self):
-
         class Person(Document):
             name = StringField()
             parent = GenericReferenceField()
@@ -1072,7 +1035,6 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(p1.name, p.parent.name)
 
     def test_save_atomicity_condition(self):
-
         class Widget(Document):
             toggle = BooleanField(default=False)
             count = IntField(default=0)
@@ -1150,7 +1112,8 @@ class InstanceTest(unittest.TestCase):
 
     def test_update(self):
         """Ensure that an existing document is updated instead of be
-        overwritten."""
+        overwritten.
+        """
         # Create person object and save it to the database
         person = self.Person(name='Test User', age=30)
         person.save()
@@ -1254,7 +1217,6 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(2, self.Person.objects.count())
 
     def test_can_save_if_not_included(self):
-
         class EmbeddedDoc(EmbeddedDocument):
             pass
 
@@ -1341,10 +1303,7 @@ class InstanceTest(unittest.TestCase):
             doc2.update(set__name=doc1.name)
 
     def test_embedded_update(self):
-        """
-        Test update on `EmbeddedDocumentField` fields
-        """
-
+        """Test update on `EmbeddedDocumentField` fields."""
         class Page(EmbeddedDocument):
             log_message = StringField(verbose_name="Log message",
                                       required=True)
@@ -1365,11 +1324,9 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(site.page.log_message, "Error: Dummy message")
 
     def test_embedded_update_db_field(self):
+        """Test update on `EmbeddedDocumentField` fields when db_field
+        is other than default.
         """
-        Test update on `EmbeddedDocumentField` fields when db_field is other
-        than default.
-        """
-
         class Page(EmbeddedDocument):
             log_message = StringField(verbose_name="Log message",
                                       db_field="page_log_message",
@@ -1392,9 +1349,7 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(site.page.log_message, "Error: Dummy message")
 
     def test_save_only_changed_fields(self):
-        """Ensure save only sets / unsets changed fields
-        """
-
+        """Ensure save only sets / unsets changed fields."""
         class User(self.Person):
             active = BooleanField(default=True)
 
@@ -1514,8 +1469,8 @@ class InstanceTest(unittest.TestCase):
             self.assertEqual(q, 3)
 
     def test_set_unset_one_operation(self):
-        """Ensure that $set and $unset actions are performed in the same
-        operation.
+        """Ensure that $set and $unset actions are performed in the
+        same operation.
         """
         class FooBar(Document):
             foo = StringField(default=None)
@@ -1536,9 +1491,7 @@ class InstanceTest(unittest.TestCase):
             self.assertEqual(1, q)
 
     def test_save_only_changed_fields_recursive(self):
-        """Ensure save only sets / unsets changed fields
-        """
-
+        """Ensure save only sets / unsets changed fields."""
         class Comment(EmbeddedDocument):
             published = BooleanField(default=True)
 
@@ -1578,8 +1531,7 @@ class InstanceTest(unittest.TestCase):
         self.assertFalse(person.comments_dict['first_post'].published)
 
     def test_delete(self):
-        """Ensure that document may be deleted using the delete method.
-        """
+        """Ensure that document may be deleted using the delete method."""
         person = self.Person(name="Test User", age=30)
         person.save()
         self.assertEqual(self.Person.objects.count(), 1)
@@ -1587,33 +1539,34 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(self.Person.objects.count(), 0)
 
     def test_save_custom_id(self):
-        """Ensure that a document may be saved with a custom _id.
-        """
+        """Ensure that a document may be saved with a custom _id."""
+
         # Create person object and save it to the database
         person = self.Person(name='Test User', age=30,
                              id='497ce96f395f2f052a494fd4')
         person.save()
+
         # Ensure that the object is in the database with the correct _id
         collection = self.db[self.Person._get_collection_name()]
         person_obj = collection.find_one({'name': 'Test User'})
         self.assertEqual(str(person_obj['_id']), '497ce96f395f2f052a494fd4')
 
     def test_save_custom_pk(self):
-        """
-        Ensure that a document may be saved with a custom _id using pk alias.
+        """Ensure that a document may be saved with a custom _id using
+        pk alias.
         """
         # Create person object and save it to the database
         person = self.Person(name='Test User', age=30,
                              pk='497ce96f395f2f052a494fd4')
         person.save()
+
         # Ensure that the object is in the database with the correct _id
         collection = self.db[self.Person._get_collection_name()]
         person_obj = collection.find_one({'name': 'Test User'})
         self.assertEqual(str(person_obj['_id']), '497ce96f395f2f052a494fd4')
 
     def test_save_list(self):
-        """Ensure that a list field may be properly saved.
-        """
+        """Ensure that a list field may be properly saved."""
         class Comment(EmbeddedDocument):
             content = StringField()
 
@@ -1636,8 +1589,6 @@ class InstanceTest(unittest.TestCase):
         for comment_obj, comment in zip(post_obj['comments'], comments):
             self.assertEqual(comment_obj['content'], comment['content'])
 
-        BlogPost.drop_collection()
-
     def test_list_search_by_embedded(self):
         class User(Document):
             username = StringField(required=True)
@@ -1697,8 +1648,8 @@ class InstanceTest(unittest.TestCase):
             list(Page.objects.filter(comments__user=u3)))
 
     def test_save_embedded_document(self):
-        """Ensure that a document with an embedded document field may be
-        saved in the database.
+        """Ensure that a document with an embedded document field may
+        be saved in the database.
         """
         class EmployeeDetails(EmbeddedDocument):
             position = StringField()
@@ -1717,13 +1668,13 @@ class InstanceTest(unittest.TestCase):
         employee_obj = collection.find_one({'name': 'Test Employee'})
         self.assertEqual(employee_obj['name'], 'Test Employee')
         self.assertEqual(employee_obj['age'], 50)
+
         # Ensure that the 'details' embedded object saved correctly
         self.assertEqual(employee_obj['details']['position'], 'Developer')
 
     def test_embedded_update_after_save(self):
-        """
-        Test update of `EmbeddedDocumentField` attached to a newly saved
-        document.
+        """Test update of `EmbeddedDocumentField` attached to a newly
+        saved document.
         """
         class Page(EmbeddedDocument):
             log_message = StringField(verbose_name="Log message",
@@ -1744,8 +1695,8 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(site.page.log_message, "Error: Dummy message")
 
     def test_updating_an_embedded_document(self):
-        """Ensure that a document with an embedded document field may be
-        saved in the database.
+        """Ensure that a document with an embedded document field may
+        be saved in the database.
         """
         class EmployeeDetails(EmbeddedDocument):
             position = StringField()
@@ -1780,7 +1731,6 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(promoted_employee.details, None)
 
     def test_object_mixins(self):
-
         class NameMixin(object):
             name = StringField()
 
@@ -1819,9 +1769,9 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(t.count, 12)
 
     def test_save_reference(self):
-        """Ensure that a document reference field may be saved in the database.
+        """Ensure that a document reference field may be saved in the
+        database.
         """
-
         class BlogPost(Document):
             meta = {'collection': 'blogpost_1'}
             content = StringField()
@@ -1852,8 +1802,6 @@ class InstanceTest(unittest.TestCase):
         author = list(self.Person.objects(name='Test User'))[-1]
         self.assertEqual(author.age, 25)
 
-        BlogPost.drop_collection()
-
     def test_duplicate_db_fields_raise_invalid_document_error(self):
         """Ensure a InvalidDocumentError is thrown if duplicate fields
         declare the same db_field.
@@ -1864,7 +1812,7 @@ class InstanceTest(unittest.TestCase):
                 name2 = StringField(db_field='name')
 
     def test_invalid_son(self):
-        """Raise an error if loading invalid data"""
+        """Raise an error if loading invalid data."""
         class Occurrence(EmbeddedDocument):
             number = IntField()
 
@@ -1887,9 +1835,9 @@ class InstanceTest(unittest.TestCase):
             Word._from_son('this is not a valid SON dict')
 
     def test_reverse_delete_rule_cascade_and_nullify(self):
-        """Ensure that a referenced document is also deleted upon deletion.
+        """Ensure that a referenced document is also deleted upon
+        deletion.
         """
-
         class BlogPost(Document):
             content = StringField()
             author = ReferenceField(self.Person, reverse_delete_rule=CASCADE)
@@ -1944,7 +1892,8 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(Book.objects.count(), 0)
 
     def test_reverse_delete_rule_with_shared_id_among_collections(self):
-        """Ensure that cascade delete rule doesn't mix id among collections.
+        """Ensure that cascade delete rule doesn't mix id among
+        collections.
         """
         class User(Document):
             id = IntField(primary_key=True)
@@ -1975,10 +1924,9 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(Book.objects.get(), book_2)
 
     def test_reverse_delete_rule_with_document_inheritance(self):
-        """Ensure that a referenced document is also deleted upon deletion
-        of a child document.
+        """Ensure that a referenced document is also deleted upon
+        deletion of a child document.
         """
-
         class Writer(self.Person):
             pass
 
@@ -2010,10 +1958,9 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(BlogPost.objects.count(), 0)
 
     def test_reverse_delete_rule_cascade_and_nullify_complex_field(self):
-        """Ensure that a referenced document is also deleted upon deletion for
-        complex fields.
+        """Ensure that a referenced document is also deleted upon
+        deletion for complex fields.
         """
-
         class BlogPost(Document):
             content = StringField()
             authors = ListField(ReferenceField(
@@ -2022,7 +1969,6 @@ class InstanceTest(unittest.TestCase):
                 self.Person, reverse_delete_rule=NULLIFY))
 
         self.Person.drop_collection()
-
         BlogPost.drop_collection()
 
         author = self.Person(name='Test User')
@@ -2046,10 +1992,10 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(BlogPost.objects.count(), 0)
 
     def test_reverse_delete_rule_cascade_triggers_pre_delete_signal(self):
-        """ ensure the pre_delete signal is triggered upon a cascading deletion
-        setup a blog post with content, an author and editor
-        delete the author which triggers deletion of blogpost via cascade
-        blog post's pre_delete signal alters an editor attribute
+        """Ensure the pre_delete signal is triggered upon a cascading
+        deletion setup a blog post with content, an author and editor
+        delete the author which triggers deletion of blogpost via
+        cascade blog post's pre_delete signal alters an editor attribute.
         """
         class Editor(self.Person):
             review_queue = IntField(default=0)
@@ -2077,6 +2023,7 @@ class InstanceTest(unittest.TestCase):
 
         # delete the author, the post is also deleted due to the CASCADE rule
         author.delete()
+
         # the pre-delete signal should have decremented the editor's queue
         editor = Editor.objects(name='Max P.').get()
         self.assertEqual(editor.review_queue, 0)
@@ -2085,7 +2032,6 @@ class InstanceTest(unittest.TestCase):
         """Ensure that Bi-Directional relationships work with
         reverse_delete_rule
         """
-
         class Bar(Document):
             content = StringField()
             foo = ReferenceField('Foo')
@@ -2131,8 +2077,8 @@ class InstanceTest(unittest.TestCase):
                 mother = ReferenceField('Person', reverse_delete_rule=DENY)
 
     def test_reverse_delete_rule_cascade_recurs(self):
-        """Ensure that a chain of documents is also deleted upon cascaded
-        deletion.
+        """Ensure that a chain of documents is also deleted upon
+        cascaded deletion.
         """
         class BlogPost(Document):
             content = StringField()
@@ -2162,15 +2108,10 @@ class InstanceTest(unittest.TestCase):
         author.delete()
         self.assertEqual(Comment.objects.count(), 0)
 
-        self.Person.drop_collection()
-        BlogPost.drop_collection()
-        Comment.drop_collection()
-
     def test_reverse_delete_rule_deny(self):
-        """Ensure that a document cannot be referenced if there are still
-        documents referring to it.
+        """Ensure that a document cannot be referenced if there are
+        still documents referring to it.
         """
-
         class BlogPost(Document):
             content = StringField()
             author = ReferenceField(self.Person, reverse_delete_rule=DENY)
@@ -2198,11 +2139,7 @@ class InstanceTest(unittest.TestCase):
         author.delete()
         self.assertEqual(self.Person.objects.count(), 1)
 
-        self.Person.drop_collection()
-        BlogPost.drop_collection()
-
     def subclasses_and_unique_keys_works(self):
-
         class A(Document):
             pass
 
@@ -2218,12 +2155,9 @@ class InstanceTest(unittest.TestCase):
 
         self.assertEqual(A.objects.count(), 2)
         self.assertEqual(B.objects.count(), 1)
-        A.drop_collection()
-        B.drop_collection()
 
     def test_document_hash(self):
-        """Test document in list, dict, set
-        """
+        """Test document in list, dict, set."""
         class User(Document):
             pass
 
@@ -2266,11 +2200,9 @@ class InstanceTest(unittest.TestCase):
 
         # in Set
         all_user_set = set(User.objects.all())
-
         self.assertTrue(u1 in all_user_set)
 
     def test_picklable(self):
-
         pickle_doc = PickleTest(number=1, string="One", lists=['1', '2'])
         pickle_doc.embedded = PickleEmbedded()
         pickled_doc = pickle.dumps(pickle_doc)  # make sure pickling works even before the doc is saved
@@ -2296,7 +2228,6 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(pickle_doc.lists, ["1", "2", "3"])
 
     def test_regular_document_pickle(self):
-
         pickle_doc = PickleTest(number=1, string="One", lists=['1', '2'])
         pickled_doc = pickle.dumps(pickle_doc)  # make sure pickling works even before the doc is saved
         pickle_doc.save()
@@ -2319,7 +2250,6 @@ class InstanceTest(unittest.TestCase):
         fixtures.PickleTest = PickleTest
 
     def test_dynamic_document_pickle(self):
-
         pickle_doc = PickleDynamicTest(
             name="test", number=1, string="One", lists=['1', '2'])
         pickle_doc.embedded = PickleDynamicEmbedded(foo="Bar")
@@ -2358,7 +2288,6 @@ class InstanceTest(unittest.TestCase):
                 validate = DictField()
 
     def test_mutating_documents(self):
-
         class B(EmbeddedDocument):
             field1 = StringField(default='field1')
 
@@ -2366,6 +2295,7 @@ class InstanceTest(unittest.TestCase):
             b = EmbeddedDocumentField(B, default=lambda: B())
 
         A.drop_collection()
+
         a = A()
         a.save()
         a.reload()
@@ -2389,12 +2319,13 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(a.b.field2.c_field, 'new value')
 
     def test_can_save_false_values(self):
-        """Ensures you can save False values on save"""
+        """Ensures you can save False values on save."""
         class Doc(Document):
             foo = StringField()
             archived = BooleanField(default=False, required=True)
 
         Doc.drop_collection()
+
         d = Doc()
         d.save()
         d.archived = False
@@ -2403,11 +2334,12 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(Doc.objects(archived=False).count(), 1)
 
     def test_can_save_false_values_dynamic(self):
-        """Ensures you can save False values on dynamic docs"""
+        """Ensures you can save False values on dynamic docs."""
         class Doc(DynamicDocument):
             foo = StringField()
 
         Doc.drop_collection()
+
         d = Doc()
         d.save()
         d.archived = False
@@ -2447,7 +2379,7 @@ class InstanceTest(unittest.TestCase):
             Collection.update = orig_update
 
     def test_db_alias_tests(self):
-        """ DB Alias tests """
+        """DB Alias tests."""
         # mongoenginetest - Is default connection alias from setUp()
         # Register Aliases
         register_connection('testdb-1', 'mongoenginetest2')
@@ -2509,8 +2441,7 @@ class InstanceTest(unittest.TestCase):
             get_db("testdb-3")[AuthorBooks._get_collection_name()])
 
     def test_db_alias_overrides(self):
-        """db_alias can be overriden
-        """
+        """Test db_alias can be overriden."""
         # Register a connection with db_alias testdb-2
         register_connection('testdb-2', 'mongoenginetest2')
 
@@ -2534,8 +2465,7 @@ class InstanceTest(unittest.TestCase):
                          B._get_collection().database.name)
 
     def test_db_alias_propagates(self):
-        """db_alias propagates?
-        """
+        """db_alias propagates?"""
         register_connection('testdb-1', 'mongoenginetest2')
 
         class A(Document):
@@ -2548,8 +2478,7 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual('testdb-1', B._meta.get('db_alias'))
 
     def test_db_ref_usage(self):
-        """ DB Ref usage  in dict_fields"""
-
+        """DB Ref usage in dict_fields."""
         class User(Document):
             name = StringField()
 
@@ -2784,7 +2713,6 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(user.thing._data['data'], [1, 2, 3])
 
     def test_spaces_in_keys(self):
-
         class Embedded(DynamicEmbeddedDocument):
             pass
 
@@ -2873,7 +2801,6 @@ class InstanceTest(unittest.TestCase):
             log.machine = "127.0.0.1"
 
     def test_kwargs_simple(self):
-
         class Embedded(EmbeddedDocument):
             name = StringField()
 
@@ -2893,7 +2820,6 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(classic_doc._data, dict_doc._data)
 
     def test_kwargs_complex(self):
-
         class Embedded(EmbeddedDocument):
             name = StringField()
 
@@ -2916,36 +2842,35 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(classic_doc._data, dict_doc._data)
 
     def test_positional_creation(self):
-        """Ensure that document may be created using positional arguments.
-        """
+        """Ensure that document may be created using positional arguments."""
         person = self.Person("Test User", 42)
         self.assertEqual(person.name, "Test User")
         self.assertEqual(person.age, 42)
 
     def test_mixed_creation(self):
-        """Ensure that document may be created using mixed arguments.
-        """
+        """Ensure that document may be created using mixed arguments."""
         person = self.Person("Test User", age=42)
         self.assertEqual(person.name, "Test User")
         self.assertEqual(person.age, 42)
 
     def test_positional_creation_embedded(self):
-        """Ensure that embedded document may be created using positional arguments.
+        """Ensure that embedded document may be created using positional
+        arguments.
         """
         job = self.Job("Test Job", 4)
         self.assertEqual(job.name, "Test Job")
         self.assertEqual(job.years, 4)
 
     def test_mixed_creation_embedded(self):
-        """Ensure that embedded document may be created using mixed arguments.
+        """Ensure that embedded document may be created using mixed
+        arguments.
         """
         job = self.Job("Test Job", years=4)
         self.assertEqual(job.name, "Test Job")
         self.assertEqual(job.years, 4)
 
     def test_mixed_creation_dynamic(self):
-        """Ensure that document may be created using mixed arguments.
-        """
+        """Ensure that document may be created using mixed arguments."""
         class Person(DynamicDocument):
             name = StringField()
 
@@ -2954,14 +2879,14 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(person.age, 42)
 
     def test_bad_mixed_creation(self):
-        """Ensure that document gives correct error when duplicating arguments
+        """Ensure that document gives correct error when duplicating
+        arguments.
         """
         with self.assertRaises(TypeError):
             return self.Person("Test User", 42, name="Bad User")
 
     def test_data_contains_id_field(self):
-        """Ensure that asking for _data returns 'id'
-        """
+        """Ensure that asking for _data returns 'id'."""
         class Person(Document):
             name = StringField()
 
@@ -2973,7 +2898,6 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(person._data.get('id'), person.id)
 
     def test_complex_nesting_document_and_embedded_document(self):
-
         class Macro(EmbeddedDocument):
             value = DynamicField(default="UNDEFINED")
 
@@ -3016,7 +2940,6 @@ class InstanceTest(unittest.TestCase):
             system.nodes["node"].parameters["param"].macros["test"].value)
 
     def test_embedded_document_equality(self):
-
         class Test(Document):
             field = StringField(required=True)
 
@@ -3202,8 +3125,7 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(idx, 2)
 
     def test_falsey_pk(self):
-        """Ensure that we can create and update a document with Falsey PK.
-        """
+        """Ensure that we can create and update a document with Falsey PK."""
         class Person(Document):
             age = IntField(primary_key=True)
             height = FloatField()

tests/fields/fields.py

@@ -5,8 +5,11 @@ import uuid
 import math
 import itertools
 import re
+import pymongo
+import sys
 
 from nose.plugins.skip import SkipTest
+from collections import OrderedDict
 import six
 
 try:
@@ -25,9 +28,12 @@ except ImportError:
 from mongoengine import *
 from mongoengine.connection import get_db
 from mongoengine.base import (BaseDict, BaseField, EmbeddedDocumentList,
-                              _document_registry)
+                              _document_registry, TopLevelDocumentMetaclass)
 
-from tests.utils import MongoDBTestCase
+from tests.utils import MongoDBTestCase, MONGO_TEST_DB
+from mongoengine.python_support import IS_PYMONGO_3
+if IS_PYMONGO_3:
+    from bson import CodecOptions
 
 __all__ = ("FieldTest", "EmbeddedDocumentListFieldTestCase")
 
@@ -337,8 +343,6 @@ class FieldTest(MongoDBTestCase):
         class Link(Document):
             url = URLField()
 
-        Link.drop_collection()
-
         link = Link()
         link.url = 'google'
         self.assertRaises(ValidationError, link.validate)
@@ -351,8 +355,6 @@ class FieldTest(MongoDBTestCase):
         class Link(Document):
             url = URLField()
 
-        Link.drop_collection()
-
         link = Link()
         link.url = u'http://привет.com'
 
@@ -1173,6 +1175,14 @@ class FieldTest(MongoDBTestCase):
         post.reload()
         self.assertEqual(post.info, ['0', '1', '2', '3', 'a', '5'])
 
+        # __setitem__(index, value) with a negative index
+        reset_post()
+        post.info[-2] = 'a'
+        self.assertEqual(post.info, ['0', '1', '2', '3', 'a', '5'])
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, ['0', '1', '2', '3', 'a', '5'])
+
         # '__setitem__(slice(i, j), listB)'
         # aka 'listA[i:j] = listB'
         # aka 'setitem(listA, slice(i, j), listB)'
@@ -1183,6 +1193,16 @@ class FieldTest(MongoDBTestCase):
         post.reload()
         self.assertEqual(post.info, ['0', 'h', 'e', 'l', 'l', 'o', '3', '4', '5'])
 
+        # '__setitem__(slice(i, j), listB)' with negative i and j
+        reset_post()
+        post.info[-5:-3] = ['h', 'e', 'l', 'l', 'o']
+        self.assertEqual(post.info, ['0', 'h', 'e', 'l', 'l', 'o', '3', '4', '5'])
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, ['0', 'h', 'e', 'l', 'l', 'o', '3', '4', '5'])
+
+        # negative
+
         # 'append'
         reset_post()
         post.info.append('h')
@@ -1883,6 +1903,51 @@ class FieldTest(MongoDBTestCase):
         doc = self.db.test.find_one()
         self.assertEqual(doc['x']['i'], 2)
 
+    def test_double_embedded_db_field(self):
+        """Make sure multiple layers of embedded docs resolve db fields
+        properly and can be initialized using dicts.
+        """
+        class C(EmbeddedDocument):
+            txt = StringField()
+
+        class B(EmbeddedDocument):
+            c = EmbeddedDocumentField(C, db_field='fc')
+
+        class A(Document):
+            b = EmbeddedDocumentField(B, db_field='fb')
+
+        a = A(
+            b=B(
+                c=C(txt='hi')
+            )
+        )
+        a.validate()
+
+        a = A(b={'c': {'txt': 'hi'}})
+        a.validate()
+
+    def test_double_embedded_db_field_from_son(self):
+        """Make sure multiple layers of embedded docs resolve db fields
+        from SON properly.
+        """
+        class C(EmbeddedDocument):
+            txt = StringField()
+
+        class B(EmbeddedDocument):
+            c = EmbeddedDocumentField(C, db_field='fc')
+
+        class A(Document):
+            b = EmbeddedDocumentField(B, db_field='fb')
+
+        a = A._from_son(SON([
+            ('fb', SON([
+                ('fc', SON([
+                    ('txt', 'hi')
+                ]))
+            ]))
+        ]))
+        self.assertEqual(a.b.c.txt, 'hi')
+
     def test_embedded_document_validation(self):
         """Ensure that invalid embedded documents cannot be assigned to
         embedded document fields.
@@ -2025,6 +2090,12 @@ class FieldTest(MongoDBTestCase):
         post1.author = post2
         self.assertRaises(ValidationError, post1.validate)
 
+        # Ensure ObjectID's are accepted as references
+        user_object_id = user.pk
+        post3 = BlogPost(content="Chips and curry sauce taste good.")
+        post3.author = user_object_id
+        post3.save()
+
         # Make sure referencing a saved document of the right type works
         user.save()
         post1.author = user
@@ -2035,6 +2106,20 @@ class FieldTest(MongoDBTestCase):
         post1.author = post2
         self.assertRaises(ValidationError, post1.validate)
 
+    def test_objectid_reference_fields(self):
+        """Make sure storing Object ID references works."""
+        class Person(Document):
+            name = StringField()
+            parent = ReferenceField('self')
+
+        Person.drop_collection()
+
+        p1 = Person(name="John").save()
+        Person(name="Ross", parent=p1.pk).save()
+
+        p = Person.objects.get(name="Ross")
+        self.assertEqual(p.parent, p1)
+
     def test_dbref_reference_fields(self):
         """Make sure storing references as bson.dbref.DBRef works."""
         class Person(Document):
@@ -3368,23 +3453,99 @@ class FieldTest(MongoDBTestCase):
         class User(Document):
             email = EmailField()
 
-        user = User(email="ross@example.com")
-        self.assertTrue(user.validate() is None)
+        user = User(email='ross@example.com')
+        user.validate()
 
-        user = User(email="ross@example.co.uk")
-        self.assertTrue(user.validate() is None)
+        user = User(email='ross@example.co.uk')
+        user.validate()
 
-        user = User(email=("Kofq@rhom0e4klgauOhpbpNdogawnyIKvQS0wk2mjqrgGQ5S"
-                           "aJIazqqWkm7.net"))
-        self.assertTrue(user.validate() is None)
+        user = User(email=('Kofq@rhom0e4klgauOhpbpNdogawnyIKvQS0wk2mjqrgGQ5S'
+                           'aJIazqqWkm7.net'))
+        user.validate()
 
-        user = User(email="new-tld@example.technology")
-        self.assertTrue(user.validate() is None)
+        user = User(email='new-tld@example.technology')
+        user.validate()
 
+        user = User(email='ross@example.com.')
+        self.assertRaises(ValidationError, user.validate)
+
+        # unicode domain
+        user = User(email=u'user@пример.рф')
+        user.validate()
+
+        # invalid unicode domain
+        user = User(email=u'user@пример')
+        self.assertRaises(ValidationError, user.validate)
+
+        # invalid data type
+        user = User(email=123)
+        self.assertRaises(ValidationError, user.validate)
+
+    def test_email_field_unicode_user(self):
+        # Don't run this test on pypy3, which doesn't support unicode regex:
+        # https://bitbucket.org/pypy/pypy/issues/1821/regular-expression-doesnt-find-unicode
+        if sys.version_info[:2] == (3, 2):
+            raise SkipTest('unicode email addresses are not supported on PyPy 3')
+
+        class User(Document):
+            email = EmailField()
+
+        # unicode user shouldn't validate by default...
+        user = User(email=u'Dörte@Sörensen.example.com')
+        self.assertRaises(ValidationError, user.validate)
+
+        # ...but it should be fine with allow_utf8_user set to True
+        class User(Document):
+            email = EmailField(allow_utf8_user=True)
+
+        user = User(email=u'Dörte@Sörensen.example.com')
+        user.validate()
+
+    def test_email_field_domain_whitelist(self):
+        class User(Document):
+            email = EmailField()
+
+        # localhost domain shouldn't validate by default...
         user = User(email='me@localhost')
         self.assertRaises(ValidationError, user.validate)
 
-        user = User(email="ross@example.com.")
+        # ...but it should be fine if it's whitelisted
+        class User(Document):
+            email = EmailField(domain_whitelist=['localhost'])
+
+        user = User(email='me@localhost')
+        user.validate()
+
+    def test_email_field_ip_domain(self):
+        class User(Document):
+            email = EmailField()
+
+        valid_ipv4 = 'email@[127.0.0.1]'
+        valid_ipv6 = 'email@[2001:dB8::1]'
+        invalid_ip = 'email@[324.0.0.1]'
+
+        # IP address as a domain shouldn't validate by default...
+        user = User(email=valid_ipv4)
+        self.assertRaises(ValidationError, user.validate)
+
+        user = User(email=valid_ipv6)
+        self.assertRaises(ValidationError, user.validate)
+
+        user = User(email=invalid_ip)
+        self.assertRaises(ValidationError, user.validate)
+
+        # ...but it should be fine with allow_ip_domain set to True
+        class User(Document):
+            email = EmailField(allow_ip_domain=True)
+
+        user = User(email=valid_ipv4)
+        user.validate()
+
+        user = User(email=valid_ipv6)
+        user.validate()
+
+        # invalid IP should still fail validation
+        user = User(email=invalid_ip)
         self.assertRaises(ValidationError, user.validate)
 
     def test_email_field_honors_regex(self):
@@ -4027,6 +4188,67 @@ class EmbeddedDocumentListFieldTestCase(MongoDBTestCase):
         self.assertTrue(hasattr(CustomData.c_field, 'custom_data'))
         self.assertEqual(custom_data['a'], CustomData.c_field.custom_data['a'])
 
+    def test_dynamicfield_with_container_class(self):
+        """
+        Tests that object can be stored in order by DynamicField class
+        with container_class parameter.
+        """
+        raw_data = [('d', 1), ('c', 2), ('b', 3), ('a', 4)]
+
+        class Doc(Document):
+            ordered_data = DynamicField(container_class=OrderedDict)
+            unordered_data = DynamicField()
+
+        Doc.drop_collection()
+
+        doc = Doc(ordered_data=OrderedDict(raw_data), unordered_data=dict(raw_data)).save()
+
+        # checks that the data is in order
+        self.assertEqual(type(doc.ordered_data), OrderedDict)
+        self.assertEqual(type(doc.unordered_data), dict)
+        self.assertEqual(','.join(doc.ordered_data.keys()), 'd,c,b,a')
+
+        # checks that the data is stored to the database in order
+        pymongo_db = pymongo.MongoClient()[MONGO_TEST_DB]
+        if IS_PYMONGO_3:
+            codec_option = CodecOptions(document_class=OrderedDict)
+            db_doc = pymongo_db.doc.with_options(codec_options=codec_option).find_one()
+        else:
+            db_doc = pymongo_db.doc.find_one(as_class=OrderedDict)
+
+        self.assertEqual(','.join(doc.ordered_data.keys()), 'd,c,b,a')
+
+    def test_dynamicfield_with_wrong_container_class(self):
+        with self.assertRaises(ValidationError):
+            class DocWithInvalidField:
+                data = DynamicField(container_class=list)
+
+    def test_dynamicfield_with_wrong_container_class_and_reload_docuemnt(self):
+        # This is because 'codec_options' is supported on pymongo3 or later
+        if IS_PYMONGO_3:
+            class OrderedDocument(Document):
+                my_metaclass = TopLevelDocumentMetaclass
+                __metaclass__ = TopLevelDocumentMetaclass
+
+                @classmethod
+                def _get_collection(cls):
+                    collection = super(OrderedDocument, cls)._get_collection()
+                    opts = CodecOptions(document_class=OrderedDict)
+
+                    return collection.with_options(codec_options=opts)
+
+            raw_data = [('d', 1), ('c', 2), ('b', 3), ('a', 4)]
+
+            class Doc(OrderedDocument):
+                data = DynamicField(container_class=OrderedDict)
+
+            Doc.drop_collection()
+
+            doc = Doc(data=OrderedDict(raw_data)).save()
+            doc.reload()
+
+            self.assertEqual(type(doc.data), OrderedDict)
+            self.assertEqual(','.join(doc.data.keys()), 'd,c,b,a')
 
 class CachedReferenceFieldTest(MongoDBTestCase):
 

tests/queryset/queryset.py

@@ -4962,20 +4962,6 @@ class QuerySetTest(unittest.TestCase):
         for p in Person.objects():
             self.assertEqual(p.name, 'a')
 
-    def test_last_field_name_like_operator(self):
-        class EmbeddedItem(EmbeddedDocument):
-            type = StringField()
-
-        class Doc(Document):
-            item = EmbeddedDocumentField(EmbeddedItem)
-
-        Doc.drop_collection()
-
-        doc = Doc(item=EmbeddedItem(type="axe"))
-        doc.save()
-
-        self.assertEqual(1, Doc.objects(item__type__="axe").count())
-
     def test_len_during_iteration(self):
         """Tests that calling len on a queyset during iteration doesn't
         stop paging.

tests/test_dereference.py

@@ -2,10 +2,15 @@
 import unittest
 
 from bson import DBRef, ObjectId
+from collections import OrderedDict
 
 from mongoengine import *
 from mongoengine.connection import get_db
 from mongoengine.context_managers import query_counter
+from mongoengine.python_support import IS_PYMONGO_3
+from mongoengine.base import TopLevelDocumentMetaclass
+if IS_PYMONGO_3:
+    from bson import CodecOptions
 
 
 class FieldTest(unittest.TestCase):
@@ -1287,5 +1292,70 @@ class FieldTest(unittest.TestCase):
 
             self.assertEqual(q, 2)
 
+    def test_dynamic_field_dereference(self):
+        class Merchandise(Document):
+            name = StringField()
+            price = IntField()
+
+        class Store(Document):
+            merchandises = DynamicField()
+
+        Merchandise.drop_collection()
+        Store.drop_collection()
+
+        merchandises = {
+            '#1': Merchandise(name='foo', price=100).save(),
+            '#2': Merchandise(name='bar', price=120).save(),
+            '#3': Merchandise(name='baz', price=110).save(),
+        }
+        Store(merchandises=merchandises).save()
+
+        store = Store.objects().first()
+        for obj in store.merchandises.values():
+            self.assertFalse(isinstance(obj, Merchandise))
+
+        store.select_related()
+        for obj in store.merchandises.values():
+            self.assertTrue(isinstance(obj, Merchandise))
+
+    def test_dynamic_field_dereference_with_ordering_guarantee_on_pymongo3(self):
+        # This is because 'codec_options' is supported on pymongo3 or later
+        if IS_PYMONGO_3:
+            class OrderedDocument(Document):
+                my_metaclass = TopLevelDocumentMetaclass
+                __metaclass__ = TopLevelDocumentMetaclass
+
+                @classmethod
+                def _get_collection(cls):
+                    collection = super(OrderedDocument, cls)._get_collection()
+                    opts = CodecOptions(document_class=OrderedDict)
+
+                    return collection.with_options(codec_options=opts)
+
+            class Merchandise(Document):
+                name = StringField()
+                price = IntField()
+
+            class Store(OrderedDocument):
+                merchandises = DynamicField(container_class=OrderedDict)
+
+            Merchandise.drop_collection()
+            Store.drop_collection()
+
+            merchandises = OrderedDict()
+            merchandises['#1'] = Merchandise(name='foo', price=100).save()
+            merchandises['#2'] = Merchandise(name='bar', price=120).save()
+            merchandises['#3'] = Merchandise(name='baz', price=110).save()
+
+            Store(merchandises=merchandises).save()
+
+            store = Store.objects().first()
+
+            store.select_related()
+
+            # confirms that the load data order is same with the one at storing
+            self.assertTrue(type(store.merchandises), OrderedDict)
+            self.assertEqual(','.join(store.merchandises.keys()), '#1,#2,#3')
+
 if __name__ == '__main__':
     unittest.main()