make EmbeddedDocument not hashable by default

added a feature to save object data in order

docs/changelog.rst

@@ -5,11 +5,27 @@ Changelog
 Development
 ===========
 - (Fill this out as you fix issues and develop your features).
-- Fixed using sets in field choices #1481
+
+Changes in 0.13.0
+=================
+- POTENTIAL BREAKING CHANGE: Added Unicode support to the `EmailField`, see
+  docs/upgrade.rst for details.
+
+Changes in 0.12.0
+=================
 - POTENTIAL BREAKING CHANGE: Fixed limit/skip/hint/batch_size chaining #1476
 - POTENTIAL BREAKING CHANGE: Changed a public `QuerySet.clone_into` method to a private `QuerySet._clone_into` #1476
+- Fixed the way `Document.objects.create` works with duplicate IDs #1485
 - Fixed connecting to a replica set with PyMongo 2.x #1436
+- Fixed using sets in field choices #1481
+- Fixed deleting items from a `ListField` #1318
 - Fixed an obscure error message when filtering by `field__in=non_iterable`. #1237
+- Fixed behavior of a `dec` update operator #1450
+- Added a `rename` update operator #1454
+- Added validation for the `db_field` parameter #1448
+- Fixed the error message displayed when querying an `EmbeddedDocumentField` by an invalid value #1440
+- Fixed the error message displayed when validating unicode URLs #1486
+- Raise an error when trying to save an abstract document #1449
 
 Changes in 0.11.0
 =================

docs/upgrade.rst

@@ -6,6 +6,20 @@ Development
 ***********
 (Fill this out whenever you introduce breaking changes to MongoEngine)
 
+0.13.0
+******
+This release adds Unicode support to the `EmailField` and changes its
+structure significantly. Previously, email addresses containing Unicode
+characters didn't work at all. Starting with v0.13.0, domains with Unicode
+characters are supported out of the box, meaning some emails that previously
+didn't pass validation now do. Make sure the rest of your application can
+accept such email addresses. Additionally, if you subclassed the `EmailField`
+in your application and overrode `EmailField.EMAIL_REGEX`, you will have to
+adjust your code to override `EmailField.USER_REGEX`, `EmailField.DOMAIN_REGEX`,
+and potentially `EmailField.UTF8_USER_REGEX`.
+
+0.12.0
+******
 This release includes various fixes for the `BaseQuerySet` methods and how they
 are chained together. Since version 0.10.1 applying limit/skip/hint/batch_size
 to an already-existing queryset wouldn't modify the underlying PyMongo cursor.

mongoengine/__init__.py

@@ -23,7 +23,7 @@ __all__ = (list(document.__all__) + list(fields.__all__) +
            list(signals.__all__) + list(errors.__all__))
 
 
-VERSION = (0, 11, 0)
+VERSION = (0, 13, 0)
 
 
 def get_version():

mongoengine/base/document.py

@@ -272,13 +272,6 @@ class BaseDocument(object):
     def __ne__(self, other):
         return not self.__eq__(other)
 
-    def __hash__(self):
-        if getattr(self, 'pk', None) is None:
-            # For new object
-            return super(BaseDocument, self).__hash__()
-        else:
-            return hash(self.pk)
-
     def clean(self):
         """
         Hook for doing document level data cleaning before validation is run.
@@ -684,8 +677,13 @@ class BaseDocument(object):
         # class if unavailable
         class_name = son.get('_cls', cls._class_name)
 
-        # Convert SON to a dict, making sure each key is a string
-        data = {str(key): value for key, value in son.iteritems()}
+        # Convert SON to a data dict, making sure each key is a string and
+        # corresponds to the right db field.
+        data = {}
+        for key, value in son.iteritems():
+            key = str(key)
+            key = cls._db_field_map.get(key, key)
+            data[key] = value
 
         # Return correct subclass for document type
         if class_name != cls._class_name:

mongoengine/dereference.py

@@ -1,3 +1,4 @@
+from collections import OrderedDict
 from bson import DBRef, SON
 import six
 
@@ -201,6 +202,10 @@ class DeReference(object):
             as_tuple = isinstance(items, tuple)
             iterator = enumerate(items)
             data = []
+        elif isinstance(items, OrderedDict):
+            is_list = False
+            iterator = items.iteritems()
+            data = OrderedDict()
         else:
             is_list = False
             iterator = items.iteritems()

mongoengine/document.py

@@ -60,6 +60,12 @@ class EmbeddedDocument(BaseDocument):
     my_metaclass = DocumentMetaclass
     __metaclass__ = DocumentMetaclass
 
+    # A generic embedded document doesn't have any immutable properties
+    # that describe it uniquely, hence it shouldn't be hashable. You can
+    # define your own __hash__ method on a subclass if you need your
+    # embedded documents to be hashable.
+    __hash__ = None
+
     def __init__(self, *args, **kwargs):
         super(EmbeddedDocument, self).__init__(*args, **kwargs)
         self._instance = None
@@ -160,6 +166,15 @@ class Document(BaseDocument):
         """Set the primary key."""
         return setattr(self, self._meta['id_field'], value)
 
+    def __hash__(self):
+        """Return the hash based on the PK of this document. If it's new
+        and doesn't have a PK yet, return the default object hash instead.
+        """
+        if self.pk is None:
+            return super(BaseDocument, self).__hash__()
+        else:
+            return hash(self.pk)
+
     @classmethod
     def _get_db(cls):
         """Some Model using other db_alias"""

mongoengine/fields.py

@@ -2,9 +2,11 @@ import datetime
 import decimal
 import itertools
 import re
+import socket
 import time
 import uuid
 import warnings
+from collections import Mapping
 from operator import itemgetter
 
 from bson import Binary, DBRef, ObjectId, SON
@@ -153,21 +155,105 @@ class EmailField(StringField):
 
     .. versionadded:: 0.4
     """
-
-    EMAIL_REGEX = re.compile(
-        # dot-atom
-        r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*"
-        # quoted-string
-        r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-011\013\014\016-\177])*"'
-        # domain (max length of an ICAAN TLD is 22 characters)
-        r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}|[A-Z0-9-]{2,}(?<!-))$', re.IGNORECASE
+    USER_REGEX = re.compile(
+        # `dot-atom` defined in RFC 5322 Section 3.2.3.
+        r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*\Z"
+        # `quoted-string` defined in RFC 5322 Section 3.2.4.
+        r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-\011\013\014\016-\177])*"\Z)',
+        re.IGNORECASE
     )
 
+    UTF8_USER_REGEX = re.compile(
+        six.u(
+            # RFC 6531 Section 3.3 extends `atext` (used by dot-atom) to
+            # include `UTF8-non-ascii`.
+            r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z\u0080-\U0010FFFF]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z\u0080-\U0010FFFF]+)*\Z"
+            # `quoted-string`
+            r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-\011\013\014\016-\177])*"\Z)'
+        ), re.IGNORECASE | re.UNICODE
+    )
+
+    DOMAIN_REGEX = re.compile(
+        r'((?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+)(?:[A-Z0-9-]{2,63}(?<!-))\Z',
+        re.IGNORECASE
+    )
+
+    error_msg = u'Invalid email address: %s'
+
+    def __init__(self, domain_whitelist=None, allow_utf8_user=False,
+                 allow_ip_domain=False, *args, **kwargs):
+        """Initialize the EmailField.
+
+        Args:
+            domain_whitelist (list) - list of otherwise invalid domain
+                                      names which you'd like to support.
+            allow_utf8_user (bool) - if True, the user part of the email
+                                     address can contain UTF8 characters.
+                                     False by default.
+            allow_ip_domain (bool) - if True, the domain part of the email
+                                     can be a valid IPv4 or IPv6 address.
+        """
+        self.domain_whitelist = domain_whitelist or []
+        self.allow_utf8_user = allow_utf8_user
+        self.allow_ip_domain = allow_ip_domain
+        super(EmailField, self).__init__(*args, **kwargs)
+
+    def validate_user_part(self, user_part):
+        """Validate the user part of the email address. Return True if
+        valid and False otherwise.
+        """
+        if self.allow_utf8_user:
+            return self.UTF8_USER_REGEX.match(user_part)
+        return self.USER_REGEX.match(user_part)
+
+    def validate_domain_part(self, domain_part):
+        """Validate the domain part of the email address. Return True if
+        valid and False otherwise.
+        """
+        # Skip domain validation if it's in the whitelist.
+        if domain_part in self.domain_whitelist:
+            return True
+
+        if self.DOMAIN_REGEX.match(domain_part):
+            return True
+
+        # Validate IPv4/IPv6, e.g. user@[192.168.0.1]
+        if (
+            self.allow_ip_domain and
+            domain_part[0] == '[' and
+            domain_part[-1] == ']'
+        ):
+            for addr_family in (socket.AF_INET, socket.AF_INET6):
+                try:
+                    socket.inet_pton(addr_family, domain_part[1:-1])
+                    return True
+                except (socket.error, UnicodeEncodeError):
+                    pass
+
+        return False
+
     def validate(self, value):
-        if not EmailField.EMAIL_REGEX.match(value):
-            self.error('Invalid email address: %s' % value)
         super(EmailField, self).validate(value)
 
+        if '@' not in value:
+            self.error(self.error_msg % value)
+
+        user_part, domain_part = value.rsplit('@', 1)
+
+        # Validate the user part.
+        if not self.validate_user_part(user_part):
+            self.error(self.error_msg % value)
+
+        # Validate the domain and, if invalid, see if it's IDN-encoded.
+        if not self.validate_domain_part(domain_part):
+            try:
+                domain_part = domain_part.encode('idna').decode('ascii')
+            except UnicodeError:
+                self.error(self.error_msg % value)
+            else:
+                if not self.validate_domain_part(domain_part):
+                    self.error(self.error_msg % value)
+
 
 class IntField(BaseField):
     """32-bit integer field."""
@@ -619,6 +705,14 @@ class DynamicField(BaseField):
 
     Used by :class:`~mongoengine.DynamicDocument` to handle dynamic data"""
 
+    def __init__(self, container_class=dict, *args, **kwargs):
+        self._container_cls = container_class
+        if not issubclass(self._container_cls, Mapping):
+            self.error('The class that is specified in `container_class` parameter '
+                       'must be a subclass of `dict`.')
+
+        super(DynamicField, self).__init__(*args, **kwargs)
+
     def to_mongo(self, value, use_db_field=True, fields=None):
         """Convert a Python type to a MongoDB compatible type.
         """
@@ -644,7 +738,7 @@ class DynamicField(BaseField):
             is_list = True
             value = {k: v for k, v in enumerate(value)}
 
-        data = {}
+        data = self._container_cls()
         for k, v in value.iteritems():
             data[k] = self.to_mongo(v, use_db_field, fields)
 
@@ -998,8 +1092,8 @@ class ReferenceField(BaseField):
 
     def validate(self, value):
 
-        if not isinstance(value, (self.document_type, DBRef)):
-            self.error('A ReferenceField only accepts DBRef or documents')
+        if not isinstance(value, (self.document_type, DBRef, ObjectId)):
+            self.error('A ReferenceField only accepts DBRef, ObjectId or documents')
 
         if isinstance(value, Document) and value.id is None:
             self.error('You can only reference documents once they have been '

mongoengine/queryset/base.py

@@ -158,44 +158,49 @@ class BaseQuerySet(object):
         # self._cursor
 
     def __getitem__(self, key):
-        """Support skip and limit using getitem and slicing syntax."""
+        """Return a document instance corresponding to a given index if
+        the key is an integer. If the key is a slice, translate its
+        bounds into a skip and a limit, and return a cloned queryset
+        with that skip/limit applied. For example:
+
+        >>> User.objects[0]
+        <User: User object>
+        >>> User.objects[1:3]
+        [<User: User object>, <User: User object>]
+        """
         queryset = self.clone()
 
-        # Slice provided
+        # Handle a slice
         if isinstance(key, slice):
-            try:
-                queryset._cursor_obj = queryset._cursor[key]
-                queryset._skip, queryset._limit = key.start, key.stop
-                if key.start and key.stop:
-                    queryset._limit = key.stop - key.start
-            except IndexError as err:
-                # PyMongo raises an error if key.start == key.stop, catch it,
-                # bin it, kill it.
-                start = key.start or 0
-                if start >= 0 and key.stop >= 0 and key.step is None:
-                    if start == key.stop:
-                        queryset.limit(0)
-                        queryset._skip = key.start
-                        queryset._limit = key.stop - start
-                        return queryset
-                raise err
+            queryset._cursor_obj = queryset._cursor[key]
+            queryset._skip, queryset._limit = key.start, key.stop
+            if key.start and key.stop:
+                queryset._limit = key.stop - key.start
+
             # Allow further QuerySet modifications to be performed
             return queryset
-        # Integer index provided
+
+        # Handle an index
         elif isinstance(key, int):
             if queryset._scalar:
                 return queryset._get_scalar(
-                    queryset._document._from_son(queryset._cursor[key],
-                                                 _auto_dereference=self._auto_dereference,
-                                                 only_fields=self.only_fields))
+                    queryset._document._from_son(
+                        queryset._cursor[key],
+                        _auto_dereference=self._auto_dereference,
+                        only_fields=self.only_fields
+                    )
+                )
 
             if queryset._as_pymongo:
                 return queryset._get_as_pymongo(queryset._cursor[key])
-            return queryset._document._from_son(queryset._cursor[key],
-                                                _auto_dereference=self._auto_dereference,
-                                                only_fields=self.only_fields)
 
-        raise AttributeError
+            return queryset._document._from_son(
+                queryset._cursor[key],
+                _auto_dereference=self._auto_dereference,
+                only_fields=self.only_fields
+            )
+
+        raise AttributeError('Provide a slice or an integer index')
 
     def __iter__(self):
         raise NotImplementedError

tests/document/instance.py

@@ -844,7 +844,7 @@ class InstanceTest(unittest.TestCase):
         class Recipient(Document):
             email = EmailField(required=True)
 
-        recipient = Recipient(email='root@localhost')
+        recipient = Recipient(email='not-an-email')
         self.assertRaises(ValidationError, recipient.save)
         recipient.save(validate=False)
 
@@ -2164,7 +2164,7 @@ class InstanceTest(unittest.TestCase):
         class BlogPost(Document):
             pass
 
-        # Clear old datas
+        # Clear old data
         User.drop_collection()
         BlogPost.drop_collection()
 
@@ -2176,17 +2176,18 @@ class InstanceTest(unittest.TestCase):
         b1 = BlogPost.objects.create()
         b2 = BlogPost.objects.create()
 
-        # in List
+        # Make sure docs are properly identified in a list (__eq__ is used
+        # for the comparison).
         all_user_list = list(User.objects.all())
-
         self.assertTrue(u1 in all_user_list)
         self.assertTrue(u2 in all_user_list)
         self.assertTrue(u3 in all_user_list)
-        self.assertFalse(u4 in all_user_list)  # New object
-        self.assertFalse(b1 in all_user_list)  # Other object
-        self.assertFalse(b2 in all_user_list)  # Other object
+        self.assertTrue(u4 not in all_user_list)  # New object
+        self.assertTrue(b1 not in all_user_list)  # Other object
+        self.assertTrue(b2 not in all_user_list)  # Other object
 
-        # in Dict
+        # Make sure docs can be used as keys in a dict (__hash__ is used
+        # for hashing the docs).
         all_user_dic = {}
         for u in User.objects.all():
             all_user_dic[u] = "OK"
@@ -2198,9 +2199,20 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(all_user_dic.get(b1, False), False)  # Other object
         self.assertEqual(all_user_dic.get(b2, False), False)  # Other object
 
-        # in Set
+        # Make sure docs are properly identified in a set (__hash__ is used
+        # for hashing the docs).
         all_user_set = set(User.objects.all())
         self.assertTrue(u1 in all_user_set)
+        self.assertTrue(u4 not in all_user_set)
+        self.assertTrue(b1 not in all_user_list)
+        self.assertTrue(b2 not in all_user_list)
+
+        # Make sure duplicate docs aren't accepted in the set
+        self.assertEqual(len(all_user_set), 3)
+        all_user_set.add(u1)
+        all_user_set.add(u2)
+        all_user_set.add(u3)
+        self.assertEqual(len(all_user_set), 3)
 
     def test_picklable(self):
         pickle_doc = PickleTest(number=1, string="One", lists=['1', '2'])

tests/fields/fields.py

@@ -5,8 +5,11 @@ import uuid
 import math
 import itertools
 import re
+import pymongo
+import sys
 
 from nose.plugins.skip import SkipTest
+from collections import OrderedDict
 import six
 
 try:
@@ -25,9 +28,12 @@ except ImportError:
 from mongoengine import *
 from mongoengine.connection import get_db
 from mongoengine.base import (BaseDict, BaseField, EmbeddedDocumentList,
-                              _document_registry)
+                              _document_registry, TopLevelDocumentMetaclass)
 
-from tests.utils import MongoDBTestCase
+from tests.utils import MongoDBTestCase, MONGO_TEST_DB
+from mongoengine.python_support import IS_PYMONGO_3
+if IS_PYMONGO_3:
+    from bson import CodecOptions
 
 __all__ = ("FieldTest", "EmbeddedDocumentListFieldTestCase")
 
@@ -337,8 +343,6 @@ class FieldTest(MongoDBTestCase):
         class Link(Document):
             url = URLField()
 
-        Link.drop_collection()
-
         link = Link()
         link.url = 'google'
         self.assertRaises(ValidationError, link.validate)
@@ -351,8 +355,6 @@ class FieldTest(MongoDBTestCase):
         class Link(Document):
             url = URLField()
 
-        Link.drop_collection()
-
         link = Link()
         link.url = u'http://привет.com'
 
@@ -1173,6 +1175,14 @@ class FieldTest(MongoDBTestCase):
         post.reload()
         self.assertEqual(post.info, ['0', '1', '2', '3', 'a', '5'])
 
+        # __setitem__(index, value) with a negative index
+        reset_post()
+        post.info[-2] = 'a'
+        self.assertEqual(post.info, ['0', '1', '2', '3', 'a', '5'])
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, ['0', '1', '2', '3', 'a', '5'])
+
         # '__setitem__(slice(i, j), listB)'
         # aka 'listA[i:j] = listB'
         # aka 'setitem(listA, slice(i, j), listB)'
@@ -1183,6 +1193,16 @@ class FieldTest(MongoDBTestCase):
         post.reload()
         self.assertEqual(post.info, ['0', 'h', 'e', 'l', 'l', 'o', '3', '4', '5'])
 
+        # '__setitem__(slice(i, j), listB)' with negative i and j
+        reset_post()
+        post.info[-5:-3] = ['h', 'e', 'l', 'l', 'o']
+        self.assertEqual(post.info, ['0', 'h', 'e', 'l', 'l', 'o', '3', '4', '5'])
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, ['0', 'h', 'e', 'l', 'l', 'o', '3', '4', '5'])
+
+        # negative
+
         # 'append'
         reset_post()
         post.info.append('h')
@@ -1883,6 +1903,51 @@ class FieldTest(MongoDBTestCase):
         doc = self.db.test.find_one()
         self.assertEqual(doc['x']['i'], 2)
 
+    def test_double_embedded_db_field(self):
+        """Make sure multiple layers of embedded docs resolve db fields
+        properly and can be initialized using dicts.
+        """
+        class C(EmbeddedDocument):
+            txt = StringField()
+
+        class B(EmbeddedDocument):
+            c = EmbeddedDocumentField(C, db_field='fc')
+
+        class A(Document):
+            b = EmbeddedDocumentField(B, db_field='fb')
+
+        a = A(
+            b=B(
+                c=C(txt='hi')
+            )
+        )
+        a.validate()
+
+        a = A(b={'c': {'txt': 'hi'}})
+        a.validate()
+
+    def test_double_embedded_db_field_from_son(self):
+        """Make sure multiple layers of embedded docs resolve db fields
+        from SON properly.
+        """
+        class C(EmbeddedDocument):
+            txt = StringField()
+
+        class B(EmbeddedDocument):
+            c = EmbeddedDocumentField(C, db_field='fc')
+
+        class A(Document):
+            b = EmbeddedDocumentField(B, db_field='fb')
+
+        a = A._from_son(SON([
+            ('fb', SON([
+                ('fc', SON([
+                    ('txt', 'hi')
+                ]))
+            ]))
+        ]))
+        self.assertEqual(a.b.c.txt, 'hi')
+
     def test_embedded_document_validation(self):
         """Ensure that invalid embedded documents cannot be assigned to
         embedded document fields.
@@ -2025,6 +2090,12 @@ class FieldTest(MongoDBTestCase):
         post1.author = post2
         self.assertRaises(ValidationError, post1.validate)
 
+        # Ensure ObjectID's are accepted as references
+        user_object_id = user.pk
+        post3 = BlogPost(content="Chips and curry sauce taste good.")
+        post3.author = user_object_id
+        post3.save()
+
         # Make sure referencing a saved document of the right type works
         user.save()
         post1.author = user
@@ -2035,6 +2106,20 @@ class FieldTest(MongoDBTestCase):
         post1.author = post2
         self.assertRaises(ValidationError, post1.validate)
 
+    def test_objectid_reference_fields(self):
+        """Make sure storing Object ID references works."""
+        class Person(Document):
+            name = StringField()
+            parent = ReferenceField('self')
+
+        Person.drop_collection()
+
+        p1 = Person(name="John").save()
+        Person(name="Ross", parent=p1.pk).save()
+
+        p = Person.objects.get(name="Ross")
+        self.assertEqual(p.parent, p1)
+
     def test_dbref_reference_fields(self):
         """Make sure storing references as bson.dbref.DBRef works."""
         class Person(Document):
@@ -3368,23 +3453,99 @@ class FieldTest(MongoDBTestCase):
         class User(Document):
             email = EmailField()
 
-        user = User(email="ross@example.com")
-        self.assertTrue(user.validate() is None)
+        user = User(email='ross@example.com')
+        user.validate()
 
-        user = User(email="ross@example.co.uk")
-        self.assertTrue(user.validate() is None)
+        user = User(email='ross@example.co.uk')
+        user.validate()
 
-        user = User(email=("Kofq@rhom0e4klgauOhpbpNdogawnyIKvQS0wk2mjqrgGQ5S"
-                           "aJIazqqWkm7.net"))
-        self.assertTrue(user.validate() is None)
+        user = User(email=('Kofq@rhom0e4klgauOhpbpNdogawnyIKvQS0wk2mjqrgGQ5S'
+                           'aJIazqqWkm7.net'))
+        user.validate()
 
-        user = User(email="new-tld@example.technology")
-        self.assertTrue(user.validate() is None)
+        user = User(email='new-tld@example.technology')
+        user.validate()
 
+        user = User(email='ross@example.com.')
+        self.assertRaises(ValidationError, user.validate)
+
+        # unicode domain
+        user = User(email=u'user@пример.рф')
+        user.validate()
+
+        # invalid unicode domain
+        user = User(email=u'user@пример')
+        self.assertRaises(ValidationError, user.validate)
+
+        # invalid data type
+        user = User(email=123)
+        self.assertRaises(ValidationError, user.validate)
+
+    def test_email_field_unicode_user(self):
+        # Don't run this test on pypy3, which doesn't support unicode regex:
+        # https://bitbucket.org/pypy/pypy/issues/1821/regular-expression-doesnt-find-unicode
+        if sys.version_info[:2] == (3, 2):
+            raise SkipTest('unicode email addresses are not supported on PyPy 3')
+
+        class User(Document):
+            email = EmailField()
+
+        # unicode user shouldn't validate by default...
+        user = User(email=u'Dörte@Sörensen.example.com')
+        self.assertRaises(ValidationError, user.validate)
+
+        # ...but it should be fine with allow_utf8_user set to True
+        class User(Document):
+            email = EmailField(allow_utf8_user=True)
+
+        user = User(email=u'Dörte@Sörensen.example.com')
+        user.validate()
+
+    def test_email_field_domain_whitelist(self):
+        class User(Document):
+            email = EmailField()
+
+        # localhost domain shouldn't validate by default...
         user = User(email='me@localhost')
         self.assertRaises(ValidationError, user.validate)
 
-        user = User(email="ross@example.com.")
+        # ...but it should be fine if it's whitelisted
+        class User(Document):
+            email = EmailField(domain_whitelist=['localhost'])
+
+        user = User(email='me@localhost')
+        user.validate()
+
+    def test_email_field_ip_domain(self):
+        class User(Document):
+            email = EmailField()
+
+        valid_ipv4 = 'email@[127.0.0.1]'
+        valid_ipv6 = 'email@[2001:dB8::1]'
+        invalid_ip = 'email@[324.0.0.1]'
+
+        # IP address as a domain shouldn't validate by default...
+        user = User(email=valid_ipv4)
+        self.assertRaises(ValidationError, user.validate)
+
+        user = User(email=valid_ipv6)
+        self.assertRaises(ValidationError, user.validate)
+
+        user = User(email=invalid_ip)
+        self.assertRaises(ValidationError, user.validate)
+
+        # ...but it should be fine with allow_ip_domain set to True
+        class User(Document):
+            email = EmailField(allow_ip_domain=True)
+
+        user = User(email=valid_ipv4)
+        user.validate()
+
+        user = User(email=valid_ipv6)
+        user.validate()
+
+        # invalid IP should still fail validation
+        user = User(email=invalid_ip)
         self.assertRaises(ValidationError, user.validate)
 
     def test_email_field_honors_regex(self):
@@ -4027,6 +4188,67 @@ class EmbeddedDocumentListFieldTestCase(MongoDBTestCase):
         self.assertTrue(hasattr(CustomData.c_field, 'custom_data'))
         self.assertEqual(custom_data['a'], CustomData.c_field.custom_data['a'])
 
+    def test_dynamicfield_with_container_class(self):
+        """
+        Tests that object can be stored in order by DynamicField class
+        with container_class parameter.
+        """
+        raw_data = [('d', 1), ('c', 2), ('b', 3), ('a', 4)]
+
+        class Doc(Document):
+            ordered_data = DynamicField(container_class=OrderedDict)
+            unordered_data = DynamicField()
+
+        Doc.drop_collection()
+
+        doc = Doc(ordered_data=OrderedDict(raw_data), unordered_data=dict(raw_data)).save()
+
+        # checks that the data is in order
+        self.assertEqual(type(doc.ordered_data), OrderedDict)
+        self.assertEqual(type(doc.unordered_data), dict)
+        self.assertEqual(','.join(doc.ordered_data.keys()), 'd,c,b,a')
+
+        # checks that the data is stored to the database in order
+        pymongo_db = pymongo.MongoClient()[MONGO_TEST_DB]
+        if IS_PYMONGO_3:
+            codec_option = CodecOptions(document_class=OrderedDict)
+            db_doc = pymongo_db.doc.with_options(codec_options=codec_option).find_one()
+        else:
+            db_doc = pymongo_db.doc.find_one(as_class=OrderedDict)
+
+        self.assertEqual(','.join(doc.ordered_data.keys()), 'd,c,b,a')
+
+    def test_dynamicfield_with_wrong_container_class(self):
+        with self.assertRaises(ValidationError):
+            class DocWithInvalidField:
+                data = DynamicField(container_class=list)
+
+    def test_dynamicfield_with_wrong_container_class_and_reload_docuemnt(self):
+        # This is because 'codec_options' is supported on pymongo3 or later
+        if IS_PYMONGO_3:
+            class OrderedDocument(Document):
+                my_metaclass = TopLevelDocumentMetaclass
+                __metaclass__ = TopLevelDocumentMetaclass
+
+                @classmethod
+                def _get_collection(cls):
+                    collection = super(OrderedDocument, cls)._get_collection()
+                    opts = CodecOptions(document_class=OrderedDict)
+
+                    return collection.with_options(codec_options=opts)
+
+            raw_data = [('d', 1), ('c', 2), ('b', 3), ('a', 4)]
+
+            class Doc(OrderedDocument):
+                data = DynamicField(container_class=OrderedDict)
+
+            Doc.drop_collection()
+
+            doc = Doc(data=OrderedDict(raw_data)).save()
+            doc.reload()
+
+            self.assertEqual(type(doc.data), OrderedDict)
+            self.assertEqual(','.join(doc.data.keys()), 'd,c,b,a')
 
 class CachedReferenceFieldTest(MongoDBTestCase):
 

tests/queryset/queryset.py

@@ -4962,20 +4962,6 @@ class QuerySetTest(unittest.TestCase):
         for p in Person.objects():
             self.assertEqual(p.name, 'a')
 
-    def test_last_field_name_like_operator(self):
-        class EmbeddedItem(EmbeddedDocument):
-            type = StringField()
-
-        class Doc(Document):
-            item = EmbeddedDocumentField(EmbeddedItem)
-
-        Doc.drop_collection()
-
-        doc = Doc(item=EmbeddedItem(type="axe"))
-        doc.save()
-
-        self.assertEqual(1, Doc.objects(item__type__="axe").count())
-
     def test_len_during_iteration(self):
         """Tests that calling len on a queyset during iteration doesn't
         stop paging.

tests/test_dereference.py

@@ -2,10 +2,15 @@
 import unittest
 
 from bson import DBRef, ObjectId
+from collections import OrderedDict
 
 from mongoengine import *
 from mongoengine.connection import get_db
 from mongoengine.context_managers import query_counter
+from mongoengine.python_support import IS_PYMONGO_3
+from mongoengine.base import TopLevelDocumentMetaclass
+if IS_PYMONGO_3:
+    from bson import CodecOptions
 
 
 class FieldTest(unittest.TestCase):
@@ -1287,5 +1292,70 @@ class FieldTest(unittest.TestCase):
 
             self.assertEqual(q, 2)
 
+    def test_dynamic_field_dereference(self):
+        class Merchandise(Document):
+            name = StringField()
+            price = IntField()
+
+        class Store(Document):
+            merchandises = DynamicField()
+
+        Merchandise.drop_collection()
+        Store.drop_collection()
+
+        merchandises = {
+            '#1': Merchandise(name='foo', price=100).save(),
+            '#2': Merchandise(name='bar', price=120).save(),
+            '#3': Merchandise(name='baz', price=110).save(),
+        }
+        Store(merchandises=merchandises).save()
+
+        store = Store.objects().first()
+        for obj in store.merchandises.values():
+            self.assertFalse(isinstance(obj, Merchandise))
+
+        store.select_related()
+        for obj in store.merchandises.values():
+            self.assertTrue(isinstance(obj, Merchandise))
+
+    def test_dynamic_field_dereference_with_ordering_guarantee_on_pymongo3(self):
+        # This is because 'codec_options' is supported on pymongo3 or later
+        if IS_PYMONGO_3:
+            class OrderedDocument(Document):
+                my_metaclass = TopLevelDocumentMetaclass
+                __metaclass__ = TopLevelDocumentMetaclass
+
+                @classmethod
+                def _get_collection(cls):
+                    collection = super(OrderedDocument, cls)._get_collection()
+                    opts = CodecOptions(document_class=OrderedDict)
+
+                    return collection.with_options(codec_options=opts)
+
+            class Merchandise(Document):
+                name = StringField()
+                price = IntField()
+
+            class Store(OrderedDocument):
+                merchandises = DynamicField(container_class=OrderedDict)
+
+            Merchandise.drop_collection()
+            Store.drop_collection()
+
+            merchandises = OrderedDict()
+            merchandises['#1'] = Merchandise(name='foo', price=100).save()
+            merchandises['#2'] = Merchandise(name='bar', price=120).save()
+            merchandises['#3'] = Merchandise(name='baz', price=110).save()
+
+            Store(merchandises=merchandises).save()
+
+            store = Store.objects().first()
+
+            store.select_related()
+
+            # confirms that the load data order is same with the one at storing
+            self.assertTrue(type(store.merchandises), OrderedDict)
+            self.assertEqual(','.join(store.merchandises.keys()), '#1,#2,#3')
+
 if __name__ == '__main__':
     unittest.main()