Merge branch 'master' of github.com:MongoEngine/mongoengine into py2py3_improve_compat

.travis.yml

@@ -3,7 +3,7 @@
 # with a very large number of jobs, hence we only test a subset of all the
 # combinations:
 # * MongoDB v3.4 & the latest PyMongo v3.x is currently the "main" setup,
-#   tested against Python v2.7, v3.5, v3.6, and PyPy.
+#   tested against Python v2.7, v3.5, v3.6, v3.7, v3.8, PyPy and PyPy3.
 # * Besides that, we test the lowest actively supported Python/MongoDB/PyMongo
 #   combination: MongoDB v3.4, PyMongo v3.4, Python v2.7.
 # * MongoDB v3.6 is tested against Python v3.6, and PyMongo v3.6, v3.7, v3.8.
@@ -23,6 +23,7 @@ python:
 - 3.5
 - 3.6
 - 3.7
+- 3.8
 - pypy
 - pypy3
 
@@ -34,14 +35,16 @@ env:
   global:
     - MONGODB_3_4=3.4.17
     - MONGODB_3_6=3.6.12
-    - PYMONGO_3_9=3.9
+    - MONGODB_4_0=4.0.13
-    - PYMONGO_3_6=3.6
+
     - PYMONGO_3_4=3.4
+    - PYMONGO_3_6=3.6
+    - PYMONGO_3_9=3.9
+    - PYMONGO_3_10=3.10
   matrix:
-    - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_9}
+    - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_10}
 
 matrix:
-
   # Finish the build as soon as one job fails
   fast_finish: true
 
@@ -52,7 +55,10 @@ matrix:
     env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_6}
   - python: 3.7
     env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_9}
-
+  - python: 3.7
+    env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_10}
+  - python: 3.8
+    env: MONGODB=${MONGODB_4_0} PYMONGO=${PYMONGO_3_10}
 
 install:
   # Install Mongo
@@ -109,5 +115,5 @@ deploy:
   on:
     tags: true
     repo: MongoEngine/mongoengine
-    condition: ($PYMONGO = ${PYMONGO_3_6}) && ($MONGODB = ${MONGODB_3_4})
+    condition: ($PYMONGO = ${PYMONGO_3_10}) && ($MONGODB = ${MONGODB_3_4})
     python: 2.7

AUTHORS

@@ -253,3 +253,6 @@ that much better:
  * Gaurav Dadhania (https://github.com/GVRV)
  * Yurii Andrieiev (https://github.com/yandrieiev)
  * Filip Kucharczyk (https://github.com/Pacu2)
+ * Eric Timmons (https://github.com/daewok)
+ * Matthew Simpson (https://github.com/mcsimps2)
+ * Leonardo Domingues (https://github.com/leodmgs)

README.rst

@@ -26,10 +26,10 @@ an `API reference <https://mongoengine-odm.readthedocs.io/apireference.html>`_.
 
 Supported MongoDB Versions
 ==========================
-MongoEngine is currently tested against MongoDB v3.4 and v3.6. Future versions
+MongoEngine is currently tested against MongoDB v3.4, v3.6 and v4.0. Future versions
 should be supported as well, but aren't actively tested at the moment. Make
 sure to open an issue or submit a pull request if you experience any problems
-with MongoDB version > 3.6.
+with MongoDB version > 4.0.
 
 Installation
 ============
@@ -91,12 +91,11 @@ Some simple examples of what MongoEngine code looks like:
 
     # Iterate over all posts using the BlogPost superclass
     >>> for post in BlogPost.objects:
-    ...     print '===', post.title, '==='
+    ...     print('===', post.title, '===')
     ...     if isinstance(post, TextPost):
-    ...         print post.content
+    ...         print(post.content)
     ...     elif isinstance(post, LinkPost):
-    ...         print 'Link:', post.url
+    ...         print('Link:', post.url)
-    ...     print
     ...
 
     # Count all blog posts and its subtypes

docs/changelog.rst

@@ -6,9 +6,18 @@ Changelog
 Development
 ===========
 - (Fill this out as you fix issues and develop your features).
-- Codebase is now compatible with both Python2 and Python3 (no more relying on 2to3 during installation) #2087
+- ATTENTION: Drop support for Python2
-- Documentation improvements:
+- Add Mongo 4.0 to Travis
-    - Documented how `pymongo.monitoring` can be used to log all queries issued by MongoEngine to the driver.
+- Fixed a bug causing inaccurate query results, while combining ``__raw__`` and regular filters for the same field #2264
+
+Changes in 0.19.1
+=================
+- Requires Pillow < 7.0.0 as it dropped Python2 support
+- DEPRECATION: The interface of ``QuerySet.aggregate`` method was changed, it no longer takes an unpacked list of
+    pipeline steps (*pipeline) but simply takes the pipeline list just like ``pymongo.Collection.aggregate`` does. #2079
+
+Changes in 0.19.0
+=================
 - BREAKING CHANGE: ``class_check`` and ``read_preference`` keyword arguments are no longer available when filtering a ``QuerySet``. #2112
     - Instead of ``Doc.objects(foo=bar, read_preference=...)`` use ``Doc.objects(foo=bar).read_preference(...)``.
     - Instead of ``Doc.objects(foo=bar, class_check=False)`` use ``Doc.objects(foo=bar).clear_cls_query(...)``.
@@ -18,17 +27,23 @@ Development
     - If you catch/use ``MongoEngineConnectionError`` in your code, you'll have to rename it.
 - BREAKING CHANGE: Positional arguments when instantiating a document are no longer supported. #2103
     - From now on keyword arguments (e.g. ``Doc(field_name=value)``) are required.
+- BREAKING CHANGE: A ``LazyReferenceField`` is now stored in the ``_data`` field of its parent as a ``DBRef``, ``Document``, or ``EmbeddedDocument`` (``ObjectId`` is no longer allowed). #2182
 - DEPRECATION: ``Q.empty`` & ``QNode.empty`` are marked as deprecated and will be removed in a next version of MongoEngine. #2210
     - Added ability to check if Q or QNode are empty by parsing them to bool.
     - Instead of ``Q(name="John").empty`` use ``not Q(name="John")``.
-- Improve error message related to InvalidDocumentError #2180
 - Fix updating/modifying/deleting/reloading a document that's sharded by a field with ``db_field`` specified. #2125
+- Only set no_cursor_timeout when requested (fixes an incompatibility with MongoDB 4.2) #2148
 - ``ListField`` now accepts an optional ``max_length`` parameter. #2110
-- Switch from nosetest to pytest as test runner #2114
+- Improve error message related to InvalidDocumentError #2180
-- The codebase is now formatted using ``black``. #2109
+- Added BulkWriteError to replace NotUniqueError which was misleading in bulk write insert #2152
-- In bulk write insert, the detailed error message would raise in exception.
 - Added ability to compare Q and Q operations #2204
 - Added ability to use a db alias on query_counter #2194
+- Added ability to specify collations for querysets with ``Doc.objects.collation`` #2024
+- Fix updates of a list field by negative index #2094
+- Switch from nosetest to pytest as test runner #2114
+- The codebase is now formatted using ``black``. #2109
+- Documentation improvements:
+    - Documented how `pymongo.monitoring` can be used to log all queries issued by MongoEngine to the driver.
 
 Changes in 0.18.2
 =================

docs/guide/defining-documents.rst

@@ -352,7 +352,7 @@ Its value can take any of the following constants:
   Deletion is denied if there still exist references to the object being
   deleted.
 :const:`mongoengine.NULLIFY`
-  Any object's fields still referring to the object being deleted are removed
+  Any object's fields still referring to the object being deleted are set to None
   (using MongoDB's "unset" operation), effectively nullifying the relationship.
 :const:`mongoengine.CASCADE`
   Any object containing fields that are referring to the object being deleted
@@ -744,7 +744,7 @@ Document inheritance
 
 To create a specialised type of a :class:`~mongoengine.Document` you have
 defined, you may subclass it and add any extra fields or methods you may need.
-As this is new class is not a direct subclass of
+As this new class is not a direct subclass of
 :class:`~mongoengine.Document`, it will not be stored in its own collection; it
 will use the same collection as its superclass uses. This allows for more
 convenient and efficient retrieval of related documents -- all you need do is
@@ -767,6 +767,27 @@ document.::
           Setting :attr:`allow_inheritance` to True should also be used in
           :class:`~mongoengine.EmbeddedDocument` class in case you need to subclass it
 
+When it comes to querying using :attr:`.objects()`, querying `Page.objects()` will query
+both `Page` and `DatedPage` whereas querying `DatedPage` will only query the `DatedPage` documents.
+Behind the scenes, MongoEngine deals with inheritance by adding a :attr:`_cls` attribute that contains
+the class name in every documents. When a document is loaded, MongoEngine checks
+it's :attr:`_cls` attribute and use that class to construct the instance.::
+
+    Page(title='a funky title').save()
+    DatedPage(title='another title', date=datetime.utcnow()).save()
+
+    print(Page.objects().count())         # 2
+    print(DatedPage.objects().count())    # 1
+
+    # print documents in their native form
+    # we remove 'id' to avoid polluting the output with unnecessary detail
+    qs = Page.objects.exclude('id').as_pymongo()
+    print(list(qs))
+    # [
+    #   {'_cls': u 'Page', 'title': 'a funky title'},
+    #   {'_cls': u 'Page.DatedPage', 'title': u 'another title', 'date': datetime.datetime(2019, 12, 13, 20, 16, 59, 993000)}
+    # ]
+
 Working with existing data
 --------------------------
 As MongoEngine no longer defaults to needing :attr:`_cls`, you can quickly and

docs/guide/gridfs.rst

@@ -10,8 +10,9 @@ Writing
 GridFS support comes in the form of the :class:`~mongoengine.fields.FileField` field
 object. This field acts as a file-like object and provides a couple of
 different ways of inserting and retrieving data. Arbitrary metadata such as
-content type can also be stored alongside the files. In the following example,
+content type can also be stored alongside the files. The object returned when accessing a
-a document is created to store details about animals, including a photo::
+FileField is a proxy to `Pymongo's GridFS <https://api.mongodb.com/python/current/examples/gridfs.html#gridfs-example>`_
+In the following example, a document is created to store details about animals, including a photo::
 
     class Animal(Document):
         genus = StringField()
@@ -20,8 +21,8 @@ a document is created to store details about animals, including a photo::
 
     marmot = Animal(genus='Marmota', family='Sciuridae')
 
-    marmot_photo = open('marmot.jpg', 'rb')
+    with open('marmot.jpg', 'rb') as fd:
-    marmot.photo.put(marmot_photo, content_type = 'image/jpeg')
+        marmot.photo.put(fd, content_type = 'image/jpeg')
     marmot.save()
 
 Retrieval
@@ -34,6 +35,20 @@ field. The file can also be retrieved just as easily::
     photo = marmot.photo.read()
     content_type = marmot.photo.content_type
 
+.. note:: If you need to read() the content of a file multiple times, you'll need to "rewind"
+    the file-like object using `seek`::
+
+        marmot = Animal.objects(genus='Marmota').first()
+        content1 = marmot.photo.read()
+        assert content1 != ""
+
+        content2 = marmot.photo.read()    # will be empty
+        assert content2 == ""
+
+        marmot.photo.seek(0)              # rewind the file by setting the current position of the cursor in the file to 0
+        content3 = marmot.photo.read()
+        assert content3 == content1
+
 Streaming
 ---------
 

docs/guide/mongomock.rst

@@ -2,10 +2,10 @@
 Use mongomock for testing
 ==============================
 
-`mongomock <https://github.com/vmalloc/mongomock/>`_ is a package to do just 
+`mongomock <https://github.com/vmalloc/mongomock/>`_ is a package to do just
 what the name implies, mocking a mongo database.
 
-To use with mongoengine, simply specify mongomock when connecting with 
+To use with mongoengine, simply specify mongomock when connecting with
 mongoengine:
 
 .. code-block:: python
@@ -21,7 +21,7 @@ or with an alias:
     conn = get_connection('testdb')
 
 Example of test file:
---------
+---------------------
 .. code-block:: python
 
     import unittest
@@ -45,4 +45,4 @@ Example of test file:
             pers.save()
 
             fresh_pers = Person.objects().first()
-            self.assertEqual(fresh_pers.name, 'John')
+            assert fresh_pers.name ==  'John'

docs/guide/querying.rst

@@ -222,6 +222,18 @@ keyword argument::
 
 .. versionadded:: 0.4
 
+Sorting/Ordering results
+========================
+It is possible to order the results by 1 or more keys using :meth:`~mongoengine.queryset.QuerySet.order_by`.
+The order may be specified by prepending each of the keys by "+" or "-". Ascending order is assumed if there's no prefix.::
+
+    # Order by ascending date
+    blogs = BlogPost.objects().order_by('date')    # equivalent to .order_by('+date')
+
+    # Order by ascending date first, then descending title
+    blogs = BlogPost.objects().order_by('+date', '-title')
+
+
 Limiting and skipping results
 =============================
 Just as with traditional ORMs, you may limit the number of results returned or
@@ -388,7 +400,7 @@ would be generating "tag-clouds"::
 
 MongoDB aggregation API
 -----------------------
-If you need to run aggregation pipelines, MongoEngine provides an entry point `Pymongo's aggregation framework <https://api.mongodb.com/python/current/examples/aggregation.html#aggregation-framework>`_
+If you need to run aggregation pipelines, MongoEngine provides an entry point to `Pymongo's aggregation framework <https://api.mongodb.com/python/current/examples/aggregation.html#aggregation-framework>`_
 through :meth:`~mongoengine.queryset.QuerySet.aggregate`. Check out Pymongo's documentation for the syntax and pipeline.
 An example of its use would be::
 
@@ -402,7 +414,7 @@ An example of its use would be::
             {"$sort" : {"name" : -1}},
             {"$project": {"_id": 0, "name": {"$toUpper": "$name"}}}
             ]
-        data = Person.objects().aggregate(*pipeline)
+        data = Person.objects().aggregate(pipeline)
         assert data == [{'name': 'BOB'}, {'name': 'JOHN'}]
 
 Query efficiency and performance
@@ -585,7 +597,8 @@ cannot use the `$` syntax in keyword arguments it has been mapped to `S`::
     ['database', 'mongodb']
 
 From MongoDB version 2.6, push operator supports $position value which allows
-to push values with index.
+to push values with index::
+
     >>> post = BlogPost(title="Test", tags=["mongo"])
     >>> post.save()
     >>> post.update(push__tags__0=["database", "code"])

docs/upgrade.rst

@@ -52,7 +52,7 @@ rename its occurrences.
 This release includes a major rehaul of MongoEngine's code quality and
 introduces a few breaking changes. It also touches many different parts of
 the package and although all the changes have been tested and scrutinized,
-you're encouraged to thorougly test the upgrade.
+you're encouraged to thoroughly test the upgrade.
 
 First breaking change involves renaming `ConnectionError` to `MongoEngineConnectionError`.
 If you import or catch this exception, you'll need to rename it in your code.

mongoengine/__init__.py

@@ -28,7 +28,7 @@ __all__ = (
 )
 
 
-VERSION = (0, 18, 2)
+VERSION = (0, 19, 1)
 
 
 def get_version():

mongoengine/base/datastructures.py

@@ -121,6 +121,9 @@ class BaseList(list):
         super(BaseList, self).__init__(list_items)
 
     def __getitem__(self, key):
+        # change index to positive value because MongoDB does not support negative one
+        if isinstance(key, int) and key < 0:
+            key = len(self) + key
         value = super(BaseList, self).__getitem__(key)
 
         if isinstance(key, slice):

mongoengine/document.py

@@ -330,7 +330,7 @@ class Document(six.with_metaclass(TopLevelDocumentMetaclass, BaseDocument)):
     ):
         """Save the :class:`~mongoengine.Document` to the database. If the
         document already exists, it will be updated, otherwise it will be
-        created.
+        created. Returns the saved object instance.
 
         :param force_insert: only try to create a new document, don't allow
             updates of existing documents.

mongoengine/fields.py

@@ -38,6 +38,7 @@ from mongoengine.common import _import_class
 from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db
 from mongoengine.document import Document, EmbeddedDocument
 from mongoengine.errors import DoesNotExist, InvalidQueryError, ValidationError
+from mongoengine.mongodb_support import MONGODB_36, get_mongodb_version
 from mongoengine.python_support import StringIO
 from mongoengine.queryset import DO_NOTHING
 from mongoengine.queryset.base import BaseQuerySet
@@ -1043,6 +1044,15 @@ def key_has_dot_or_dollar(d):
             return True
 
 
+def key_starts_with_dollar(d):
+    """Helper function to recursively determine if any key in a
+    dictionary starts with a dollar
+    """
+    for k, v in d.items():
+        if (k.startswith("$")) or (isinstance(v, dict) and key_starts_with_dollar(v)):
+            return True
+
+
 class DictField(ComplexBaseField):
     """A dictionary field that wraps a standard Python dictionary. This is
     similar to an embedded document, but the structure is not defined.
@@ -1069,11 +1079,18 @@ class DictField(ComplexBaseField):
         if key_not_string(value):
             msg = "Invalid dictionary key - documents must have only string keys"
             self.error(msg)
-        if key_has_dot_or_dollar(value):
+
+        curr_mongo_ver = get_mongodb_version()
+
+        if curr_mongo_ver < MONGODB_36 and key_has_dot_or_dollar(value):
             self.error(
                 'Invalid dictionary key name - keys may not contain "."'
                 ' or startswith "$" characters'
             )
+        elif curr_mongo_ver >= MONGODB_36 and key_starts_with_dollar(value):
+            self.error(
+                'Invalid dictionary key name - keys may not startswith "$" characters'
+            )
         super(DictField, self).validate(value)
 
     def lookup_member(self, member_name):
@@ -2494,6 +2511,13 @@ class LazyReferenceField(BaseField):
         else:
             return pk
 
+    def to_python(self, value):
+        """Convert a MongoDB-compatible type to a Python type."""
+        if not isinstance(value, (DBRef, Document, EmbeddedDocument)):
+            collection = self.document_type._get_collection_name()
+            value = DBRef(collection, self.document_type.id.to_python(value))
+        return value
+
     def validate(self, value):
         if isinstance(value, LazyReference):
             if value.collection != self.document_type._get_collection_name():

mongoengine/queryset/base.py

@@ -302,7 +302,7 @@ class BaseQuerySet(object):
                 ``insert(..., {w: 2, fsync: True})`` will wait until at least
                 two servers have recorded the write and will force an fsync on
                 each server being written to.
-        :parm signal_kwargs: (optional) kwargs dictionary to be passed to
+        :param signal_kwargs: (optional) kwargs dictionary to be passed to
             the signal calls.
 
         By default returns document instances, set ``load_bulk`` to False to
@@ -1255,16 +1255,27 @@ class BaseQuerySet(object):
             for data in son_data
         ]
 
-    def aggregate(self, *pipeline, **kwargs):
+    def aggregate(self, pipeline, *suppl_pipeline, **kwargs):
-        """
+        """Perform a aggregate function based in your queryset params
-        Perform a aggregate function based in your queryset params
+
         :param pipeline: list of aggregation commands,\
             see: http://docs.mongodb.org/manual/core/aggregation-pipeline/
-
+        :param suppl_pipeline: unpacked list of pipeline (added to support deprecation of the old interface)
+            parameter will be removed shortly
+        :param kwargs: (optional) kwargs dictionary to be passed to pymongo's aggregate call
+            See https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.aggregate
         .. versionadded:: 0.9
         """
-        initial_pipeline = []
+        using_deprecated_interface = isinstance(pipeline, dict) or bool(suppl_pipeline)
+        user_pipeline = [pipeline] if isinstance(pipeline, dict) else list(pipeline)
 
+        if using_deprecated_interface:
+            msg = "Calling .aggregate() with un unpacked list (*pipeline) is deprecated, it will soon change and will expect a list (similar to pymongo.Collection.aggregate interface), see documentation"
+            warnings.warn(msg, DeprecationWarning)
+
+        user_pipeline += suppl_pipeline
+
+        initial_pipeline = []
         if self._query:
             initial_pipeline.append({"$match": self._query})
 
@@ -1281,14 +1292,14 @@ class BaseQuerySet(object):
         if self._skip is not None:
             initial_pipeline.append({"$skip": self._skip})
 
-        pipeline = initial_pipeline + list(pipeline)
+        final_pipeline = initial_pipeline + user_pipeline
 
+        collection = self._collection
         if self._read_preference is not None:
-            return self._collection.with_options(
+            collection = self._collection.with_options(
                 read_preference=self._read_preference
-            ).aggregate(pipeline, cursor={}, **kwargs)
+            )
-
+        return collection.aggregate(final_pipeline, cursor={}, **kwargs)
-        return self._collection.aggregate(pipeline, cursor={}, **kwargs)
 
     # JS functionality
     def map_reduce(

mongoengine/queryset/transform.py

@@ -169,9 +169,9 @@ def query(_doc_cls=None, **kwargs):
 
         key = ".".join(parts)
 
-        if op is None or key not in mongo_query:
+        if key not in mongo_query:
             mongo_query[key] = value
-        elif key in mongo_query:
+        else:
             if isinstance(mongo_query[key], dict) and isinstance(value, dict):
                 mongo_query[key].update(value)
                 # $max/minDistance needs to come last - convert to SON

setup.py

@@ -108,6 +108,10 @@ CLASSIFIERS = [
     "Topic :: Software Development :: Libraries :: Python Modules",
 ]
 
+PYTHON_VERSION = sys.version_info[0]
+PY3 = PYTHON_VERSION == 3
+PY2 = PYTHON_VERSION == 2
+
 extra_opts = {
     "packages": find_packages(exclude=["tests", "tests.*"]),
     "tests_require": [
@@ -115,11 +119,12 @@ extra_opts = {
         "pytest-cov",
         "coverage<5.0",  # recent coverage switched to sqlite format for the .coverage file which isn't handled properly by coveralls
         "blinker",
-        "Pillow>=2.0.0",
+        "Pillow>=2.0.0, <7.0.0",  # 7.0.0 dropped Python2 support
+        "zipp<2.0.0",  # (dependency of pytest) dropped python2 support
     ],
 }
 
-if sys.version_info[0] == 3:
+if PY3:
     if "test" in sys.argv:
         extra_opts["packages"] = find_packages()
         extra_opts["package_data"] = {
@@ -143,7 +148,7 @@ setup(
     long_description=LONG_DESCRIPTION,
     platforms=["any"],
     classifiers=CLASSIFIERS,
-    install_requires=["pymongo>=3.4", "six", "future"],
+    install_requires=["pymongo>=3.4, <4.0", "six>=1.10.0", "future"],
     cmdclass={"test": PyTest},
     **extra_opts
 )

tests/document/test_instance.py

@@ -41,7 +41,7 @@ from tests.utils import MongoDBTestCase, get_as_pymongo
 TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), "../fields/mongoengine.png")
 
 
-class TestInstance(MongoDBTestCase):
+class TestDocumentInstance(MongoDBTestCase):
     def setUp(self):
         class Job(EmbeddedDocument):
             name = StringField()
@@ -3319,6 +3319,39 @@ class TestInstance(MongoDBTestCase):
         f1.ref  # Dereferences lazily
         assert f1 == f2
 
+    def test_embedded_document_equality_with_lazy_ref(self):
+        class Job(EmbeddedDocument):
+            boss = LazyReferenceField("Person")
+            boss_dbref = LazyReferenceField("Person", dbref=True)
+
+        class Person(Document):
+            job = EmbeddedDocumentField(Job)
+
+        Person.drop_collection()
+
+        boss = Person()
+        worker = Person(job=Job(boss=boss, boss_dbref=boss))
+        boss.save()
+        worker.save()
+
+        worker1 = Person.objects.get(id=worker.id)
+
+        # worker1.job should be equal to the job used originally to create the
+        # document.
+        assert worker1.job == worker.job
+
+        # worker1.job should be equal to a newly created Job EmbeddedDocument
+        # using either the Boss object or his ID.
+        assert worker1.job == Job(boss=boss, boss_dbref=boss)
+        assert worker1.job == Job(boss=boss.id, boss_dbref=boss.id)
+
+        # The above equalities should also hold after worker1.job.boss has been
+        # fetch()ed.
+        worker1.job.boss.fetch()
+        assert worker1.job == worker.job
+        assert worker1.job == Job(boss=boss, boss_dbref=boss)
+        assert worker1.job == Job(boss=boss.id, boss_dbref=boss.id)
+
     def test_dbref_equality(self):
         class Test2(Document):
             name = StringField()
@@ -3584,6 +3617,51 @@ class TestInstance(MongoDBTestCase):
             assert b._instance == a
         assert idx == 2
 
+    def test_updating_listfield_manipulate_list(self):
+        class Company(Document):
+            name = StringField()
+            employees = ListField(field=DictField())
+
+        Company.drop_collection()
+
+        comp = Company(name="BigBank", employees=[{"name": "John"}])
+        comp.save()
+        comp.employees.append({"name": "Bill"})
+        comp.save()
+
+        stored_comp = get_as_pymongo(comp)
+        self.assertEqual(
+            stored_comp,
+            {
+                "_id": comp.id,
+                "employees": [{"name": "John"}, {"name": "Bill"}],
+                "name": "BigBank",
+            },
+        )
+
+        comp = comp.reload()
+        comp.employees[0]["color"] = "red"
+        comp.employees[-1]["color"] = "blue"
+        comp.employees[-1].update({"size": "xl"})
+        comp.save()
+
+        assert len(comp.employees) == 2
+        assert comp.employees[0] == {"name": "John", "color": "red"}
+        assert comp.employees[1] == {"name": "Bill", "size": "xl", "color": "blue"}
+
+        stored_comp = get_as_pymongo(comp)
+        self.assertEqual(
+            stored_comp,
+            {
+                "_id": comp.id,
+                "employees": [
+                    {"name": "John", "color": "red"},
+                    {"size": "xl", "color": "blue", "name": "Bill"},
+                ],
+                "name": "BigBank",
+            },
+        )
+
     def test_falsey_pk(self):
         """Ensure that we can create and update a document with Falsey PK."""
 
@@ -3660,13 +3738,13 @@ class TestInstance(MongoDBTestCase):
         value = u"I_should_be_a_dict"
         coll.insert_one({"light_saber": value})
 
-        with self.assertRaises(InvalidDocumentError) as cm:
+        with pytest.raises(InvalidDocumentError) as exc_info:
             list(Jedi.objects)
 
-        self.assertEqual(
+        assert str(
-            str(cm.exception),
+            exc_info.value
-            "Invalid data to create a `Jedi` instance.\nField 'light_saber' - The source SON object needs to be of type 'dict' but a '%s' was found"
+        ) == "Invalid data to create a `Jedi` instance.\nField 'light_saber' - The source SON object needs to be of type 'dict' but a '%s' was found" % type(
-            % type(value),
+            value
         )
 
 

tests/fields/test_dict_field.py

@@ -3,6 +3,7 @@ import pytest
 
 from mongoengine import *
 from mongoengine.base import BaseDict
+from mongoengine.mongodb_support import MONGODB_36, get_mongodb_version
 
 from tests.utils import MongoDBTestCase, get_as_pymongo
 
@@ -43,11 +44,7 @@ class TestDictField(MongoDBTestCase):
         with pytest.raises(ValidationError):
             post.validate()
 
-        post.info = {"the.title": "test"}
+        post.info = {"$title.test": "test"}
-        with pytest.raises(ValidationError):
-            post.validate()
-
-        post.info = {"nested": {"the.title": "test"}}
         with pytest.raises(ValidationError):
             post.validate()
 
@@ -55,6 +52,20 @@ class TestDictField(MongoDBTestCase):
         with pytest.raises(ValidationError):
             post.validate()
 
+        post.info = {"nested": {"the.title": "test"}}
+        if get_mongodb_version() < MONGODB_36:
+            with pytest.raises(ValidationError):
+                post.validate()
+        else:
+            post.validate()
+
+        post.info = {"dollar_and_dot": {"te$st.test": "test"}}
+        if get_mongodb_version() < MONGODB_36:
+            with pytest.raises(ValidationError):
+                post.validate()
+        else:
+            post.validate()
+
         post.info = {"title": "test"}
         post.save()
 

tests/fields/test_file_field.py

@@ -151,7 +151,7 @@ class TestFileField(MongoDBTestCase):
         result = StreamFile.objects.first()
         assert streamfile == result
         assert result.the_file.read() == text + more_text
-        # self.assertEqual(result.the_file.content_type, content_type)
+        # assert result.the_file.content_type == content_type
         result.the_file.seek(0)
         assert result.the_file.tell() == 0
         assert result.the_file.read(len(text)) == text

tests/queryset/test_queryset.py

@@ -14,7 +14,7 @@ import six
 from six import iteritems
 
 from mongoengine import *
-from mongoengine.connection import get_connection, get_db
+from mongoengine.connection import get_db
 from mongoengine.context_managers import query_counter, switch_db
 from mongoengine.errors import InvalidQueryError
 from mongoengine.mongodb_support import MONGODB_36, get_mongodb_version
@@ -4658,21 +4658,6 @@ class TestQueryset(unittest.TestCase):
         )
         assert_read_pref(bars, ReadPreference.SECONDARY_PREFERRED)
 
-    def test_read_preference_aggregation_framework(self):
-        class Bar(Document):
-            txt = StringField()
-
-            meta = {"indexes": ["txt"]}
-
-        # Aggregates with read_preference
-        bars = Bar.objects.read_preference(
-            ReadPreference.SECONDARY_PREFERRED
-        ).aggregate()
-        assert (
-            bars._CommandCursor__collection.read_preference
-            == ReadPreference.SECONDARY_PREFERRED
-        )
-
     def test_json_simple(self):
         class Embedded(EmbeddedDocument):
             string = StringField()
@@ -5399,225 +5384,6 @@ class TestQueryset(unittest.TestCase):
             assert Person.objects.first().name == "A"
             assert Person.objects._has_data(), "Cursor has data and returned False"
 
-    def test_queryset_aggregation_framework(self):
-        class Person(Document):
-            name = StringField()
-            age = IntField()
-
-        Person.drop_collection()
-
-        p1 = Person(name="Isabella Luanna", age=16)
-        p2 = Person(name="Wilson Junior", age=21)
-        p3 = Person(name="Sandra Mara", age=37)
-        Person.objects.insert([p1, p2, p3])
-
-        data = Person.objects(age__lte=22).aggregate(
-            {"$project": {"name": {"$toUpper": "$name"}}}
-        )
-
-        assert list(data) == [
-            {"_id": p1.pk, "name": "ISABELLA LUANNA"},
-            {"_id": p2.pk, "name": "WILSON JUNIOR"},
-        ]
-
-        data = (
-            Person.objects(age__lte=22)
-            .order_by("-name")
-            .aggregate({"$project": {"name": {"$toUpper": "$name"}}})
-        )
-
-        assert list(data) == [
-            {"_id": p2.pk, "name": "WILSON JUNIOR"},
-            {"_id": p1.pk, "name": "ISABELLA LUANNA"},
-        ]
-
-        data = (
-            Person.objects(age__gte=17, age__lte=40)
-            .order_by("-age")
-            .aggregate(
-                {"$group": {"_id": None, "total": {"$sum": 1}, "avg": {"$avg": "$age"}}}
-            )
-        )
-        assert list(data) == [{"_id": None, "avg": 29, "total": 2}]
-
-        data = Person.objects().aggregate({"$match": {"name": "Isabella Luanna"}})
-        assert list(data) == [{u"_id": p1.pk, u"age": 16, u"name": u"Isabella Luanna"}]
-
-    def test_queryset_aggregation_with_skip(self):
-        class Person(Document):
-            name = StringField()
-            age = IntField()
-
-        Person.drop_collection()
-
-        p1 = Person(name="Isabella Luanna", age=16)
-        p2 = Person(name="Wilson Junior", age=21)
-        p3 = Person(name="Sandra Mara", age=37)
-        Person.objects.insert([p1, p2, p3])
-
-        data = Person.objects.skip(1).aggregate(
-            {"$project": {"name": {"$toUpper": "$name"}}}
-        )
-
-        assert list(data) == [
-            {"_id": p2.pk, "name": "WILSON JUNIOR"},
-            {"_id": p3.pk, "name": "SANDRA MARA"},
-        ]
-
-    def test_queryset_aggregation_with_limit(self):
-        class Person(Document):
-            name = StringField()
-            age = IntField()
-
-        Person.drop_collection()
-
-        p1 = Person(name="Isabella Luanna", age=16)
-        p2 = Person(name="Wilson Junior", age=21)
-        p3 = Person(name="Sandra Mara", age=37)
-        Person.objects.insert([p1, p2, p3])
-
-        data = Person.objects.limit(1).aggregate(
-            {"$project": {"name": {"$toUpper": "$name"}}}
-        )
-
-        assert list(data) == [{"_id": p1.pk, "name": "ISABELLA LUANNA"}]
-
-    def test_queryset_aggregation_with_sort(self):
-        class Person(Document):
-            name = StringField()
-            age = IntField()
-
-        Person.drop_collection()
-
-        p1 = Person(name="Isabella Luanna", age=16)
-        p2 = Person(name="Wilson Junior", age=21)
-        p3 = Person(name="Sandra Mara", age=37)
-        Person.objects.insert([p1, p2, p3])
-
-        data = Person.objects.order_by("name").aggregate(
-            {"$project": {"name": {"$toUpper": "$name"}}}
-        )
-
-        assert list(data) == [
-            {"_id": p1.pk, "name": "ISABELLA LUANNA"},
-            {"_id": p3.pk, "name": "SANDRA MARA"},
-            {"_id": p2.pk, "name": "WILSON JUNIOR"},
-        ]
-
-    def test_queryset_aggregation_with_skip_with_limit(self):
-        class Person(Document):
-            name = StringField()
-            age = IntField()
-
-        Person.drop_collection()
-
-        p1 = Person(name="Isabella Luanna", age=16)
-        p2 = Person(name="Wilson Junior", age=21)
-        p3 = Person(name="Sandra Mara", age=37)
-        Person.objects.insert([p1, p2, p3])
-
-        data = list(
-            Person.objects.skip(1)
-            .limit(1)
-            .aggregate({"$project": {"name": {"$toUpper": "$name"}}})
-        )
-
-        assert list(data) == [{"_id": p2.pk, "name": "WILSON JUNIOR"}]
-
-        # Make sure limit/skip chaining order has no impact
-        data2 = (
-            Person.objects.limit(1)
-            .skip(1)
-            .aggregate({"$project": {"name": {"$toUpper": "$name"}}})
-        )
-
-        assert data == list(data2)
-
-    def test_queryset_aggregation_with_sort_with_limit(self):
-        class Person(Document):
-            name = StringField()
-            age = IntField()
-
-        Person.drop_collection()
-
-        p1 = Person(name="Isabella Luanna", age=16)
-        p2 = Person(name="Wilson Junior", age=21)
-        p3 = Person(name="Sandra Mara", age=37)
-        Person.objects.insert([p1, p2, p3])
-
-        data = (
-            Person.objects.order_by("name")
-            .limit(2)
-            .aggregate({"$project": {"name": {"$toUpper": "$name"}}})
-        )
-
-        assert list(data) == [
-            {"_id": p1.pk, "name": "ISABELLA LUANNA"},
-            {"_id": p3.pk, "name": "SANDRA MARA"},
-        ]
-
-        # Verify adding limit/skip steps works as expected
-        data = (
-            Person.objects.order_by("name")
-            .limit(2)
-            .aggregate({"$project": {"name": {"$toUpper": "$name"}}}, {"$limit": 1})
-        )
-
-        assert list(data) == [{"_id": p1.pk, "name": "ISABELLA LUANNA"}]
-
-        data = (
-            Person.objects.order_by("name")
-            .limit(2)
-            .aggregate(
-                {"$project": {"name": {"$toUpper": "$name"}}},
-                {"$skip": 1},
-                {"$limit": 1},
-            )
-        )
-
-        assert list(data) == [{"_id": p3.pk, "name": "SANDRA MARA"}]
-
-    def test_queryset_aggregation_with_sort_with_skip(self):
-        class Person(Document):
-            name = StringField()
-            age = IntField()
-
-        Person.drop_collection()
-
-        p1 = Person(name="Isabella Luanna", age=16)
-        p2 = Person(name="Wilson Junior", age=21)
-        p3 = Person(name="Sandra Mara", age=37)
-        Person.objects.insert([p1, p2, p3])
-
-        data = (
-            Person.objects.order_by("name")
-            .skip(2)
-            .aggregate({"$project": {"name": {"$toUpper": "$name"}}})
-        )
-
-        assert list(data) == [{"_id": p2.pk, "name": "WILSON JUNIOR"}]
-
-    def test_queryset_aggregation_with_sort_with_skip_with_limit(self):
-        class Person(Document):
-            name = StringField()
-            age = IntField()
-
-        Person.drop_collection()
-
-        p1 = Person(name="Isabella Luanna", age=16)
-        p2 = Person(name="Wilson Junior", age=21)
-        p3 = Person(name="Sandra Mara", age=37)
-        Person.objects.insert([p1, p2, p3])
-
-        data = (
-            Person.objects.order_by("name")
-            .skip(1)
-            .limit(1)
-            .aggregate({"$project": {"name": {"$toUpper": "$name"}}})
-        )
-
-        assert list(data) == [{"_id": p3.pk, "name": "SANDRA MARA"}]
-
     def test_delete_count(self):
         [self.Person(name="User {0}".format(i), age=i * 10).save() for i in range(1, 4)]
         assert (

tests/queryset/test_queryset_aggregation.py

@@ -0,0 +1,255 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+import warnings
+
+from pymongo.read_preferences import ReadPreference
+
+from mongoengine import *
+from tests.utils import MongoDBTestCase
+
+
+class TestQuerysetAggregate(MongoDBTestCase):
+    def test_read_preference_aggregation_framework(self):
+        class Bar(Document):
+            txt = StringField()
+
+            meta = {"indexes": ["txt"]}
+
+        # Aggregates with read_preference
+        pipeline = []
+        bars = Bar.objects.read_preference(
+            ReadPreference.SECONDARY_PREFERRED
+        ).aggregate(pipeline)
+        assert (
+            bars._CommandCursor__collection.read_preference
+            == ReadPreference.SECONDARY_PREFERRED
+        )
+
+    def test_queryset_aggregation_framework(self):
+        class Person(Document):
+            name = StringField()
+            age = IntField()
+
+        Person.drop_collection()
+
+        p1 = Person(name="Isabella Luanna", age=16)
+        p2 = Person(name="Wilson Junior", age=21)
+        p3 = Person(name="Sandra Mara", age=37)
+        Person.objects.insert([p1, p2, p3])
+
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}]
+        data = Person.objects(age__lte=22).aggregate(pipeline)
+
+        assert list(data) == [
+            {"_id": p1.pk, "name": "ISABELLA LUANNA"},
+            {"_id": p2.pk, "name": "WILSON JUNIOR"},
+        ]
+
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}]
+        data = Person.objects(age__lte=22).order_by("-name").aggregate(pipeline)
+
+        assert list(data) == [
+            {"_id": p2.pk, "name": "WILSON JUNIOR"},
+            {"_id": p1.pk, "name": "ISABELLA LUANNA"},
+        ]
+
+        pipeline = [
+            {"$group": {"_id": None, "total": {"$sum": 1}, "avg": {"$avg": "$age"}}}
+        ]
+        data = (
+            Person.objects(age__gte=17, age__lte=40)
+            .order_by("-age")
+            .aggregate(pipeline)
+        )
+        assert list(data) == [{"_id": None, "avg": 29, "total": 2}]
+
+        pipeline = [{"$match": {"name": "Isabella Luanna"}}]
+        data = Person.objects().aggregate(pipeline)
+        assert list(data) == [{u"_id": p1.pk, u"age": 16, u"name": u"Isabella Luanna"}]
+
+    def test_queryset_aggregation_with_skip(self):
+        class Person(Document):
+            name = StringField()
+            age = IntField()
+
+        Person.drop_collection()
+
+        p1 = Person(name="Isabella Luanna", age=16)
+        p2 = Person(name="Wilson Junior", age=21)
+        p3 = Person(name="Sandra Mara", age=37)
+        Person.objects.insert([p1, p2, p3])
+
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}]
+        data = Person.objects.skip(1).aggregate(pipeline)
+
+        assert list(data) == [
+            {"_id": p2.pk, "name": "WILSON JUNIOR"},
+            {"_id": p3.pk, "name": "SANDRA MARA"},
+        ]
+
+    def test_queryset_aggregation_with_limit(self):
+        class Person(Document):
+            name = StringField()
+            age = IntField()
+
+        Person.drop_collection()
+
+        p1 = Person(name="Isabella Luanna", age=16)
+        p2 = Person(name="Wilson Junior", age=21)
+        p3 = Person(name="Sandra Mara", age=37)
+        Person.objects.insert([p1, p2, p3])
+
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}]
+        data = Person.objects.limit(1).aggregate(pipeline)
+
+        assert list(data) == [{"_id": p1.pk, "name": "ISABELLA LUANNA"}]
+
+    def test_queryset_aggregation_with_sort(self):
+        class Person(Document):
+            name = StringField()
+            age = IntField()
+
+        Person.drop_collection()
+
+        p1 = Person(name="Isabella Luanna", age=16)
+        p2 = Person(name="Wilson Junior", age=21)
+        p3 = Person(name="Sandra Mara", age=37)
+        Person.objects.insert([p1, p2, p3])
+
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}]
+        data = Person.objects.order_by("name").aggregate(pipeline)
+
+        assert list(data) == [
+            {"_id": p1.pk, "name": "ISABELLA LUANNA"},
+            {"_id": p3.pk, "name": "SANDRA MARA"},
+            {"_id": p2.pk, "name": "WILSON JUNIOR"},
+        ]
+
+    def test_queryset_aggregation_with_skip_with_limit(self):
+        class Person(Document):
+            name = StringField()
+            age = IntField()
+
+        Person.drop_collection()
+
+        p1 = Person(name="Isabella Luanna", age=16)
+        p2 = Person(name="Wilson Junior", age=21)
+        p3 = Person(name="Sandra Mara", age=37)
+        Person.objects.insert([p1, p2, p3])
+
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}]
+        data = list(Person.objects.skip(1).limit(1).aggregate(pipeline))
+
+        assert list(data) == [{"_id": p2.pk, "name": "WILSON JUNIOR"}]
+
+        # Make sure limit/skip chaining order has no impact
+        data2 = Person.objects.limit(1).skip(1).aggregate(pipeline)
+
+        assert data == list(data2)
+
+    def test_queryset_aggregation_with_sort_with_limit(self):
+        class Person(Document):
+            name = StringField()
+            age = IntField()
+
+        Person.drop_collection()
+
+        p1 = Person(name="Isabella Luanna", age=16)
+        p2 = Person(name="Wilson Junior", age=21)
+        p3 = Person(name="Sandra Mara", age=37)
+        Person.objects.insert([p1, p2, p3])
+
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}]
+        data = Person.objects.order_by("name").limit(2).aggregate(pipeline)
+
+        assert list(data) == [
+            {"_id": p1.pk, "name": "ISABELLA LUANNA"},
+            {"_id": p3.pk, "name": "SANDRA MARA"},
+        ]
+
+        # Verify adding limit/skip steps works as expected
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}, {"$limit": 1}]
+        data = Person.objects.order_by("name").limit(2).aggregate(pipeline)
+
+        assert list(data) == [{"_id": p1.pk, "name": "ISABELLA LUANNA"}]
+
+        pipeline = [
+            {"$project": {"name": {"$toUpper": "$name"}}},
+            {"$skip": 1},
+            {"$limit": 1},
+        ]
+        data = Person.objects.order_by("name").limit(2).aggregate(pipeline)
+
+        assert list(data) == [{"_id": p3.pk, "name": "SANDRA MARA"}]
+
+    def test_queryset_aggregation_with_sort_with_skip(self):
+        class Person(Document):
+            name = StringField()
+            age = IntField()
+
+        Person.drop_collection()
+
+        p1 = Person(name="Isabella Luanna", age=16)
+        p2 = Person(name="Wilson Junior", age=21)
+        p3 = Person(name="Sandra Mara", age=37)
+        Person.objects.insert([p1, p2, p3])
+
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}]
+        data = Person.objects.order_by("name").skip(2).aggregate(pipeline)
+
+        assert list(data) == [{"_id": p2.pk, "name": "WILSON JUNIOR"}]
+
+    def test_queryset_aggregation_with_sort_with_skip_with_limit(self):
+        class Person(Document):
+            name = StringField()
+            age = IntField()
+
+        Person.drop_collection()
+
+        p1 = Person(name="Isabella Luanna", age=16)
+        p2 = Person(name="Wilson Junior", age=21)
+        p3 = Person(name="Sandra Mara", age=37)
+        Person.objects.insert([p1, p2, p3])
+
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}]
+        data = Person.objects.order_by("name").skip(1).limit(1).aggregate(pipeline)
+
+        assert list(data) == [{"_id": p3.pk, "name": "SANDRA MARA"}]
+
+    def test_queryset_aggregation_deprecated_interface(self):
+        class Person(Document):
+            name = StringField()
+
+        Person.drop_collection()
+
+        p1 = Person(name="Isabella Luanna")
+        p2 = Person(name="Wilson Junior")
+        p3 = Person(name="Sandra Mara")
+        Person.objects.insert([p1, p2, p3])
+
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}]
+
+        # Make sure a warning is emitted
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", DeprecationWarning)
+            with self.assertRaises(DeprecationWarning):
+                Person.objects.order_by("name").limit(2).aggregate(*pipeline)
+
+        # Make sure old interface works as expected with a 1-step pipeline
+        data = Person.objects.order_by("name").limit(2).aggregate(*pipeline)
+
+        assert list(data) == [
+            {"_id": p1.pk, "name": "ISABELLA LUANNA"},
+            {"_id": p3.pk, "name": "SANDRA MARA"},
+        ]
+
+        # Make sure old interface works as expected with a 2-steps pipeline
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}, {"$limit": 1}]
+        data = Person.objects.order_by("name").limit(2).aggregate(*pipeline)
+
+        assert list(data) == [{"_id": p1.pk, "name": "ISABELLA LUANNA"}]
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/queryset/test_transform.py

@@ -24,6 +24,12 @@ class TestTransform(unittest.TestCase):
         }
         assert transform.query(friend__age__gte=30) == {"friend.age": {"$gte": 30}}
         assert transform.query(name__exists=True) == {"name": {"$exists": True}}
+        assert transform.query(name=["Mark"], __raw__={"name": {"$in": "Tom"}}) == {
+            "$and": [{"name": ["Mark"]}, {"name": {"$in": "Tom"}}]
+        }
+        assert transform.query(name__in=["Tom"], __raw__={"name": "Mark"}) == {
+            "$and": [{"name": {"$in": ["Tom"]}}, {"name": "Mark"}]
+        }
 
     def test_transform_update(self):
         class LisDoc(Document):

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = {py27,py35,pypy,pypy3}-{mg34,mg36}
+envlist = {py27,py35,pypy,pypy3}-{mg34,mg36,mg39,mg310}
 
 [testenv]
 commands =
@@ -7,6 +7,7 @@ commands =
 deps =
     mg34: pymongo>=3.4,<3.5
     mg36: pymongo>=3.6,<3.7
-    mg39: pymongo>=3.9,<4.0
+    mg39: pymongo>=3.9,<3.10
+    mg310: pymongo>=3.10,<3.11
 setenv =
     PYTHON_EGG_CACHE = {envdir}/python-eggs