Merge pull request #2447 from bagerard/_changelog_0_22_0

prepare 0.22.0

.landscape.yml

@@ -1,17 +0,0 @@
-pylint:
-    disable:
-        # We use this a lot (e.g. via document._meta)
-        - protected-access
-
-    options:
-        additional-builtins:
-            # add long as valid built-ins.
-            - long
-
-pyflakes:
-    disable:
-        # undefined variables are already covered by pylint (and exclude long)
-        - F821
-
-ignore-paths:
-    - benchmark.py

.pre-commit-config.yaml

@@ -1,12 +1,17 @@
 fail_fast: false
 repos:
-    - repo: https://github.com/ambv/black
-      rev: 19.10b0
-      hooks:
-        - id: black
-    - repo: https://gitlab.com/pycqa/flake8
-      rev: 3.8.0a2
-      hooks:
-        - id: flake8
-          additional_dependencies:
-            - flake8-import-order
+  - repo: https://github.com/ambv/black
+    rev: 20.8b1
+    hooks:
+      - id: black
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: 3.8.4
+    hooks:
+      - id: flake8
+        additional_dependencies:
+          - flake8-import-order
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v2.7.4
+    hooks:
+      - id: pyupgrade
+        args: [--py36-plus]

.readthedocs.yml

@@ -0,0 +1,20 @@
+# .readthedocs.yml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# Optionally set the version of Python and requirements required to build your docs
+python:
+  version: 3.7
+  install:
+    - requirements: docs/requirements.txt
+    # docs/conf.py is importing mongoengine
+    # so mongoengine needs to be installed as well
+    - method: setuptools
+      path: .

.travis.yml

@@ -16,40 +16,40 @@
 language: python
 dist: xenial
 python:
-- 3.5
-- 3.6
-- 3.7
-- 3.8
-- pypy3
+  - 3.6
+  - 3.7
+  - 3.8
+  - 3.9
+  - pypy3
 
 env:
   global:
-    - MONGODB_3_4=3.4.17
-    - MONGODB_3_6=3.6.12
+    - MONGODB_3_4=3.4.19
+    - MONGODB_3_6=3.6.13
     - MONGODB_4_0=4.0.13
 
     - PYMONGO_3_4=3.4
     - PYMONGO_3_6=3.6
     - PYMONGO_3_9=3.9
-    - PYMONGO_3_10=3.10
+    - PYMONGO_3_11=3.11
 
     - MAIN_PYTHON_VERSION=3.7
   matrix:
-    - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_10}
+    - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_11}
 
 matrix:
   # Finish the build as soon as one job fails
   fast_finish: true
 
   include:
-  - python: 3.7
-    env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_6}
-  - python: 3.7
-    env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_9}
-  - python: 3.7
-    env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_10}
-  - python: 3.8
-    env: MONGODB=${MONGODB_4_0} PYMONGO=${PYMONGO_3_10}
+    - python: 3.7
+      env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_6}
+    - python: 3.7
+      env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_9}
+    - python: 3.7
+      env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_11}
+    - python: 3.8
+      env: MONGODB=${MONGODB_4_0} PYMONGO=${PYMONGO_3_11}
 
 install:
   # Install Mongo
@@ -69,13 +69,13 @@ before_script:
   - ${PWD}/mongodb-linux-x86_64-${MONGODB}/bin/mongod --dbpath ${PWD}/mongodb-linux-x86_64-${MONGODB}/data --logpath ${PWD}/mongodb-linux-x86_64-${MONGODB}/mongodb.log --fork
   # Run pre-commit hooks (black, flake8, etc) on entire codebase
   - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then pre-commit run -a; else echo "pre-commit checks only runs on py37"; fi
-  - mongo --eval 'db.version();'    # Make sure mongo is awake
+  - mongo --eval 'db.version();' # Make sure mongo is awake
 
 script:
   - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -a "--cov=mongoengine"
 
 after_success:
-- - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then coveralls --verbose; else echo "coveralls only sent for py37"; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then coveralls --verbose; else echo "coveralls only sent for py37"; fi
 
 notifications:
   irc: irc.freenode.org#mongoengine
@@ -83,8 +83,8 @@ notifications:
 # Only run builds on the master branch and GitHub releases (tagged as vX.Y.Z)
 branches:
   only:
-  - master
-  - /^v.*$/
+    - master
+    - /^v.*$/
 
 # Whenever a new release is created via GitHub, publish it on PyPI.
 deploy:
@@ -103,5 +103,5 @@ deploy:
   on:
     tags: true
     repo: MongoEngine/mongoengine
-    condition: ($PYMONGO = ${PYMONGO_3_10}) && ($MONGODB = ${MONGODB_3_4})
+    condition: ($PYMONGO = ${PYMONGO_3_11}) && ($MONGODB = ${MONGODB_3_4})
     python: 3.7

AUTHORS

@@ -257,3 +257,5 @@ that much better:
  * Matthew Simpson (https://github.com/mcsimps2)
  * Leonardo Domingues (https://github.com/leodmgs)
  * Agustin Barto (https://github.com/abarto)
+ * Stankiewicz Mateusz (https://github.com/mas15)
+ * Felix Schultheiß (https://github.com/felix-smashdocs)

README.rst

@@ -12,9 +12,8 @@ MongoEngine
 .. image:: https://coveralls.io/repos/github/MongoEngine/mongoengine/badge.svg?branch=master
   :target: https://coveralls.io/github/MongoEngine/mongoengine?branch=master
 
-.. image:: https://landscape.io/github/MongoEngine/mongoengine/master/landscape.svg?style=flat
-  :target: https://landscape.io/github/MongoEngine/mongoengine/master
-  :alt: Code Health
+.. image:: https://img.shields.io/badge/code%20style-black-000000.svg
+  :target: https://github.com/ambv/black
 
 About
 =====

benchmarks/test_basic_doc_ops.py

@@ -45,7 +45,7 @@ def test_basic():
 
     print(
         "Doc setattr: %.3fus"
-        % (timeit(lambda: setattr(b, "name", "New name"), 10000) * 10 ** 6)
+        % (timeit(lambda: setattr(b, "name", "New name"), 10000) * 10 ** 6)  # noqa B010
     )
 
     print("Doc to mongo: %.3fus" % (timeit(b.to_mongo, 1000) * 10 ** 6))

docs/Makefile

@@ -33,7 +33,7 @@ clean:
 html:
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
-	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+	@echo "Build finished. Check $(BUILDDIR)/html/index.html"
 
 dirhtml:
 	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml

docs/changelog.rst

@@ -7,6 +7,29 @@ Development
 ===========
 - (Fill this out as you fix issues and develop your features).
 
+Changes in 0.22.0
+=================
+- Fix LazyReferenceField dereferencing in embedded documents #2426
+- Fix regarding the recent use of Cursor.__spec in .count() that was interfering with mongomock #2425
+- Drop support for Python 3.5 by introducing f-strings in the codebase
+
+Changes in 0.21.0
+=================
+- Bug fix in DynamicDocument which is not parsing known fields in constructor like Document do #2412
+- When using pymongo >= 3.7, make use of Collection.count_documents instead of Collection.count
+    and Cursor.count that got deprecated in pymongo >= 3.7.
+    This should have a negative impact on performance of count see Issue #2219
+- Fix a bug that made the queryset drop the read_preference after clone().
+- Remove Py3.5 from CI as it reached EOL and add Python 3.9
+- Fix some issues related with db_field/field conflict in constructor #2414
+- BREAKING CHANGE: Fix the behavior of Doc.objects.limit(0) which should return all documents (similar to mongodb) #2311
+- Bug fix in ListField when updating the first item, it was saving the whole list, instead of
+    just replacing the first item (as usually done when updating 1 item of the list) #2392
+- Add EnumField: ``mongoengine.fields.EnumField``
+- Refactoring - Remove useless code related to Document.__only_fields and Queryset.only_fields
+- Fix query transformation regarding special operators #2365
+- Bug Fix: Document.save() fails when shard_key is not _id #2154
+
 Changes in 0.20.0
 =================
 - ATTENTION: Drop support for Python2
@@ -28,7 +51,7 @@ Changes in 0.20.0
 
 Changes in 0.19.1
 =================
-- Requires Pillow < 7.0.0 as it dropped Python2 support
+- Tests require Pillow < 7.0.0 as it dropped Python2 support
 - DEPRECATION: The interface of ``QuerySet.aggregate`` method was changed, it no longer takes an unpacked list of
     pipeline steps (*pipeline) but simply takes the pipeline list just like ``pymongo.Collection.aggregate`` does. #2079
 
@@ -456,9 +479,6 @@ Changes in 0.8.3
 - Document.select_related() now respects ``db_alias`` (#377)
 - Reload uses shard_key if applicable (#384)
 - Dynamic fields are ordered based on creation and stored in _fields_ordered (#396)
-
-  **Potential breaking change:** http://docs.mongoengine.org/en/latest/upgrade.html#to-0-8-3
-
 - Fixed pickling dynamic documents ``_dynamic_fields`` (#387)
 - Fixed ListField setslice and delslice dirty tracking (#390)
 - Added Django 1.5 PY3 support (#392)

docs/conf.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 #
 # MongoEngine documentation build configuration file, created by
 # sphinx-quickstart on Sun Nov 22 18:14:13 2009.
@@ -42,8 +41,8 @@ source_suffix = ".rst"
 master_doc = "index"
 
 # General information about the project.
-project = u"MongoEngine"
-copyright = u"2009, MongoEngine Authors"
+project = "MongoEngine"
+copyright = "2009, MongoEngine Authors"  # noqa: A001
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

docs/guide/connecting.rst

@@ -31,6 +31,8 @@ the :attr:`host` to
 
     connect('project1', host='mongodb://localhost/database_name')
 
+.. note:: URI containing SRV records (e.g mongodb+srv://server.example.com/) can be used as well as the :attr:`host`
+
 .. note:: Database, username and password from URI string overrides
     corresponding parameters in :func:`~mongoengine.connect`: ::
 

docs/guide/defining-documents.rst

@@ -76,6 +76,7 @@ are as follows:
 * :class:`~mongoengine.fields.EmailField`
 * :class:`~mongoengine.fields.EmbeddedDocumentField`
 * :class:`~mongoengine.fields.EmbeddedDocumentListField`
+* :class:`~mongoengine.fields.EnumField`
 * :class:`~mongoengine.fields.FileField`
 * :class:`~mongoengine.fields.FloatField`
 * :class:`~mongoengine.fields.GenericEmbeddedDocumentField`
@@ -426,19 +427,6 @@ either a single field name, or a list or tuple of field names::
         first_name = StringField()
         last_name = StringField(unique_with='first_name')
 
-Skipping Document validation on save
-------------------------------------
-You can also skip the whole document validation process by setting
-``validate=False`` when calling the :meth:`~mongoengine.document.Document.save`
-method::
-
-    class Recipient(Document):
-        name = StringField()
-        email = EmailField()
-
-    recipient = Recipient(name='admin', email='root@localhost')
-    recipient.save()               # will raise a ValidationError while
-    recipient.save(validate=False) # won't
 
 Document collections
 ====================

docs/guide/document-instances.rst

@@ -41,35 +41,6 @@ already exist, then any changes will be updated atomically.  For example::
 .. seealso::
     :ref:`guide-atomic-updates`
 
-Pre save data validation and cleaning
--------------------------------------
-MongoEngine allows you to create custom cleaning rules for your documents when
-calling :meth:`~mongoengine.Document.save`.  By providing a custom
-:meth:`~mongoengine.Document.clean` method you can do any pre validation / data
-cleaning.
-
-This might be useful if you want to ensure a default value based on other
-document values for example::
-
-    class Essay(Document):
-        status = StringField(choices=('Published', 'Draft'), required=True)
-        pub_date = DateTimeField()
-
-        def clean(self):
-            """Ensures that only published essays have a `pub_date` and
-            automatically sets `pub_date` if essay is published and `pub_date`
-            is not set"""
-            if self.status == 'Draft' and self.pub_date is not None:
-                msg = 'Draft entries should not have a publication date.'
-                raise ValidationError(msg)
-            # Set the pub_date for published items if not set.
-            if self.status == 'Published' and self.pub_date is None:
-                self.pub_date = datetime.now()
-
-.. note::
-    Cleaning is only called if validation is turned on and when calling
-    :meth:`~mongoengine.Document.save`.
-
 Cascading Saves
 ---------------
 If your document contains :class:`~mongoengine.fields.ReferenceField` or

docs/guide/gridfs.rst

@@ -2,8 +2,6 @@
 GridFS
 ======
 
-.. versionadded:: 0.4
-
 Writing
 -------
 

docs/guide/index.rst

@@ -10,8 +10,10 @@ User Guide
    defining-documents
    document-instances
    querying
+   validation
    gridfs
    signals
    text-indexes
+   migration
    logging-monitoring
    mongomock

docs/guide/migration.rst

@@ -0,0 +1,267 @@
+===================
+Documents migration
+===================
+
+The structure of your documents and their associated mongoengine schemas are likely
+to change over the lifetime of an application. This section provides guidance and
+recommendations on how to deal with migrations.
+
+Due to the very flexible nature of mongodb, migrations of models aren't trivial and
+for people that know about `alembic` for `sqlalchemy`, there is unfortunately no equivalent
+library that will manage the migration in an automatic fashion for mongoengine.
+
+Example 1: Addition of a field
+==============================
+
+Let's start by taking a simple example of a model change and review the different option you
+have to deal with the migration.
+
+Let's assume we start with the following schema and save an instance:
+
+.. code-block:: python
+
+    class User(Document):
+        name = StringField()
+
+    User(name="John Doe").save()
+
+    # print the objects as they exist in mongodb
+    print(User.objects().as_pymongo())    # [{u'_id': ObjectId('5d06b9c3d7c1f18db3e7c874'), u'name': u'John Doe'}]
+
+On the next version of your application, let's now assume that a new field `enabled` gets added to the
+existing ``User`` model with a `default=True`. Thus you simply update the ``User`` class to the following:
+
+.. code-block:: python
+
+    class User(Document):
+        name = StringField(required=True)
+        enabled = BooleaField(default=True)
+
+Without applying any migration, we now reload an object from the database into the ``User`` class
+and checks its `enabled` attribute:
+
+.. code-block:: python
+
+    assert User.objects.count() == 1
+    user = User.objects().first()
+    assert user.enabled is True
+    assert User.objects(enabled=True).count() == 0    # uh?
+    assert User.objects(enabled=False).count() == 0   # uh?
+
+    # this is consistent with what we have in the database
+    # in fact, 'enabled' does not exist
+    print(User.objects().as_pymongo().first())    # {u'_id': ObjectId('5d06b9c3d7c1f18db3e7c874'), u'name': u'John'}
+    assert User.objects(enabled=None).count() == 1
+
+As you can see, even if the document wasn't updated, mongoengine applies the default value seamlessly when it
+loads the pymongo dict into a ``User`` instance. At first sight it looks like you don't need to migrate the
+existing documents when adding new fields but this actually leads to inconsistencies when it comes to querying.
+
+In fact, when querying, mongoengine isn't trying to account for the default value of the new field and so
+if you don't actually migrate the existing documents, you are taking a risk that querying/updating
+will be missing relevant record.
+
+When adding fields/modifying default values, you can use any of the following to do the migration
+as a standalone script:
+
+.. code-block:: python
+
+    # Use mongoengine to set a default value for a given field
+    User.objects().update(enabled=True)
+    # or use pymongo
+    user_coll = User._get_collection()
+    user_coll.update_many({}, {'$set': {'enabled': True}})
+
+
+Example 2: Inheritance change
+=============================
+
+Let's consider the following example:
+
+.. code-block:: python
+
+    class Human(Document):
+        name = StringField()
+        meta = {"allow_inheritance": True}
+
+    class Jedi(Human):
+        dark_side = BooleanField()
+        light_saber_color = StringField()
+
+    Jedi(name="Darth Vader", dark_side=True, light_saber_color="red").save()
+    Jedi(name="Obi Wan Kenobi", dark_side=False, light_saber_color="blue").save()
+
+    assert Human.objects.count() == 2
+    assert Jedi.objects.count() == 2
+
+    # Let's check how these documents got stored in mongodb
+    print(Jedi.objects.as_pymongo())
+    # [
+    #   {'_id': ObjectId('5fac4aaaf61d7fb06046e0f9'), '_cls': 'Human.Jedi', 'name': 'Darth Vader', 'dark_side': True, 'light_saber_color': 'red'},
+    #   {'_id': ObjectId('5fac4ac4f61d7fb06046e0fa'), '_cls': 'Human.Jedi', 'name': 'Obi Wan Kenobi', 'dark_side': False, 'light_saber_color': 'blue'}
+    # ]
+
+As you can observe, when you use inheritance, MongoEngine stores a field named '_cls' behind the scene to keep
+track of the Document class.
+
+Let's now take the scenario that you want to refactor the inheritance schema and:
+- Have the Jedi's with dark_side=True/False become GoodJedi's/DarkSith
+- get rid of the 'dark_side' field
+
+move to the following schemas:
+
+.. code-block:: python
+
+    # unchanged
+    class Human(Document):
+        name = StringField()
+        meta = {"allow_inheritance": True}
+
+    # attribute 'dark_side' removed
+    class GoodJedi(Human):
+        light_saber_color = StringField()
+
+    # new class
+    class BadSith(Human):
+        light_saber_color = StringField()
+
+MongoEngine doesn't know about the change or how to map them with the existing data
+so if you don't apply any migration, you will observe a strange behavior, as if the collection was suddenly
+empty.
+
+.. code-block:: python
+
+    # As a reminder, the documents that we inserted
+    # have the _cls field = 'Human.Jedi'
+
+    # Following has no match
+    # because the query that is used behind the scene is
+    # filtering on {'_cls': 'Human.GoodJedi'}
+    assert GoodJedi.objects().count() == 0
+
+    # Following has also no match
+    # because it is filtering on {'_cls': {'$in': ('Human', 'Human.GoodJedi', 'Human.BadSith')}}
+    # which has no match
+    assert Human.objects.count() == 0
+    assert Human.objects.first() is None
+
+    # If we bypass MongoEngine and make use of underlying driver (PyMongo)
+    # we can see that the documents are there
+    humans_coll = Human._get_collection()
+    assert humans_coll.count_documents({}) == 2
+    # print first document
+    print(humans_coll.find_one())
+    # {'_id': ObjectId('5fac4aaaf61d7fb06046e0f9'), '_cls': 'Human.Jedi', 'name': 'Darth Vader', 'dark_side': True, 'light_saber_color': 'red'}
+
+As you can see, first obvious problem is that we need to modify '_cls' values based on existing values of
+'dark_side' documents.
+
+.. code-block:: python
+
+    humans_coll = Human._get_collection()
+    old_class = 'Human.Jedi'
+    good_jedi_class = 'Human.GoodJedi'
+    bad_sith_class = 'Human.BadSith'
+    humans_coll.update_many({'_cls': old_class, 'dark_side': False}, {'$set': {'_cls': good_jedi_class}})
+    humans_coll.update_many({'_cls': old_class, 'dark_side': True}, {'$set': {'_cls': bad_sith_class}})
+
+Let's now check if querying improved in MongoEngine:
+
+.. code-block:: python
+
+    assert GoodJedi.objects().count() == 1  # Hoorah!
+    assert BadSith.objects().count() == 1   # Hoorah!
+    assert Human.objects.count() == 2       # Hoorah!
+
+    # let's now check that documents load correctly
+    jedi = GoodJedi.objects().first()
+    # raises FieldDoesNotExist: The fields "{'dark_side'}" do not exist on the document "Human.GoodJedi"
+
+In fact we only took care of renaming the _cls values but we havn't removed the 'dark_side' fields
+which does not exist anymore on the GoodJedi's and BadSith's models.
+Let's remove the field from the collections:
+
+.. code-block:: python
+
+    humans_coll = Human._get_collection()
+    humans_coll.update_many({}, {'$unset': {'dark_side': 1}})
+
+.. note:: We did this migration in 2 different steps for the sake of example but it could have been combined
+    with the migration of the _cls fields: ::
+
+        humans_coll.update_many(
+            {'_cls': old_class, 'dark_side': False},
+            {
+                '$set': {'_cls': good_jedi_class},
+                '$unset': {'dark_side': 1}
+            }
+        )
+
+
+And verify that the documents now load correctly:
+
+.. code-block:: python
+
+    jedi = GoodJedi.objects().first()
+    assert jedi.name == "Obi Wan Kenobi"
+
+    sith = BadSith.objects().first()
+    assert sith.name == "Darth Vader"
+
+
+An other way of dealing with this migration is to iterate over
+the documents and update/replace them one by one. This is way slower but
+it is often useful for complex migrations of Document models.
+
+.. code-block:: python
+
+    for doc in humans_coll.find():
+        if doc['_cls'] == 'Human.Jedi':
+            doc['_cls'] =  'Human.BadSith' if doc['dark_side'] else 'Human.GoodJedi'
+            doc.pop('dark_side')
+            humans_coll.replace_one({'_id': doc['_id']}, doc)
+
+.. warning:: Be aware of this `flaw <https://groups.google.com/g/mongodb-user/c/AFC1ia7MHzk>`_ if you modify documents while iterating
+
+Recommendations
+===============
+
+- Write migration scripts whenever you do changes to the model schemas
+- Using :class:`~mongoengine.DynamicDocument` or ``meta = {"strict": False}`` may help to avoid some migrations or to have the 2 versions of your application to co-exist.
+- Write post-processing checks to verify that migrations script worked. See below
+
+Post-processing checks
+======================
+
+The following recipe can be used to sanity check a Document collection after you applied migration.
+It does not make any assumption on what was migrated, it will fetch 1000 objects randomly and
+run some quick checks on the documents to make sure the document looks OK. As it is, it will fail
+on the first occurrence of an error but this is something that can be adapted based on your needs.
+
+.. code-block:: python
+
+    def get_random_oids(collection, sample_size):
+        pipeline = [{"$project": {'_id': 1}}, {"$sample": {"size": sample_size}}]
+        return [s['_id'] for s in collection.aggregate(pipeline)]
+
+    def get_random_documents(DocCls, sample_size):
+        doc_collection = DocCls._get_collection()
+        random_oids = get_random_oids(doc_collection, sample_size)
+        return DocCls.objects(id__in=random_oids)
+
+    def check_documents(DocCls, sample_size):
+        for doc in get_random_documents(DocCls, sample_size):
+            # general validation (types and values)
+            doc.validate()
+
+            # load all subfields,
+            # this may trigger additional queries if you have ReferenceFields
+            # so it may be slow
+            for field in doc._fields:
+                try:
+                    getattr(doc, field)
+                except Exception:
+                    LOG.warning(f"Could not load field {field} in Document {doc.id}")
+                    raise
+
+    check_documents(Human, sample_size=1000)

docs/guide/querying.rst

@@ -609,7 +609,7 @@ to push values with index::
 .. note::
     Currently only top level lists are handled, future versions of mongodb /
     pymongo plan to support nested positional operators.  See `The $ positional
-    operator <http://www.mongodb.org/display/DOCS/Updating#Updating-The%24positionaloperator>`_.
+    operator <https://docs.mongodb.com/manual/tutorial/update-documents/#Updating-The%24positionaloperator>`_.
 
 Server-side javascript execution
 ================================

docs/guide/validation.rst

@@ -0,0 +1,123 @@
+====================
+Document Validation
+====================
+
+By design, MongoEngine strictly validates the documents right before they are inserted in MongoDB
+and makes sure they are consistent with the fields defined in your models.
+
+MongoEngine makes the assumption that the documents that exists in the DB are compliant with the schema.
+This means that Mongoengine will not validate a document when an object is loaded from the DB into an instance
+of your model but this operation may fail under some circumstances (e.g. if there is a field in
+the document fetched from the database that is not defined in your model).
+
+
+Built-in validation
+===================
+
+Mongoengine provides different fields that encapsulate the corresponding validation
+out of the box. Validation runs when calling `.validate()` or `.save()`
+
+.. code-block:: python
+
+    from mongoengine import Document, EmailField
+
+    class User(Document):
+        email = EmailField()
+        age = IntField(min_value=0, max_value=99)
+
+    user = User(email='invalid@', age=24)
+    user.validate()     # raises ValidationError (Invalid email address: ['email'])
+    user.save()         # raises ValidationError (Invalid email address: ['email'])
+
+    user2 = User(email='john.doe@garbage.com', age=1000)
+    user2.save()        # raises ValidationError (Integer value is too large: ['age'])
+
+Custom validation
+=================
+
+The following feature can be used to customize the validation:
+
+* Field `validation` parameter
+
+.. code-block:: python
+
+    def not_john_doe(name):
+        if name == 'John Doe':
+            raise ValidationError("John Doe is not a valid name")
+
+    class Person(Document):
+        full_name = StringField(validation=not_john_doe)
+
+    Person(full_name='Billy Doe').save()
+    Person(full_name='John Doe').save()  # raises ValidationError (John Doe is not a valid name)
+
+
+* Document `clean` method
+
+This method is called as part of :meth:`~mongoengine.document.Document.save` and should be used to provide
+custom model validation and/or to modify some of the field values prior to validation.
+For instance, you could use it to automatically provide a value for a field, or to do validation
+that requires access to more than a single field.
+
+.. code-block:: python
+
+    class Essay(Document):
+        status = StringField(choices=('Published', 'Draft'), required=True)
+        pub_date = DateTimeField()
+
+        def clean(self):
+            # Validate that only published essays have a `pub_date`
+            if self.status == 'Draft' and self.pub_date is not None:
+                raise ValidationError('Draft entries should not have a publication date.')
+            # Set the pub_date for published items if not set.
+            if self.status == 'Published' and self.pub_date is None:
+                self.pub_date = datetime.now()
+
+.. note::
+    Cleaning is only called if validation is turned on and when calling
+    :meth:`~mongoengine.Document.save`.
+
+* Adding custom Field classes
+
+We recommend as much as possible to use fields provided by MongoEngine. However, it is also possible
+to subclass a Field and encapsulate some validation by overriding the `validate` method
+
+.. code-block:: python
+
+    class AgeField(IntField):
+
+        def validate(self, value):
+            super(AgeField, self).validate(value)     # let IntField.validate run first
+            if value == 60:
+                self.error('60 is not allowed')
+
+    class Person(Document):
+        age = AgeField(min_value=0, max_value=99)
+
+    Person(age=20).save()   # passes
+    Person(age=1000).save() # raises ValidationError (Integer value is too large: ['age'])
+    Person(age=60).save()   # raises ValidationError (Person:None) (60 is not allowed: ['age'])
+
+
+.. note::
+
+   When overriding `validate`, use `self.error("your-custom-error")` instead of raising ValidationError explicitly,
+   it will provide a better context with the error message
+
+Skipping validation
+====================
+
+Although discouraged as it allows to violate fields constraints, if for some reason you need to disable
+the validation and cleaning of a document when you call :meth:`~mongoengine.document.Document.save`, you can use `.save(validate=False)`.
+
+.. code-block:: python
+
+    class Person(Document):
+        age = IntField(max_value=100)
+
+    Person(age=1000).save()    # raises ValidationError (Integer value is too large)
+
+    Person(age=1000).save(validate=False)
+    person = Person.objects.first()
+    assert person.age == 1000
+

docs/requirements.txt

@@ -0,0 +1,3 @@
+pymongo>=3.11
+Sphinx==3.2.1
+sphinx-rtd-theme==0.5.0

mongoengine/__init__.py

@@ -10,12 +10,12 @@ from mongoengine import signals
 # mongoengine, e.g. instead of `from mongoengine.connection import connect`,
 # users can simply use `from mongoengine import connect`, or even
 # `from mongoengine import *` and then `connect('testdb')`.
-from mongoengine.connection import *
-from mongoengine.document import *
-from mongoengine.errors import *
-from mongoengine.fields import *
-from mongoengine.queryset import *
-from mongoengine.signals import *
+from mongoengine.connection import *  # noqa: F401
+from mongoengine.document import *  # noqa: F401
+from mongoengine.errors import *  # noqa: F401
+from mongoengine.fields import *  # noqa: F401
+from mongoengine.queryset import *  # noqa: F401
+from mongoengine.signals import *  # noqa: F401
 
 
 __all__ = (
@@ -28,7 +28,7 @@ __all__ = (
 )
 
 
-VERSION = (0, 20, 0)
+VERSION = (0, 22, 0)
 
 
 def get_version():

mongoengine/base/datastructures.py

@@ -67,11 +67,11 @@ class BaseDict(dict):
         if isinstance(value, EmbeddedDocument) and value._instance is None:
             value._instance = self._instance
         elif isinstance(value, dict) and not isinstance(value, BaseDict):
-            value = BaseDict(value, None, "{}.{}".format(self._name, key))
+            value = BaseDict(value, None, f"{self._name}.{key}")
             super().__setitem__(key, value)
             value._instance = self._instance
         elif isinstance(value, list) and not isinstance(value, BaseList):
-            value = BaseList(value, None, "{}.{}".format(self._name, key))
+            value = BaseList(value, None, f"{self._name}.{key}")
             super().__setitem__(key, value)
             value._instance = self._instance
         return value
@@ -97,7 +97,7 @@ class BaseDict(dict):
     def _mark_as_changed(self, key=None):
         if hasattr(self._instance, "_mark_as_changed"):
             if key:
-                self._instance._mark_as_changed("{}.{}".format(self._name, key))
+                self._instance._mark_as_changed(f"{self._name}.{key}")
             else:
                 self._instance._mark_as_changed(self._name)
 
@@ -133,12 +133,12 @@ class BaseList(list):
             value._instance = self._instance
         elif isinstance(value, dict) and not isinstance(value, BaseDict):
             # Replace dict by BaseDict
-            value = BaseDict(value, None, "{}.{}".format(self._name, key))
+            value = BaseDict(value, None, f"{self._name}.{key}")
             super().__setitem__(key, value)
             value._instance = self._instance
         elif isinstance(value, list) and not isinstance(value, BaseList):
             # Replace list by BaseList
-            value = BaseList(value, None, "{}.{}".format(self._name, key))
+            value = BaseList(value, None, f"{self._name}.{key}")
             super().__setitem__(key, value)
             value._instance = self._instance
         return value
@@ -179,10 +179,8 @@ class BaseList(list):
 
     def _mark_as_changed(self, key=None):
         if hasattr(self._instance, "_mark_as_changed"):
-            if key:
-                self._instance._mark_as_changed(
-                    "{}.{}".format(self._name, key % len(self))
-                )
+            if key is not None:
+                self._instance._mark_as_changed(f"{self._name}.{key % len(self)}")
             else:
                 self._instance._mark_as_changed(self._name)
 
@@ -215,7 +213,7 @@ class EmbeddedDocumentList(BaseList):
         Filters the list by only including embedded documents with the
         given keyword arguments.
 
-        This method only supports simple comparison (e.g: .filter(name='John Doe'))
+        This method only supports simple comparison (e.g. .filter(name='John Doe'))
         and does not support operators like __gte, __lte, __icontains like queryset.filter does
 
         :param kwargs: The keyword arguments corresponding to the fields to
@@ -429,7 +427,7 @@ class StrictDict:
 
                 def __repr__(self):
                     return "{%s}" % ", ".join(
-                        '"{!s}": {!r}'.format(k, v) for k, v in self.items()
+                        f'"{k!s}": {v!r}' for k, v in self.items()
                     )
 
             cls._classes[allowed_keys] = SpecificStrictDict
@@ -472,4 +470,4 @@ class LazyReference(DBRef):
             raise AttributeError()
 
     def __repr__(self):
-        return "<LazyReference({}, {!r})>".format(self.document_type, self.pk)
+        return f"<LazyReference({self.document_type}, {self.pk!r})>"

mongoengine/base/document.py

@@ -64,8 +64,6 @@ class BaseDocument:
             It may contain additional reserved keywords, e.g. "__auto_convert".
         :param __auto_convert: If True, supplied values will be converted
             to Python-type values via each field's `to_python` method.
-        :param __only_fields: A set of fields that have been loaded for
-            this document. Empty if all fields have been loaded.
         :param _created: Indicates whether this is a brand new document
             or whether it's already been persisted before. Defaults to true.
         """
@@ -80,8 +78,6 @@ class BaseDocument:
 
         __auto_convert = values.pop("__auto_convert", True)
 
-        __only_fields = set(values.pop("__only_fields", values))
-
         _created = values.pop("_created", True)
 
         signals.pre_init.send(self.__class__, document=self, values=values)
@@ -93,9 +89,7 @@ class BaseDocument:
                 list(self._fields.keys()) + ["id", "pk", "_cls", "_text_score"]
             )
             if _undefined_fields:
-                msg = ('The fields "{}" do not exist on the document "{}"').format(
-                    _undefined_fields, self._class_name
-                )
+                msg = f'The fields "{_undefined_fields}" do not exist on the document "{self._class_name}"'
                 raise FieldDoesNotExist(msg)
 
         if self.STRICT and not self._dynamic:
@@ -105,37 +99,32 @@ class BaseDocument:
 
         self._dynamic_fields = SON()
 
-        # Assign default values to the instance.
-        # We set default values only for fields loaded from DB. See
-        # https://github.com/mongoengine/mongoengine/issues/399 for more info.
-        for key, field in self._fields.items():
-            if self._db_field_map.get(key, key) in __only_fields:
+        # Assign default values for fields
+        # not set in the constructor
+        for field_name in self._fields:
+            if field_name in values:
                 continue
-            value = getattr(self, key, None)
-            setattr(self, key, value)
+            value = getattr(self, field_name, None)
+            setattr(self, field_name, value)
 
         if "_cls" not in values:
             self._cls = self._class_name
 
-        # Set passed values after initialisation
-        if self._dynamic:
-            dynamic_data = {}
-            for key, value in values.items():
-                if key in self._fields or key == "_id":
-                    setattr(self, key, value)
-                else:
+        # Set actual values
+        dynamic_data = {}
+        FileField = _import_class("FileField")
+        for key, value in values.items():
+            field = self._fields.get(key)
+            if field or key in ("id", "pk", "_cls"):
+                if __auto_convert and value is not None:
+                    if field and not isinstance(field, FileField):
+                        value = field.to_python(value)
+                setattr(self, key, value)
+            else:
+                if self._dynamic:
                     dynamic_data[key] = value
-        else:
-            FileField = _import_class("FileField")
-            for key, value in values.items():
-                key = self._reverse_db_field_map.get(key, key)
-                if key in self._fields or key in ("id", "pk", "_cls"):
-                    if __auto_convert and value is not None:
-                        field = self._fields.get(key)
-                        if field and not isinstance(field, FileField):
-                            value = field.to_python(value)
-                    setattr(self, key, value)
                 else:
+                    # For strict Document
                     self._data[key] = value
 
         # Set any get_<field>_display methods
@@ -240,10 +229,10 @@ class BaseDocument:
                 setattr(self, k, data[k])
         if "_fields_ordered" in data:
             if self._dynamic:
-                setattr(self, "_fields_ordered", data["_fields_ordered"])
+                self._fields_ordered = data["_fields_ordered"]
             else:
                 _super_fields_ordered = type(self)._fields_ordered
-                setattr(self, "_fields_ordered", _super_fields_ordered)
+                self._fields_ordered = _super_fields_ordered
 
         dynamic_fields = data.get("_dynamic_fields") or SON()
         for k in dynamic_fields.keys():
@@ -253,8 +242,7 @@ class BaseDocument:
         return iter(self._fields_ordered)
 
     def __getitem__(self, name):
-        """Dictionary-style field access, return a field's value if present.
-        """
+        """Dictionary-style field access, return a field's value if present."""
         try:
             if name in self._fields_ordered:
                 return getattr(self, name)
@@ -263,8 +251,7 @@ class BaseDocument:
         raise KeyError(name)
 
     def __setitem__(self, name, value):
-        """Dictionary-style field access, set a field's value.
-        """
+        """Dictionary-style field access, set a field's value."""
         # Ensure that the field exists before settings its value
         if not self._dynamic and name not in self._fields:
             raise KeyError(name)
@@ -286,7 +273,7 @@ class BaseDocument:
         except (UnicodeEncodeError, UnicodeDecodeError):
             u = "[Bad Unicode data]"
         repr_type = str if u is None else type(u)
-        return repr_type("<{}: {}>".format(self.__class__.__name__, u))
+        return repr_type(f"<{self.__class__.__name__}: {u}>")
 
     def __str__(self):
         # TODO this could be simpler?
@@ -314,7 +301,8 @@ class BaseDocument:
 
     def clean(self):
         """
-        Hook for doing document level data cleaning before validation is run.
+        Hook for doing document level data cleaning (usually validation or assignment)
+        before validation is run.
 
         Any ValidationError raised by this method will not be associated with
         a particular field; it will have a special-case association with the
@@ -441,7 +429,7 @@ class BaseDocument:
                 pk = self.pk
             elif self._instance and hasattr(self._instance, "pk"):
                 pk = self._instance.pk
-            message = "ValidationError ({}:{}) ".format(self._class_name, pk)
+            message = f"ValidationError ({self._class_name}:{pk}) "
             raise ValidationError(message, errors=errors)
 
     def to_json(self, *args, **kwargs):
@@ -514,7 +502,7 @@ class BaseDocument:
         if "." in key:
             key, rest = key.split(".", 1)
             key = self._db_field_map.get(key, key)
-            key = "{}.{}".format(key, rest)
+            key = f"{key}.{rest}"
         else:
             key = self._db_field_map.get(key, key)
 
@@ -537,6 +525,9 @@ class BaseDocument:
         """Using _get_changed_fields iterate and remove any fields that
         are marked as changed.
         """
+        ReferenceField = _import_class("ReferenceField")
+        GenericReferenceField = _import_class("GenericReferenceField")
+
         for changed in self._get_changed_fields():
             parts = changed.split(".")
             data = self
@@ -549,7 +540,8 @@ class BaseDocument:
                 elif isinstance(data, dict):
                     data = data.get(part, None)
                 else:
-                    data = getattr(data, part, None)
+                    field_name = data._reverse_db_field_map.get(part, part)
+                    data = getattr(data, field_name, None)
 
                 if not isinstance(data, LazyReference) and hasattr(
                     data, "_changed_fields"
@@ -558,10 +550,40 @@ class BaseDocument:
                         continue
 
                     data._changed_fields = []
+                elif isinstance(data, (list, tuple, dict)):
+                    if hasattr(data, "field") and isinstance(
+                        data.field, (ReferenceField, GenericReferenceField)
+                    ):
+                        continue
+                    BaseDocument._nestable_types_clear_changed_fields(data)
 
         self._changed_fields = []
 
-    def _nestable_types_changed_fields(self, changed_fields, base_key, data):
+    @staticmethod
+    def _nestable_types_clear_changed_fields(data):
+        """Inspect nested data for changed fields
+
+        :param data: data to inspect for changes
+        """
+        Document = _import_class("Document")
+
+        # Loop list / dict fields as they contain documents
+        # Determine the iterator to use
+        if not hasattr(data, "items"):
+            iterator = enumerate(data)
+        else:
+            iterator = data.items()
+
+        for _index_or_key, value in iterator:
+            if hasattr(value, "_get_changed_fields") and not isinstance(
+                value, Document
+            ):  # don't follow references
+                value._clear_changed_fields()
+            elif isinstance(value, (list, tuple, dict)):
+                BaseDocument._nestable_types_clear_changed_fields(value)
+
+    @staticmethod
+    def _nestable_types_changed_fields(changed_fields, base_key, data):
         """Inspect nested data for changed fields
 
         :param changed_fields: Previously collected changed fields
@@ -576,7 +598,7 @@ class BaseDocument:
             iterator = data.items()
 
         for index_or_key, value in iterator:
-            item_key = "{}{}.".format(base_key, index_or_key)
+            item_key = f"{base_key}{index_or_key}."
             # don't check anything lower if this key is already marked
             # as changed.
             if item_key[:-1] in changed_fields:
@@ -584,13 +606,14 @@ class BaseDocument:
 
             if hasattr(value, "_get_changed_fields"):
                 changed = value._get_changed_fields()
-                changed_fields += ["{}{}".format(item_key, k) for k in changed if k]
+                changed_fields += [f"{item_key}{k}" for k in changed if k]
             elif isinstance(value, (list, tuple, dict)):
-                self._nestable_types_changed_fields(changed_fields, item_key, value)
+                BaseDocument._nestable_types_changed_fields(
+                    changed_fields, item_key, value
+                )
 
     def _get_changed_fields(self):
-        """Return a list of all fields that have explicitly been changed.
-        """
+        """Return a list of all fields that have explicitly been changed."""
         EmbeddedDocument = _import_class("EmbeddedDocument")
         ReferenceField = _import_class("ReferenceField")
         GenericReferenceField = _import_class("GenericReferenceField")
@@ -615,7 +638,7 @@ class BaseDocument:
             if isinstance(data, EmbeddedDocument):
                 # Find all embedded fields that have been changed
                 changed = data._get_changed_fields()
-                changed_fields += ["{}{}".format(key, k) for k in changed if k]
+                changed_fields += [f"{key}{k}" for k in changed if k]
             elif isinstance(data, (list, tuple, dict)):
                 if hasattr(field, "field") and isinstance(
                     field.field, (ReferenceField, GenericReferenceField)
@@ -721,11 +744,8 @@ class BaseDocument:
         return cls._meta.get("collection", None)
 
     @classmethod
-    def _from_son(cls, son, _auto_dereference=True, only_fields=None, created=False):
-        """Create an instance of a Document (subclass) from a PyMongo SON."""
-        if not only_fields:
-            only_fields = []
-
+    def _from_son(cls, son, _auto_dereference=True, created=False):
+        """Create an instance of a Document (subclass) from a PyMongo SON (dict)"""
         if son and not isinstance(son, dict):
             raise ValueError(
                 "The source SON object needs to be of type 'dict' but a '%s' was found"
@@ -738,6 +758,8 @@ class BaseDocument:
 
         # Convert SON to a data dict, making sure each key is a string and
         # corresponds to the right db field.
+        # This is needed as _from_son is currently called both from BaseDocument.__init__
+        # and from EmbeddedDocumentField.to_python
         data = {}
         for key, value in son.items():
             key = str(key)
@@ -768,11 +790,10 @@ class BaseDocument:
                     errors_dict[field_name] = e
 
         if errors_dict:
-            errors = "\n".join(
-                ["Field '{}' - {}".format(k, v) for k, v in errors_dict.items()]
-            )
+            errors = "\n".join([f"Field '{k}' - {v}" for k, v in errors_dict.items()])
             msg = "Invalid data to create a `{}` instance.\n{}".format(
-                cls._class_name, errors,
+                cls._class_name,
+                errors,
             )
             raise InvalidDocumentError(msg)
 
@@ -780,9 +801,7 @@ class BaseDocument:
         if cls.STRICT:
             data = {k: v for k, v in data.items() if k in cls._fields}
 
-        obj = cls(
-            __auto_convert=False, _created=created, __only_fields=only_fields, **data
-        )
+        obj = cls(__auto_convert=False, _created=created, **data)
         obj._changed_fields = []
         if not _auto_dereference:
             obj._fields = fields
@@ -942,10 +961,7 @@ class BaseDocument:
                     unique_fields += unique_with
 
                 # Add the new index to the list
-                fields = [
-                    ("{}{}".format(namespace, f), pymongo.ASCENDING)
-                    for f in unique_fields
-                ]
+                fields = [(f"{namespace}{f}", pymongo.ASCENDING) for f in unique_fields]
                 index = {"fields": fields, "unique": True, "sparse": sparse}
                 unique_indexes.append(index)
 
@@ -981,9 +997,7 @@ class BaseDocument:
             "PolygonField",
         )
 
-        geo_field_types = tuple(
-            [_import_class(field) for field in geo_field_type_names]
-        )
+        geo_field_types = tuple(_import_class(field) for field in geo_field_type_names)
 
         for field in cls._fields.values():
             if not isinstance(field, geo_field_types):
@@ -1001,7 +1015,7 @@ class BaseDocument:
             elif field._geo_index:
                 field_name = field.db_field
                 if parent_field:
-                    field_name = "{}.{}".format(parent_field, field_name)
+                    field_name = f"{parent_field}.{field_name}"
                 geo_indices.append({"fields": [(field_name, field._geo_index)]})
 
         return geo_indices
@@ -1139,8 +1153,7 @@ class BaseDocument:
 
     @classmethod
     def _translate_field_name(cls, field, sep="."):
-        """Translate a field attribute name to a database field name.
-        """
+        """Translate a field attribute name to a database field name."""
         parts = field.split(sep)
         parts = [f.db_field for f in cls._lookup_field(parts)]
         return ".".join(parts)

mongoengine/base/fields.py

@@ -1,5 +1,4 @@
 import operator
-import warnings
 import weakref
 
 from bson import DBRef, ObjectId, SON
@@ -16,11 +15,9 @@ __all__ = ("BaseField", "ComplexBaseField", "ObjectIdField", "GeoJsonBaseField")
 class BaseField:
     """A base class for fields in a MongoDB document. Instances of this class
     may be added to subclasses of `Document` to define a document's schema.
-
-    .. versionchanged:: 0.5 - added verbose and help text
     """
 
-    name = None
+    name = None  # set in TopLevelDocumentMetaclass
     _geo_index = False
     _auto_gen = False  # Call `generate` to generate a value
     _auto_dereference = True
@@ -43,7 +40,7 @@ class BaseField:
         choices=None,
         null=False,
         sparse=False,
-        **kwargs
+        **kwargs,
     ):
         """
         :param db_field: The database field to store this field in
@@ -120,8 +117,7 @@ class BaseField:
             BaseField.creation_counter += 1
 
     def __get__(self, instance, owner):
-        """Descriptor for retrieving a value from a field in a document.
-        """
+        """Descriptor for retrieving a value from a field in a document."""
         if instance is None:
             # Document class being used rather than a document object
             return self
@@ -265,11 +261,11 @@ class ComplexBaseField(BaseField):
     Allows for nesting of embedded documents inside complex types.
     Handles the lazy dereferencing of a queryset by lazily dereferencing all
     items in a list / dict rather than one at a time.
-
-    .. versionadded:: 0.5
     """
 
-    field = None
+    def __init__(self, field=None, **kwargs):
+        self.field = field
+        super().__init__(**kwargs)
 
     def __get__(self, instance, owner):
         """Descriptor to automatically dereference references."""
@@ -469,9 +465,7 @@ class ComplexBaseField(BaseField):
 
             if errors:
                 field_class = self.field.__class__.__name__
-                self.error(
-                    "Invalid {} item ({})".format(field_class, value), errors=errors
-                )
+                self.error(f"Invalid {field_class} item ({value})", errors=errors)
         # Don't allow empty values if required
         if self.required and not value:
             self.error("Field is required and cannot be empty")
@@ -520,10 +514,7 @@ class ObjectIdField(BaseField):
 
 
 class GeoJsonBaseField(BaseField):
-    """A geo json field storing a geojson style object.
-
-    .. versionadded:: 0.8
-    """
+    """A geo json field storing a geojson style object."""
 
     _geo_index = pymongo.GEOSPHERE
     _type = "GeoBase"
@@ -543,7 +534,7 @@ class GeoJsonBaseField(BaseField):
         if isinstance(value, dict):
             if set(value.keys()) == {"type", "coordinates"}:
                 if value["type"] != self._type:
-                    self.error('{} type must be "{}"'.format(self._name, self._type))
+                    self.error(f'{self._name} type must be "{self._type}"')
                 return self.validate(value["coordinates"])
             else:
                 self.error(

mongoengine/base/metaclasses.py

@@ -337,7 +337,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
         # allow_inheritance to False. If the base Document allows inheritance,
         # none of its subclasses can override allow_inheritance to False.
         simple_class = all(
-            [b._meta.get("abstract") for b in flattened_bases if hasattr(b, "_meta")]
+            b._meta.get("abstract") for b in flattened_bases if hasattr(b, "_meta")
         )
         if (
             not simple_class
@@ -439,8 +439,8 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
 
         id_basename, id_db_basename, i = ("auto_id", "_auto_id", 0)
         for i in itertools.count():
-            id_name = "{}_{}".format(id_basename, i)
-            id_db_name = "{}_{}".format(id_db_basename, i)
+            id_name = f"{id_basename}_{i}"
+            id_db_name = f"{id_db_basename}_{i}"
             if id_name not in existing_fields and id_db_name not in existing_db_fields:
                 return id_name, id_db_name
 

mongoengine/connection.py

@@ -54,7 +54,7 @@ def _get_connection_settings(
     password=None,
     authentication_source=None,
     authentication_mechanism=None,
-    **kwargs
+    **kwargs,
 ):
     """Get the connection settings as a dict
 
@@ -74,8 +74,6 @@ def _get_connection_settings(
     : param kwargs: ad-hoc parameters to be passed into the pymongo driver,
         for example maxpoolsize, tz_aware, etc. See the documentation
         for pymongo's `MongoClient` for a full list.
-
-    .. versionchanged:: 0.10.6 - added mongomock support
     """
     conn_settings = {
         "name": name or db or DEFAULT_DATABASE_NAME,
@@ -179,7 +177,7 @@ def register_connection(
     password=None,
     authentication_source=None,
     authentication_mechanism=None,
-    **kwargs
+    **kwargs,
 ):
     """Register the connection settings.
 
@@ -201,8 +199,6 @@ def register_connection(
     : param kwargs: ad-hoc parameters to be passed into the pymongo driver,
         for example maxpoolsize, tz_aware, etc. See the documentation
         for pymongo's `MongoClient` for a full list.
-
-    .. versionchanged:: 0.10.6 - added mongomock support
     """
     conn_settings = _get_connection_settings(
         db=db,
@@ -214,7 +210,7 @@ def register_connection(
         password=password,
         authentication_source=authentication_source,
         authentication_mechanism=authentication_mechanism,
-        **kwargs
+        **kwargs,
     )
     _connection_settings[alias] = conn_settings
 
@@ -317,7 +313,7 @@ def _create_connection(alias, connection_class, **connection_settings):
     try:
         return connection_class(**connection_settings)
     except Exception as e:
-        raise ConnectionFailure("Cannot connect to database {} :\n{}".format(alias, e))
+        raise ConnectionFailure(f"Cannot connect to database {alias} :\n{e}")
 
 
 def _find_existing_connection(connection_settings):
@@ -386,8 +382,6 @@ def connect(db=None, alias=DEFAULT_CONNECTION_NAME, **kwargs):
 
     See the docstring for `register_connection` for more details about all
     supported kwargs.
-
-    .. versionchanged:: 0.6 - added multiple database support.
     """
     if alias in _connections:
         prev_conn_setting = _connection_settings[alias]

mongoengine/context_managers.py

@@ -184,8 +184,7 @@ class query_counter:
     """
 
     def __init__(self, alias=DEFAULT_CONNECTION_NAME):
-        """Construct the query_counter
-        """
+        """Construct the query_counter"""
         self.db = get_db(alias=alias)
         self.initial_profiling_level = None
         self._ctx_query_counter = 0  # number of queries issued by the context

mongoengine/dereference.py

@@ -51,10 +51,10 @@ class DeReference:
                 doc_type = doc_type.document_type
                 is_list = not hasattr(items, "items")
 
-                if is_list and all([i.__class__ == doc_type for i in items]):
+                if is_list and all(i.__class__ == doc_type for i in items):
                     return items
                 elif not is_list and all(
-                    [i.__class__ == doc_type for i in items.values()]
+                    i.__class__ == doc_type for i in items.values()
                 ):
                     return items
                 elif not field.dbref:
@@ -157,8 +157,7 @@ class DeReference:
         return reference_map
 
     def _fetch_objects(self, doc_type=None):
-        """Fetch all references and convert to their document objects
-        """
+        """Fetch all references and convert to their document objects"""
         object_map = {}
         for collection, dbrefs in self.reference_map.items():
 
@@ -272,12 +271,12 @@ class DeReference:
                             (v["_ref"].collection, v["_ref"].id), v
                         )
                     elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:
-                        item_name = "{}.{}.{}".format(name, k, field_name)
+                        item_name = f"{name}.{k}.{field_name}"
                         data[k]._data[field_name] = self._attach_objects(
                             v, depth, instance=instance, name=item_name
                         )
             elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:
-                item_name = "{}.{}".format(name, k) if name else name
+                item_name = f"{name}.{k}" if name else name
                 data[k] = self._attach_objects(
                     v, depth - 1, instance=instance, name=item_name
                 )

mongoengine/document.py

@@ -1,5 +1,4 @@
 import re
-import warnings
 
 from bson.dbref import DBRef
 import pymongo
@@ -367,15 +366,6 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
             meta['cascade'] = True.  Also you can pass different kwargs to
             the cascade save using cascade_kwargs which overwrites the
             existing kwargs with custom values.
-        .. versionchanged:: 0.8.5
-            Optional save_condition that only overwrites existing documents
-            if the condition is satisfied in the current db record.
-        .. versionchanged:: 0.10
-            :class:`OperationError` exception raised if save_condition fails.
-        .. versionchanged:: 0.10.1
-            :class: save_condition failure now raises a `SaveConditionError`
-        .. versionchanged:: 0.10.7
-            Add signal_kwargs argument
         """
         signal_kwargs = signal_kwargs or {}
 
@@ -464,9 +454,9 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
             # insert_one will provoke UniqueError alongside save does not
             # therefore, it need to catch and call replace_one.
             if "_id" in doc:
-                raw_object = wc_collection.find_one_and_replace(
-                    {"_id": doc["_id"]}, doc
-                )
+                select_dict = {"_id": doc["_id"]}
+                select_dict = self._integrate_shard_key(doc, select_dict)
+                raw_object = wc_collection.find_one_and_replace(select_dict, doc)
                 if raw_object:
                     return doc["_id"]
 
@@ -489,6 +479,23 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
 
         return update_doc
 
+    def _integrate_shard_key(self, doc, select_dict):
+        """Integrates the collection's shard key to the `select_dict`, which will be used for the query.
+        The value from the shard key is taken from the `doc` and finally the select_dict is returned.
+        """
+
+        # Need to add shard key to query, or you get an error
+        shard_key = self._meta.get("shard_key", tuple())
+        for k in shard_key:
+            path = self._lookup_field(k.split("."))
+            actual_key = [p.db_field for p in path]
+            val = doc
+            for ak in actual_key:
+                val = val[ak]
+            select_dict[".".join(actual_key)] = val
+
+        return select_dict
+
     def _save_update(self, doc, save_condition, write_concern):
         """Update an existing document.
 
@@ -504,15 +511,7 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
 
         select_dict["_id"] = object_id
 
-        # Need to add shard key to query, or you get an error
-        shard_key = self._meta.get("shard_key", tuple())
-        for k in shard_key:
-            path = self._lookup_field(k.split("."))
-            actual_key = [p.db_field for p in path]
-            val = doc
-            for ak in actual_key:
-                val = val[ak]
-            select_dict[".".join(actual_key)] = val
+        select_dict = self._integrate_shard_key(doc, select_dict)
 
         update_doc = self._get_update_doc()
         if update_doc:
@@ -621,9 +620,6 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
             For example, ``save(..., w: 2, fsync: True)`` will
             wait until at least two servers have recorded the write and
             will force an fsync on the primary server.
-
-        .. versionchanged:: 0.10.7
-            Add signal_kwargs argument
         """
         signal_kwargs = signal_kwargs or {}
         signals.pre_delete.send(self.__class__, document=self, **signal_kwargs)
@@ -639,7 +635,7 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
                 write_concern=write_concern, _from_doc_delete=True
             )
         except pymongo.errors.OperationFailure as err:
-            message = "Could not delete document (%s)" % err.message
+            message = "Could not delete document (%s)" % err.args
             raise OperationError(message)
         signals.post_delete.send(self.__class__, document=self, **signal_kwargs)
 
@@ -705,8 +701,6 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
     def select_related(self, max_depth=1):
         """Handles dereferencing of :class:`~bson.dbref.DBRef` objects to
         a maximum depth in order to cut down the number queries to mongodb.
-
-        .. versionadded:: 0.5
         """
         DeReference = _import_class("DeReference")
         DeReference()([self], max_depth + 1)
@@ -717,10 +711,6 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
 
         :param fields: (optional) args list of fields to reload
         :param max_depth: (optional) depth of dereferencing to follow
-
-        .. versionadded:: 0.1.2
-        .. versionchanged:: 0.6  Now chainable
-        .. versionchanged:: 0.9  Can provide specific fields to reload
         """
         max_depth = 1
         if fields and isinstance(fields[0], int):
@@ -822,9 +812,6 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
 
         Raises :class:`OperationError` if the document has no collection set
         (i.g. if it is `abstract`)
-
-        .. versionchanged:: 0.10.7
-            :class:`OperationError` exception raised if no collection available
         """
         coll_name = cls._get_collection_name()
         if not coll_name:
@@ -919,7 +906,7 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
 
     @classmethod
     def list_indexes(cls):
-        """ Lists all of the indexes that should be created for given
+        """Lists all of the indexes that should be created for given
         collection. It includes all the indexes from super- and sub-classes.
         """
         if cls._meta.get("abstract"):
@@ -984,7 +971,7 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
 
     @classmethod
     def compare_indexes(cls):
-        """ Compares the indexes defined in MongoEngine with the ones
+        """Compares the indexes defined in MongoEngine with the ones
         existing in the database. Returns any missing/extra indexes.
         """
 
@@ -1079,8 +1066,6 @@ class MapReduceDocument:
                 an ``ObjectId`` found in the given ``collection``,
                 the object can be accessed via the ``object`` property.
     :param value: The result(s) for this key.
-
-    .. versionadded:: 0.3
     """
 
     def __init__(self, document, collection, key, value):

mongoengine/errors.py

@@ -94,7 +94,7 @@ class ValidationError(AssertionError):
         return str(self.message)
 
     def __repr__(self):
-        return "{}({},)".format(self.__class__.__name__, self.message)
+        return f"{self.__class__.__name__}({self.message},)"
 
     def __getattribute__(self, name):
         message = super().__getattribute__(name)
@@ -102,7 +102,7 @@ class ValidationError(AssertionError):
             if self.field_name:
                 message = "%s" % message
             if self.errors:
-                message = "{}({})".format(message, self._format_errors())
+                message = f"{message}({self._format_errors()})"
         return message
 
     def _get_message(self):
@@ -147,13 +147,13 @@ class ValidationError(AssertionError):
             elif isinstance(value, dict):
                 value = " ".join([generate_key(v, k) for k, v in value.items()])
 
-            results = "{}.{}".format(prefix, value) if prefix else value
+            results = f"{prefix}.{value}" if prefix else value
             return results
 
         error_dict = defaultdict(list)
         for k, v in self.to_dict().items():
             error_dict[generate_key(v)].append(k)
-        return " ".join(["{}: {}".format(k, v) for k, v in error_dict.items()])
+        return " ".join([f"{k}: {v}" for k, v in error_dict.items()])
 
 
 class DeprecatedError(Exception):

mongoengine/fields.py

@@ -36,7 +36,6 @@ from mongoengine.common import _import_class
 from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db
 from mongoengine.document import Document, EmbeddedDocument
 from mongoengine.errors import DoesNotExist, InvalidQueryError, ValidationError
-from mongoengine.mongodb_support import MONGODB_36, get_mongodb_version
 from mongoengine.queryset import DO_NOTHING
 from mongoengine.queryset.base import BaseQuerySet
 from mongoengine.queryset.transform import STRING_OPERATORS
@@ -87,6 +86,7 @@ __all__ = (
     "PolygonField",
     "SequenceField",
     "UUIDField",
+    "EnumField",
     "MultiPointField",
     "MultiLineStringField",
     "MultiPolygonField",
@@ -100,6 +100,12 @@ class StringField(BaseField):
     """A unicode string field."""
 
     def __init__(self, regex=None, max_length=None, min_length=None, **kwargs):
+        """
+        :param regex: (optional) A string pattern that will be applied during validation
+        :param max_length: (optional) A max length that will be applied during validation
+        :param min_length: (optional) A min length that will be applied during validation
+        :param kwargs: Keyword arguments passed into the parent :class:`~mongoengine.BaseField`
+        """
         self.regex = re.compile(regex) if regex else None
         self.max_length = max_length
         self.min_length = min_length
@@ -155,10 +161,7 @@ class StringField(BaseField):
 
 
 class URLField(StringField):
-    """A field that validates input as an URL.
-
-    .. versionadded:: 0.3
-    """
+    """A field that validates input as an URL."""
 
     _URL_REGEX = LazyRegexCompiler(
         r"^(?:[a-z0-9\.\-]*)://"  # scheme is validated separately
@@ -173,6 +176,11 @@ class URLField(StringField):
     _URL_SCHEMES = ["http", "https", "ftp", "ftps"]
 
     def __init__(self, url_regex=None, schemes=None, **kwargs):
+        """
+        :param url_regex: (optional) Overwrite the default regex used for validation
+        :param schemes: (optional) Overwrite the default URL schemes that are allowed
+        :param kwargs: Keyword arguments passed into the parent :class:`~mongoengine.StringField`
+        """
         self.url_regex = url_regex or self._URL_REGEX
         self.schemes = schemes or self._URL_SCHEMES
         super().__init__(**kwargs)
@@ -181,18 +189,15 @@ class URLField(StringField):
         # Check first if the scheme is valid
         scheme = value.split("://")[0].lower()
         if scheme not in self.schemes:
-            self.error("Invalid scheme {} in URL: {}".format(scheme, value))
+            self.error(f"Invalid scheme {scheme} in URL: {value}")
 
         # Then check full URL
         if not self.url_regex.match(value):
-            self.error("Invalid URL: {}".format(value))
+            self.error(f"Invalid URL: {value}")
 
 
 class EmailField(StringField):
-    """A field that validates input as an email address.
-
-    .. versionadded:: 0.4
-    """
+    """A field that validates input as an email address."""
 
     USER_REGEX = LazyRegexCompiler(
         # `dot-atom` defined in RFC 5322 Section 3.2.3.
@@ -226,18 +231,13 @@ class EmailField(StringField):
         allow_utf8_user=False,
         allow_ip_domain=False,
         *args,
-        **kwargs
+        **kwargs,
     ):
-        """Initialize the EmailField.
-
-        Args:
-            domain_whitelist (list) - list of otherwise invalid domain
-                                      names which you'd like to support.
-            allow_utf8_user (bool) - if True, the user part of the email
-                                     address can contain UTF8 characters.
-                                     False by default.
-            allow_ip_domain (bool) - if True, the domain part of the email
-                                     can be a valid IPv4 or IPv6 address.
+        """
+        :param domain_whitelist: (optional) list of valid domain names applied during validation
+        :param allow_utf8_user: Allow user part of the email to contain utf8 char
+        :param allow_ip_domain: Allow domain part of the email to be an IPv4 or IPv6 address
+        :param kwargs: Keyword arguments passed into the parent :class:`~mongoengine.StringField`
         """
         self.domain_whitelist = domain_whitelist or []
         self.allow_utf8_user = allow_utf8_user
@@ -309,6 +309,11 @@ class IntField(BaseField):
     """32-bit integer field."""
 
     def __init__(self, min_value=None, max_value=None, **kwargs):
+        """
+        :param min_value: (optional) A min value that will be applied during validation
+        :param max_value: (optional) A max value that will be applied during validation
+        :param kwargs: Keyword arguments passed into the parent :class:`~mongoengine.BaseField`
+        """
         self.min_value, self.max_value = min_value, max_value
         super().__init__(**kwargs)
 
@@ -342,6 +347,11 @@ class LongField(BaseField):
     """64-bit integer field. (Equivalent to IntField since the support to Python2 was dropped)"""
 
     def __init__(self, min_value=None, max_value=None, **kwargs):
+        """
+        :param min_value: (optional) A min value that will be applied during validation
+        :param max_value: (optional) A max value that will be applied during validation
+        :param kwargs: Keyword arguments passed into the parent :class:`~mongoengine.BaseField`
+        """
         self.min_value, self.max_value = min_value, max_value
         super().__init__(**kwargs)
 
@@ -378,6 +388,11 @@ class FloatField(BaseField):
     """Floating point number field."""
 
     def __init__(self, min_value=None, max_value=None, **kwargs):
+        """
+        :param min_value: (optional) A min value that will be applied during validation
+        :param max_value: (optional) A max value that will be applied during validation
+        :param kwargs: Keyword arguments passed into the parent :class:`~mongoengine.BaseField`
+        """
         self.min_value, self.max_value = min_value, max_value
         super().__init__(**kwargs)
 
@@ -414,9 +429,6 @@ class FloatField(BaseField):
 class DecimalField(BaseField):
     """Fixed-point decimal number field. Stores the value as a float by default unless `force_string` is used.
     If using floats, beware of Decimal to float conversion (potential precision loss)
-
-    .. versionchanged:: 0.8
-    .. versionadded:: 0.3
     """
 
     def __init__(
@@ -426,14 +438,14 @@ class DecimalField(BaseField):
         force_string=False,
         precision=2,
         rounding=decimal.ROUND_HALF_UP,
-        **kwargs
+        **kwargs,
     ):
         """
-        :param min_value: Validation rule for the minimum acceptable value.
-        :param max_value: Validation rule for the maximum acceptable value.
+        :param min_value: (optional) A min value that will be applied during validation
+        :param max_value: (optional) A max value that will be applied during validation
         :param force_string: Store the value as a string (instead of a float).
          Be aware that this affects query sorting and operation like lte, gte (as string comparison is applied)
-         and some query operator won't work (e.g: inc, dec)
+         and some query operator won't work (e.g. inc, dec)
         :param precision: Number of decimal places to store.
         :param rounding: The rounding rule from the python decimal library:
 
@@ -447,7 +459,7 @@ class DecimalField(BaseField):
             - decimal.ROUND_05UP (away from zero if last digit after rounding towards zero would have been 0 or 5; otherwise towards zero)
 
             Defaults to: ``decimal.ROUND_HALF_UP``
-
+        :param kwargs: Keyword arguments passed into the parent :class:`~mongoengine.BaseField`
         """
         self.min_value = min_value
         self.max_value = max_value
@@ -497,10 +509,7 @@ class DecimalField(BaseField):
 
 
 class BooleanField(BaseField):
-    """Boolean field type.
-
-    .. versionadded:: 0.1.2
-    """
+    """Boolean field type."""
 
     def to_python(self, value):
         try:
@@ -545,12 +554,13 @@ class DateTimeField(BaseField):
         if callable(value):
             return value()
 
-        if not isinstance(value, str):
+        if isinstance(value, str):
+            return self._parse_datetime(value)
+        else:
             return None
 
-        return self._parse_datetime(value)
-
-    def _parse_datetime(self, value):
+    @staticmethod
+    def _parse_datetime(value):
         # Attempt to parse a datetime from a string
         value = value.strip()
         if not value:
@@ -626,13 +636,12 @@ class ComplexDateTimeField(StringField):
     keyword when initializing the field.
 
     Note: To default the field to the current datetime, use: DateTimeField(default=datetime.utcnow)
-
-    .. versionadded:: 0.5
     """
 
     def __init__(self, separator=",", **kwargs):
         """
         :param separator: Allows to customize the separator used for storage (default ``,``)
+        :param kwargs: Keyword arguments passed into the parent :class:`~mongoengine.StringField`
         """
         self.separator = separator
         self.format = separator.join(["%Y", "%m", "%d", "%H", "%M", "%S", "%f"])
@@ -773,6 +782,9 @@ class EmbeddedDocumentField(BaseField):
 
     def prepare_query_value(self, op, value):
         if value is not None and not isinstance(value, self.document_type):
+            # Short circuit for special operators, returning them as is
+            if isinstance(value, dict) and all(k.startswith("$") for k in value.keys()):
+                return value
             try:
                 value = self.document_type._from_son(value)
             except ValueError:
@@ -844,8 +856,7 @@ class DynamicField(BaseField):
     Used by :class:`~mongoengine.DynamicDocument` to handle dynamic data"""
 
     def to_mongo(self, value, use_db_field=True, fields=None):
-        """Convert a Python type to a MongoDB compatible type.
-        """
+        """Convert a Python type to a MongoDB compatible type."""
 
         if isinstance(value, str):
             return value
@@ -910,10 +921,9 @@ class ListField(ComplexBaseField):
     """
 
     def __init__(self, field=None, max_length=None, **kwargs):
-        self.field = field
         self.max_length = max_length
         kwargs.setdefault("default", lambda: [])
-        super().__init__(**kwargs)
+        super().__init__(field=field, **kwargs)
 
     def __get__(self, instance, owner):
         if instance is None:
@@ -972,16 +982,13 @@ class EmbeddedDocumentListField(ListField):
     .. note::
         The only valid list values are subclasses of
         :class:`~mongoengine.EmbeddedDocument`.
-
-    .. versionadded:: 0.9
     """
 
     def __init__(self, document_type, **kwargs):
         """
         :param document_type: The type of
          :class:`~mongoengine.EmbeddedDocument` the list will hold.
-        :param kwargs: Keyword arguments passed directly into the parent
-         :class:`~mongoengine.ListField`.
+        :param kwargs: Keyword arguments passed into the parent :class:`~mongoengine.ListField`
         """
         super().__init__(field=EmbeddedDocumentField(document_type), **kwargs)
 
@@ -996,19 +1003,11 @@ class SortedListField(ListField):
         save the whole list then other processes trying to save the whole list
         as well could overwrite changes.  The safest way to append to a list is
         to perform a push operation.
-
-    .. versionadded:: 0.4
-    .. versionchanged:: 0.6 - added reverse keyword
     """
 
-    _ordering = None
-    _order_reverse = False
-
     def __init__(self, field, **kwargs):
-        if "ordering" in kwargs.keys():
-            self._ordering = kwargs.pop("ordering")
-        if "reverse" in kwargs.keys():
-            self._order_reverse = kwargs.pop("reverse")
+        self._ordering = kwargs.pop("ordering", None)
+        self._order_reverse = kwargs.pop("reverse", False)
         super().__init__(field, **kwargs)
 
     def to_mongo(self, value, use_db_field=True, fields=None):
@@ -1055,17 +1054,13 @@ class DictField(ComplexBaseField):
 
     .. note::
         Required means it cannot be empty - as the default for DictFields is {}
-
-    .. versionadded:: 0.3
-    .. versionchanged:: 0.5 - Can now handle complex / varying types of data
     """
 
     def __init__(self, field=None, *args, **kwargs):
-        self.field = field
         self._auto_dereference = False
 
         kwargs.setdefault("default", lambda: {})
-        super().__init__(*args, **kwargs)
+        super().__init__(*args, field=field, **kwargs)
 
     def validate(self, value):
         """Make sure that a list of valid fields is being used."""
@@ -1121,8 +1116,6 @@ class MapField(DictField):
     """A field that maps a name to a specified field type. Similar to
     a DictField, except the 'value' of each item must match the specified
     field type.
-
-    .. versionadded:: 0.5
     """
 
     def __init__(self, field=None, *args, **kwargs):
@@ -1170,8 +1163,6 @@ class ReferenceField(BaseField):
             org = ReferenceField('Org', reverse_delete_rule=CASCADE)
 
         User.register_delete_rule(Org, 'owner', DENY)
-
-    .. versionchanged:: 0.5 added `reverse_delete_rule`
     """
 
     def __init__(
@@ -1179,10 +1170,12 @@ class ReferenceField(BaseField):
     ):
         """Initialises the Reference Field.
 
+        :param document_type: The type of Document that will be referenced
         :param dbref:  Store the reference as :class:`~pymongo.dbref.DBRef`
           or as the :class:`~pymongo.objectid.ObjectId`.id .
         :param reverse_delete_rule: Determines what to do when the referring
           object is deleted
+        :param kwargs: Keyword arguments passed into the parent :class:`~mongoengine.BaseField`
 
         .. note ::
             A reference to an abstract document type is always stored as a
@@ -1304,17 +1297,15 @@ class ReferenceField(BaseField):
 
 
 class CachedReferenceField(BaseField):
-    """
-    A referencefield with cache fields to purpose pseudo-joins
-
-    .. versionadded:: 0.9
-    """
+    """A referencefield with cache fields to purpose pseudo-joins"""
 
     def __init__(self, document_type, fields=None, auto_sync=True, **kwargs):
         """Initialises the Cached Reference Field.
 
+        :param document_type: The type of Document that will be referenced
         :param fields:  A list of fields to be cached in document
-        :param auto_sync: if True documents are auto updated.
+        :param auto_sync: if True documents are auto updated
+        :param kwargs: Keyword arguments passed into the parent :class:`~mongoengine.BaseField`
         """
         if fields is None:
             fields = []
@@ -1343,7 +1334,7 @@ class CachedReferenceField(BaseField):
             return None
 
         update_kwargs = {
-            "set__{}__{}".format(self.name, key): val
+            f"set__{self.name}__{key}": val
             for key, val in document._delta()[0].items()
             if key in self.fields
         }
@@ -1482,8 +1473,6 @@ class GenericReferenceField(BaseField):
           it.
 
         * You can use the choices param to limit the acceptable Document types
-
-    .. versionadded:: 0.3
     """
 
     def __init__(self, *args, **kwargs):
@@ -1619,17 +1608,76 @@ class BinaryField(BaseField):
         return super().prepare_query_value(op, self.to_mongo(value))
 
 
+class EnumField(BaseField):
+    """Enumeration Field. Values are stored underneath as is,
+    so it will only work with simple types (str, int, etc) that
+    are bson encodable
+     Example usage:
+    .. code-block:: python
+
+        class Status(Enum):
+            NEW = 'new'
+            DONE = 'done'
+
+        class ModelWithEnum(Document):
+            status = EnumField(Status, default=Status.NEW)
+
+        ModelWithEnum(status='done')
+        ModelWithEnum(status=Status.DONE)
+
+    Enum fields can be searched using enum or its value:
+    .. code-block:: python
+
+        ModelWithEnum.objects(status='new').count()
+        ModelWithEnum.objects(status=Status.NEW).count()
+
+    Note that choices cannot be set explicitly, they are derived
+    from the provided enum class.
+    """
+
+    def __init__(self, enum, **kwargs):
+        self._enum_cls = enum
+        if "choices" in kwargs:
+            raise ValueError(
+                "'choices' can't be set on EnumField, "
+                "it is implicitly set as the enum class"
+            )
+        kwargs["choices"] = list(self._enum_cls)
+        super().__init__(**kwargs)
+
+    def __set__(self, instance, value):
+        is_legal_value = value is None or isinstance(value, self._enum_cls)
+        if not is_legal_value:
+            try:
+                value = self._enum_cls(value)
+            except Exception:
+                pass
+        return super().__set__(instance, value)
+
+    def to_mongo(self, value):
+        if isinstance(value, self._enum_cls):
+            return value.value
+        return value
+
+    def validate(self, value):
+        if value and not isinstance(value, self._enum_cls):
+            try:
+                self._enum_cls(value)
+            except Exception as e:
+                self.error(str(e))
+
+    def prepare_query_value(self, op, value):
+        if value is None:
+            return value
+        return super().prepare_query_value(op, self.to_mongo(value))
+
+
 class GridFSError(Exception):
     pass
 
 
 class GridFSProxy:
-    """Proxy object to handle writing and reading of files to and from GridFS
-
-    .. versionadded:: 0.4
-    .. versionchanged:: 0.5 - added optional size param to read
-    .. versionchanged:: 0.6 - added collection name param
-    """
+    """Proxy object to handle writing and reading of files to and from GridFS"""
 
     _fs = None
 
@@ -1687,12 +1735,12 @@ class GridFSProxy:
         return self.__copy__()
 
     def __repr__(self):
-        return "<{}: {}>".format(self.__class__.__name__, self.grid_id)
+        return f"<{self.__class__.__name__}: {self.grid_id}>"
 
     def __str__(self):
         gridout = self.get()
-        filename = getattr(gridout, "filename") if gridout else "<no file>"
-        return "<{}: {} ({})>".format(self.__class__.__name__, filename, self.grid_id)
+        filename = gridout.filename if gridout else "<no file>"
+        return f"<{self.__class__.__name__}: {filename} ({self.grid_id})>"
 
     def __eq__(self, other):
         if isinstance(other, GridFSProxy):
@@ -1791,12 +1839,7 @@ class GridFSProxy:
 
 
 class FileField(BaseField):
-    """A GridFS storage field.
-
-    .. versionadded:: 0.4
-    .. versionchanged:: 0.5 added optional size param for read
-    .. versionchanged:: 0.6 added db_alias for multidb support
-    """
+    """A GridFS storage field."""
 
     proxy_class = GridFSProxy
 
@@ -1878,11 +1921,7 @@ class FileField(BaseField):
 
 
 class ImageGridFsProxy(GridFSProxy):
-    """
-    Proxy for ImageField
-
-    versionadded: 0.6
-    """
+    """Proxy for ImageField"""
 
     def put(self, file_obj, **kwargs):
         """
@@ -2016,8 +2055,6 @@ class ImageField(FileField):
     :param size: max size to store images, provided as (width, height, force)
         if larger, it will be automatically resized (ex: size=(800, 600, True))
     :param thumbnail_size: size to generate a thumbnail, provided as (width, height, force)
-
-    .. versionadded:: 0.6
     """
 
     proxy_class = ImageGridFsProxy
@@ -2042,7 +2079,7 @@ class ImageField(FileField):
 
 class SequenceField(BaseField):
     """Provides a sequential counter see:
-     http://www.mongodb.org/display/DOCS/Object+IDs#ObjectIDs-SequenceNumbers
+     https://docs.mongodb.com/manual/reference/method/ObjectId/#ObjectIDs-SequenceNumbers
 
     .. note::
 
@@ -2065,9 +2102,6 @@ class SequenceField(BaseField):
         In case the counter is defined in the abstract document, it will be
         common to all inherited documents and the default sequence name will
         be the class name of the abstract document.
-
-    .. versionadded:: 0.5
-    .. versionchanged:: 0.8 added `value_decorator`
     """
 
     _auto_gen = True
@@ -2081,7 +2115,7 @@ class SequenceField(BaseField):
         sequence_name=None,
         value_decorator=None,
         *args,
-        **kwargs
+        **kwargs,
     ):
         self.collection_name = collection_name or self.COLLECTION_NAME
         self.db_alias = db_alias or DEFAULT_CONNECTION_NAME
@@ -2096,7 +2130,7 @@ class SequenceField(BaseField):
         Generate and Increment the counter
         """
         sequence_name = self.get_sequence_name()
-        sequence_id = "{}.{}".format(sequence_name, self.name)
+        sequence_id = f"{sequence_name}.{self.name}"
         collection = get_db(alias=self.db_alias)[self.collection_name]
 
         counter = collection.find_one_and_update(
@@ -2110,7 +2144,7 @@ class SequenceField(BaseField):
     def set_next_value(self, value):
         """Helper method to set the next sequence value"""
         sequence_name = self.get_sequence_name()
-        sequence_id = "{}.{}".format(sequence_name, self.name)
+        sequence_id = f"{sequence_name}.{self.name}"
         collection = get_db(alias=self.db_alias)[self.collection_name]
         counter = collection.find_one_and_update(
             filter={"_id": sequence_id},
@@ -2127,7 +2161,7 @@ class SequenceField(BaseField):
         as it is only fixed on set.
         """
         sequence_name = self.get_sequence_name()
-        sequence_id = "{}.{}".format(sequence_name, self.name)
+        sequence_id = f"{sequence_name}.{self.name}"
         collection = get_db(alias=self.db_alias)[self.collection_name]
         data = collection.find_one({"_id": sequence_id})
 
@@ -2180,10 +2214,7 @@ class SequenceField(BaseField):
 
 
 class UUIDField(BaseField):
-    """A UUID field.
-
-    .. versionadded:: 0.6
-    """
+    """A UUID field."""
 
     _binary = None
 
@@ -2192,9 +2223,6 @@ class UUIDField(BaseField):
         Store UUID data in the database
 
         :param binary: if False store as a string.
-
-        .. versionchanged:: 0.8.0
-        .. versionchanged:: 0.6.19
         """
         self._binary = binary
         super().__init__(**kwargs)
@@ -2239,8 +2267,6 @@ class GeoPointField(BaseField):
         representing a geo point. It admits 2d indexes but not "2dsphere" indexes
         in MongoDB > 2.4 which are more natural for modeling geospatial points.
         See :ref:`geospatial-indexes`
-
-    .. versionadded:: 0.4
     """
 
     _geo_index = pymongo.GEO2D
@@ -2272,8 +2298,6 @@ class PointField(GeoJsonBaseField):
     to set the value.
 
     Requires mongodb >= 2.4
-
-    .. versionadded:: 0.8
     """
 
     _type = "Point"
@@ -2292,8 +2316,6 @@ class LineStringField(GeoJsonBaseField):
     You can either pass a dict with the full information or a list of points.
 
     Requires mongodb >= 2.4
-
-    .. versionadded:: 0.8
     """
 
     _type = "LineString"
@@ -2315,8 +2337,6 @@ class PolygonField(GeoJsonBaseField):
     holes.
 
     Requires mongodb >= 2.4
-
-    .. versionadded:: 0.8
     """
 
     _type = "Polygon"
@@ -2336,8 +2356,6 @@ class MultiPointField(GeoJsonBaseField):
     to set the value.
 
     Requires mongodb >= 2.6
-
-    .. versionadded:: 0.9
     """
 
     _type = "MultiPoint"
@@ -2357,8 +2375,6 @@ class MultiLineStringField(GeoJsonBaseField):
     You can either pass a dict with the full information or a list of points.
 
     Requires mongodb >= 2.6
-
-    .. versionadded:: 0.9
     """
 
     _type = "MultiLineString"
@@ -2385,8 +2401,6 @@ class MultiPolygonField(GeoJsonBaseField):
     of Polygons.
 
     Requires mongodb >= 2.6
-
-    .. versionadded:: 0.9
     """
 
     _type = "MultiPolygon"
@@ -2399,8 +2413,6 @@ class LazyReferenceField(BaseField):
     Instead, access will return a :class:`~mongoengine.base.LazyReference` class
     instance, allowing access to `pk` or manual dereference by using
     ``fetch()`` method.
-
-    .. versionadded:: 0.15
     """
 
     def __init__(
@@ -2409,7 +2421,7 @@ class LazyReferenceField(BaseField):
         passthrough=False,
         dbref=False,
         reverse_delete_rule=DO_NOTHING,
-        **kwargs
+        **kwargs,
     ):
         """Initialises the Reference Field.
 
@@ -2503,6 +2515,7 @@ class LazyReferenceField(BaseField):
         if not isinstance(value, (DBRef, Document, EmbeddedDocument)):
             collection = self.document_type._get_collection_name()
             value = DBRef(collection, self.document_type.id.to_python(value))
+            value = self.build_lazyref(value)
         return value
 
     def validate(self, value):
@@ -2563,8 +2576,6 @@ class GenericLazyReferenceField(GenericReferenceField):
           it.
 
         * You can use the choices param to limit the acceptable Document types
-
-    .. versionadded:: 0.15
     """
 
     def __init__(self, *args, **kwargs):

mongoengine/pymongo_support.py

@@ -2,6 +2,7 @@
 Helper functions, constants, and types to aid with PyMongo v2.7 - v3.x support.
 """
 import pymongo
+from pymongo.errors import OperationFailure
 
 _PYMONGO_37 = (3, 7)
 
@@ -10,13 +11,40 @@ PYMONGO_VERSION = tuple(pymongo.version_tuple[:2])
 IS_PYMONGO_GTE_37 = PYMONGO_VERSION >= _PYMONGO_37
 
 
-def count_documents(collection, filter):
+def count_documents(
+    collection, filter, skip=None, limit=None, hint=None, collation=None
+):
     """Pymongo>3.7 deprecates count in favour of count_documents"""
+    if limit == 0:
+        return 0  # Pymongo raises an OperationFailure if called with limit=0
+
+    kwargs = {}
+    if skip is not None:
+        kwargs["skip"] = skip
+    if limit is not None:
+        kwargs["limit"] = limit
+    if hint not in (-1, None):
+        kwargs["hint"] = hint
+    if collation is not None:
+        kwargs["collation"] = collation
+
+    # count_documents appeared in pymongo 3.7
     if IS_PYMONGO_GTE_37:
-        return collection.count_documents(filter)
-    else:
-        count = collection.find(filter).count()
-    return count
+        try:
+            return collection.count_documents(filter=filter, **kwargs)
+        except OperationFailure:
+            # OperationFailure - accounts for some operators that used to work
+            # with .count but are no longer working with count_documents (i.e $geoNear, $near, and $nearSphere)
+            # fallback to deprecated Cursor.count
+            # Keeping this should be reevaluated the day pymongo removes .count entirely
+            pass
+
+    cursor = collection.find(filter)
+    for option, option_value in kwargs.items():
+        cursor_method = getattr(cursor, option)
+        cursor = cursor_method(option_value)
+    with_limit_and_skip = "skip" in kwargs or "limit" in kwargs
+    return cursor.count(with_limit_and_skip=with_limit_and_skip)
 
 
 def list_collection_names(db, include_system_collections=False):

mongoengine/queryset/base.py

@@ -29,6 +29,7 @@ from mongoengine.errors import (
     NotUniqueError,
     OperationError,
 )
+from mongoengine.pymongo_support import count_documents
 from mongoengine.queryset import transform
 from mongoengine.queryset.field_list import QueryFieldList
 from mongoengine.queryset.visitor import Q, QNode
@@ -83,13 +84,20 @@ class BaseQuerySet:
         self._cursor_obj = None
         self._limit = None
         self._skip = None
+
         self._hint = -1  # Using -1 as None is a valid value for hint
         self._collation = None
         self._batch_size = None
-        self.only_fields = []
         self._max_time_ms = None
         self._comment = None
 
+        # Hack - As people expect cursor[5:5] to return
+        # an empty result set. It's hard to do that right, though, because the
+        # server uses limit(0) to mean 'no limit'. So we set _empty
+        # in that case and check for it when iterating. We also unset
+        # it anytime we change _limit. Inspired by how it is done in pymongo.Cursor
+        self._empty = False
+
     def __call__(self, q_obj=None, **query):
         """Filter the selected documents by calling the
         :class:`~mongoengine.queryset.QuerySet` with a query.
@@ -162,6 +170,7 @@ class BaseQuerySet:
         [<User: User object>, <User: User object>]
         """
         queryset = self.clone()
+        queryset._empty = False
 
         # Handle a slice
         if isinstance(key, slice):
@@ -169,6 +178,8 @@ class BaseQuerySet:
             queryset._skip, queryset._limit = key.start, key.stop
             if key.start and key.stop:
                 queryset._limit = key.stop - key.start
+            if queryset._limit == 0:
+                queryset._empty = True
 
             # Allow further QuerySet modifications to be performed
             return queryset
@@ -180,7 +191,6 @@ class BaseQuerySet:
                     queryset._document._from_son(
                         queryset._cursor[key],
                         _auto_dereference=self._auto_dereference,
-                        only_fields=self.only_fields,
                     )
                 )
 
@@ -190,7 +200,6 @@ class BaseQuerySet:
             return queryset._document._from_son(
                 queryset._cursor[key],
                 _auto_dereference=self._auto_dereference,
-                only_fields=self.only_fields,
             )
 
         raise TypeError("Provide a slice or an integer index")
@@ -249,8 +258,6 @@ class BaseQuerySet:
         `DocumentName.MultipleObjectsReturned` exception if multiple results
         and :class:`~mongoengine.queryset.DoesNotExist` or
         `DocumentName.DoesNotExist` if no results are found.
-
-        .. versionadded:: 0.3
         """
         queryset = self.clone()
         queryset = queryset.order_by().limit(2)
@@ -274,10 +281,7 @@ class BaseQuerySet:
         )
 
     def create(self, **kwargs):
-        """Create new object. Returns the saved object instance.
-
-        .. versionadded:: 0.4
-        """
+        """Create new object. Returns the saved object instance."""
         return self._document(**kwargs).save(force_insert=True)
 
     def first(self):
@@ -309,10 +313,6 @@ class BaseQuerySet:
 
         By default returns document instances, set ``load_bulk`` to False to
         return just ``ObjectIds``
-
-        .. versionadded:: 0.5
-        .. versionchanged:: 0.10.7
-            Add signal_kwargs argument
         """
         Document = _import_class("Document")
 
@@ -394,9 +394,36 @@ class BaseQuerySet:
             :meth:`skip` that has been applied to this cursor into account when
             getting the count
         """
-        if self._limit == 0 and with_limit_and_skip is False or self._none:
+        # mimic the fact that setting .limit(0) in pymongo sets no limit
+        # https://docs.mongodb.com/manual/reference/method/cursor.limit/#zero-value
+        if (
+            self._limit == 0
+            and with_limit_and_skip is False
+            or self._none
+            or self._empty
+        ):
             return 0
-        count = self._cursor.count(with_limit_and_skip=with_limit_and_skip)
+
+        kwargs = (
+            {"limit": self._limit, "skip": self._skip} if with_limit_and_skip else {}
+        )
+
+        if self._limit == 0:
+            # mimic the fact that historically .limit(0) sets no limit
+            kwargs.pop("limit", None)
+
+        if self._hint not in (-1, None):
+            kwargs["hint"] = self._hint
+
+        if self._collation:
+            kwargs["collation"] = self._collation
+
+        count = count_documents(
+            collection=self._cursor.collection,
+            filter=self._query,
+            **kwargs,
+        )
+
         self._cursor_obj = None
         return count
 
@@ -498,7 +525,7 @@ class BaseQuerySet:
         write_concern=None,
         read_concern=None,
         full_result=False,
-        **update
+        **update,
     ):
         """Perform an atomic update on the fields matched by the query.
 
@@ -516,8 +543,6 @@ class BaseQuerySet:
         :param update: Django-style update keyword arguments
 
         :returns the number of updated documents (unless ``full_result`` is True)
-
-        .. versionadded:: 0.2
         """
         if not update and not upsert:
             raise OperationError("No update parameters, would remove data")
@@ -569,8 +594,6 @@ class BaseQuerySet:
         :param update: Django-style update keyword arguments
 
         :returns the new or overwritten document
-
-        .. versionadded:: 0.10.2
         """
 
         atomic_update = self.update(
@@ -579,7 +602,7 @@ class BaseQuerySet:
             write_concern=write_concern,
             read_concern=read_concern,
             full_result=True,
-            **update
+            **update,
         )
 
         if atomic_update.raw_result["updatedExisting"]:
@@ -604,14 +627,13 @@ class BaseQuerySet:
         :param update: Django-style update keyword arguments
             full_result
         :returns the number of updated documents (unless ``full_result`` is True)
-        .. versionadded:: 0.2
         """
         return self.update(
             upsert=upsert,
             multi=False,
             write_concern=write_concern,
             full_result=full_result,
-            **update
+            **update,
         )
 
     def modify(
@@ -636,8 +658,6 @@ class BaseQuerySet:
         :param new: return updated rather than original document
             (default ``False``)
         :param update: Django-style update keyword arguments
-
-        .. versionadded:: 0.9
         """
 
         if remove and new:
@@ -671,7 +691,7 @@ class BaseQuerySet:
                     upsert=upsert,
                     sort=sort,
                     return_document=return_doc,
-                    **self._cursor_args
+                    **self._cursor_args,
                 )
         except pymongo.errors.DuplicateKeyError as err:
             raise NotUniqueError("Update failed (%s)" % err)
@@ -680,12 +700,10 @@ class BaseQuerySet:
 
         if full_response:
             if result["value"] is not None:
-                result["value"] = self._document._from_son(
-                    result["value"], only_fields=self.only_fields
-                )
+                result["value"] = self._document._from_son(result["value"])
         else:
             if result is not None:
-                result = self._document._from_son(result, only_fields=self.only_fields)
+                result = self._document._from_son(result)
 
         return result
 
@@ -695,8 +713,6 @@ class BaseQuerySet:
         `None` if no document exists with that id.
 
         :param object_id: the value for the id of the document to look up
-
-        .. versionchanged:: 0.6 Raises InvalidQueryError if filter has been set
         """
         queryset = self.clone()
         if not queryset._query_obj.empty:
@@ -705,22 +721,18 @@ class BaseQuerySet:
         return queryset.filter(pk=object_id).first()
 
     def in_bulk(self, object_ids):
-        """"Retrieve a set of documents by their ids.
+        """ "Retrieve a set of documents by their ids.
 
         :param object_ids: a list or tuple of ObjectId's
         :rtype: dict of ObjectId's as keys and collection-specific
                 Document subclasses as values.
-
-        .. versionadded:: 0.3
         """
         doc_map = {}
 
         docs = self._collection.find({"_id": {"$in": object_ids}}, **self._cursor_args)
         if self._scalar:
             for doc in docs:
-                doc_map[doc["_id"]] = self._get_scalar(
-                    self._document._from_son(doc, only_fields=self.only_fields)
-                )
+                doc_map[doc["_id"]] = self._get_scalar(self._document._from_son(doc))
         elif self._as_pymongo:
             for doc in docs:
                 doc_map[doc["_id"]] = doc
@@ -728,14 +740,15 @@ class BaseQuerySet:
             for doc in docs:
                 doc_map[doc["_id"]] = self._document._from_son(
                     doc,
-                    only_fields=self.only_fields,
                     _auto_dereference=self._auto_dereference,
                 )
 
         return doc_map
 
     def none(self):
-        """Helper that just returns a list"""
+        """Returns a queryset that never returns any objects and no query will be executed when accessing the results
+        inspired by django none() https://docs.djangoproject.com/en/dev/ref/models/querysets/#none
+        """
         queryset = self.clone()
         queryset._none = True
         return queryset
@@ -755,8 +768,6 @@ class BaseQuerySet:
         evaluated against if you are using more than one database.
 
         :param alias: The database alias
-
-        .. versionadded:: 0.9
         """
 
         with switch_db(self._document, alias) as cls:
@@ -789,16 +800,17 @@ class BaseQuerySet:
             "_snapshot",
             "_timeout",
             "_read_preference",
+            "_read_concern",
             "_iter",
             "_scalar",
             "_as_pymongo",
             "_limit",
             "_skip",
+            "_empty",
             "_hint",
             "_collation",
             "_auto_dereference",
             "_search_text",
-            "only_fields",
             "_max_time_ms",
             "_comment",
             "_batch_size",
@@ -817,8 +829,6 @@ class BaseQuerySet:
         """Handles dereferencing of :class:`~bson.dbref.DBRef` objects or
         :class:`~bson.object_id.ObjectId` a maximum depth in order to cut down
         the number queries to mongodb.
-
-        .. versionadded:: 0.5
         """
         # Make select related work the same for querysets
         max_depth += 1
@@ -834,6 +844,7 @@ class BaseQuerySet:
         """
         queryset = self.clone()
         queryset._limit = n
+        queryset._empty = False  # cancels the effect of empty
 
         # If a cursor object has already been created, apply the limit to it.
         if queryset._cursor_obj:
@@ -866,8 +877,6 @@ class BaseQuerySet:
 
         Hinting will not do anything if the corresponding index does not exist.
         The last hint applied to this cursor takes precedence over all others.
-
-        .. versionadded:: 0.5
         """
         queryset = self.clone()
         queryset._hint = index
@@ -929,10 +938,6 @@ class BaseQuerySet:
 
         .. note:: This is a command and won't take ordering or limit into
            account.
-
-        .. versionadded:: 0.4
-        .. versionchanged:: 0.5 - Fixed handling references
-        .. versionchanged:: 0.6 - Improved db_field refrence handling
         """
         queryset = self.clone()
 
@@ -996,12 +1001,8 @@ class BaseQuerySet:
         field filters.
 
         :param fields: fields to include
-
-        .. versionadded:: 0.3
-        .. versionchanged:: 0.5 - Added subfield support
         """
         fields = {f: QueryFieldList.ONLY for f in fields}
-        self.only_fields = list(fields.keys())
         return self.fields(True, **fields)
 
     def exclude(self, *fields):
@@ -1018,8 +1019,6 @@ class BaseQuerySet:
         field filters.
 
         :param fields: fields to exclude
-
-        .. versionadded:: 0.5
         """
         fields = {f: QueryFieldList.EXCLUDE for f in fields}
         return self.fields(**fields)
@@ -1046,8 +1045,6 @@ class BaseQuerySet:
 
         :param kwargs: A set of keyword arguments identifying what to
             include, exclude, or slice.
-
-        .. versionadded:: 0.5
         """
 
         # Check for an operator and transform to mongo-style if there is
@@ -1089,8 +1086,6 @@ class BaseQuerySet:
         .exclude(). ::
 
             post = BlogPost.objects.exclude('comments').all_fields()
-
-        .. versionadded:: 0.5
         """
         queryset = self.clone()
         queryset._loaded_fields = QueryFieldList(
@@ -1163,9 +1158,6 @@ class BaseQuerySet:
         """Enable or disable snapshot mode when querying.
 
         :param enabled: whether or not snapshot mode is enabled
-
-        ..versionchanged:: 0.5 - made chainable
-        .. deprecated:: Ignored with PyMongo 3+
         """
         msg = "snapshot is deprecated as it has no impact when using PyMongo 3+."
         warnings.warn(msg, DeprecationWarning)
@@ -1177,8 +1169,6 @@ class BaseQuerySet:
         """Enable or disable the default mongod timeout when querying. (no_cursor_timeout option)
 
         :param enabled: whether or not the timeout is used
-
-        ..versionchanged:: 0.5 - made chainable
         """
         queryset = self.clone()
         queryset._timeout = enabled
@@ -1203,7 +1193,7 @@ class BaseQuerySet:
             preference.
         """
         if read_concern is not None and not isinstance(read_concern, Mapping):
-            raise TypeError("%r is not a valid read concern." % (read_concern,))
+            raise TypeError(f"{read_concern!r} is not a valid read concern.")
 
         queryset = self.clone()
         queryset._read_concern = (
@@ -1266,10 +1256,7 @@ class BaseQuerySet:
     def from_json(self, json_data):
         """Converts json data to unsaved objects"""
         son_data = json_util.loads(json_data)
-        return [
-            self._document._from_son(data, only_fields=self.only_fields)
-            for data in son_data
-        ]
+        return [self._document._from_son(data) for data in son_data]
 
     def aggregate(self, pipeline, *suppl_pipeline, **kwargs):
         """Perform a aggregate function based in your queryset params
@@ -1280,7 +1267,6 @@ class BaseQuerySet:
             parameter will be removed shortly
         :param kwargs: (optional) kwargs dictionary to be passed to pymongo's aggregate call
             See https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.aggregate
-        .. versionadded:: 0.9
         """
         using_deprecated_interface = isinstance(pipeline, dict) or bool(suppl_pipeline)
         user_pipeline = [pipeline] if isinstance(pipeline, dict) else list(pipeline)
@@ -1311,10 +1297,11 @@ class BaseQuerySet:
         final_pipeline = initial_pipeline + user_pipeline
 
         collection = self._collection
-        if self._read_preference is not None:
+        if self._read_preference is not None or self._read_concern is not None:
             collection = self._collection.with_options(
-                read_preference=self._read_preference
+                read_preference=self._read_preference, read_concern=self._read_concern
             )
+
         return collection.aggregate(final_pipeline, cursor={}, **kwargs)
 
     # JS functionality
@@ -1351,12 +1338,6 @@ class BaseQuerySet:
             Map/Reduce changed in server version **>= 1.7.4**. The PyMongo
             :meth:`~pymongo.collection.Collection.map_reduce` helper requires
             PyMongo version **>= 1.11**.
-
-        .. versionchanged:: 0.5
-           - removed ``keep_temp`` keyword argument, which was only relevant
-             for MongoDB server versions older than 1.7.4
-
-        .. versionadded:: 0.3
         """
         queryset = self.clone()
 
@@ -1493,8 +1474,6 @@ class BaseQuerySet:
         .. note:: When using this mode of query, the database will call your
                   function, or evaluate your predicate clause, for each object
                   in the collection.
-
-        .. versionadded:: 0.5
         """
         queryset = self.clone()
         where_clause = queryset._sub_js_fields(where_clause)
@@ -1571,9 +1550,6 @@ class BaseQuerySet:
         :param field: the field to use
         :param normalize: normalize the results so they add to 1.0
         :param map_reduce: Use map_reduce over exec_js
-
-        .. versionchanged:: 0.5 defaults to map_reduce and can handle embedded
-                            document lookups
         """
         if map_reduce:
             return self._item_frequencies_map_reduce(field, normalize=normalize)
@@ -1582,9 +1558,8 @@ class BaseQuerySet:
     # Iterator helpers
 
     def __next__(self):
-        """Wrap the result in a :class:`~mongoengine.Document` object.
-        """
-        if self._limit == 0 or self._none:
+        """Wrap the result in a :class:`~mongoengine.Document` object."""
+        if self._none or self._empty:
             raise StopIteration
 
         raw_doc = next(self._cursor)
@@ -1595,7 +1570,6 @@ class BaseQuerySet:
         doc = self._document._from_son(
             raw_doc,
             _auto_dereference=self._auto_dereference,
-            only_fields=self.only_fields,
         )
 
         if self._scalar:
@@ -1603,13 +1577,8 @@ class BaseQuerySet:
 
         return doc
 
-    next = __next__  # For Python2 support
-
     def rewind(self):
-        """Rewind the cursor to its unevaluated state.
-
-        .. versionadded:: 0.3
-        """
+        """Rewind the cursor to its unevaluated state."""
         self._iter = False
         self._cursor.rewind()
 

mongoengine/queryset/queryset.py

@@ -144,15 +144,13 @@ class QuerySet(BaseQuerySet):
             return super().count(with_limit_and_skip)
 
         if self._len is None:
+            # cache the length
             self._len = super().count(with_limit_and_skip)
 
         return self._len
 
     def no_cache(self):
-        """Convert to a non-caching queryset
-
-        .. versionadded:: 0.8.3 Convert to non caching queryset
-        """
+        """Convert to a non-caching queryset"""
         if self._result_cache is not None:
             raise OperationError("QuerySet already cached")
 
@@ -163,17 +161,11 @@ class QuerySetNoCache(BaseQuerySet):
     """A non caching QuerySet"""
 
     def cache(self):
-        """Convert to a caching queryset
-
-        .. versionadded:: 0.8.3 Convert to caching queryset
-        """
+        """Convert to a caching queryset"""
         return self._clone_into(QuerySet(self._document, self._collection))
 
     def __repr__(self):
-        """Provides the string representation of the QuerySet
-
-        .. versionchanged:: 0.6.13 Now doesnt modify the cursor
-        """
+        """Provides the string representation of the QuerySet"""
         if self._iter:
             return ".. queryset mid-iteration .."
 

mongoengine/queryset/visitor.py

@@ -7,18 +7,20 @@ from mongoengine.queryset import transform
 __all__ = ("Q", "QNode")
 
 
+def warn_empty_is_deprecated():
+    msg = "'empty' property is deprecated in favour of using 'not bool(filter)'"
+    warnings.warn(msg, DeprecationWarning, stacklevel=2)
+
+
 class QNodeVisitor:
-    """Base visitor class for visiting Q-object nodes in a query tree.
-    """
+    """Base visitor class for visiting Q-object nodes in a query tree."""
 
     def visit_combination(self, combination):
-        """Called by QCombination objects.
-        """
+        """Called by QCombination objects."""
         return combination
 
     def visit_query(self, query):
-        """Called by (New)Q objects.
-        """
+        """Called by (New)Q objects."""
         return query
 
 
@@ -44,8 +46,7 @@ class SimplificationVisitor(QNodeVisitor):
         return combination
 
     def _query_conjunction(self, queries):
-        """Merges query dicts - effectively &ing them together.
-        """
+        """Merges query dicts - effectively &ing them together."""
         query_ops = set()
         combined_query = {}
         for query in queries:
@@ -98,19 +99,18 @@ class QNode:
         object.
         """
         # If the other Q() is empty, ignore it and just use `self`.
-        if getattr(other, "empty", True):
+        if not bool(other):
             return self
 
         # Or if this Q is empty, ignore it and just use `other`.
-        if self.empty:
+        if not bool(self):
             return other
 
         return QCombination(operation, [self, other])
 
     @property
     def empty(self):
-        msg = "'empty' property is deprecated in favour of using 'not bool(filter)'"
-        warnings.warn(msg, DeprecationWarning)
+        warn_empty_is_deprecated()
         return False
 
     def __or__(self, other):
@@ -152,8 +152,7 @@ class QCombination(QNode):
 
     @property
     def empty(self):
-        msg = "'empty' property is deprecated in favour of using 'not bool(filter)'"
-        warnings.warn(msg, DeprecationWarning)
+        warn_empty_is_deprecated()
         return not bool(self.children)
 
     def __eq__(self, other):
@@ -186,4 +185,5 @@ class Q(QNode):
 
     @property
     def empty(self):
+        warn_empty_is_deprecated()
         return not bool(self.query)

requirements.txt

@@ -1,3 +0,0 @@
-pymongo>=3.4
-Sphinx==1.5.5
-sphinx-rtd-theme==0.2.4

setup.cfg

@@ -1,5 +1,5 @@
 [flake8]
-ignore=E501,F401,F403,F405,I201,I202,W504, W605, W503
+ignore=E501,F403,F405,I201,I202,W504,W605,W503,B007
 exclude=build,dist,docs,venv,venv3,.tox,.eggs,tests
 max-complexity=47
 application-import-names=mongoengine,tests

setup.py

@@ -7,7 +7,7 @@ from setuptools.command.test import test as TestCommand
 
 # Hack to silence atexit traceback in newer python versions
 try:
-    import multiprocessing
+    import multiprocessing  # noqa: F401
 except ImportError:
     pass
 
@@ -115,7 +115,7 @@ extra_opts = {
         "pytest-cov",
         "coverage<5.0",  # recent coverage switched to sqlite format for the .coverage file which isn't handled properly by coveralls
         "blinker",
-        "Pillow>=2.0.0, <7.0.0",  # 7.0.0 dropped Python2 support
+        "Pillow>=7.0.0",
     ],
 }
 

tests/document/test_class_methods.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 
 from mongoengine import *
@@ -27,16 +26,14 @@ class TestClassMethods(unittest.TestCase):
             self.db.drop_collection(collection)
 
     def test_definition(self):
-        """Ensure that document may be defined using fields.
-        """
+        """Ensure that document may be defined using fields."""
         assert ["_cls", "age", "id", "name"] == sorted(self.Person._fields.keys())
         assert ["IntField", "ObjectIdField", "StringField", "StringField"] == sorted(
-            [x.__class__.__name__ for x in self.Person._fields.values()]
+            x.__class__.__name__ for x in self.Person._fields.values()
         )
 
     def test_get_db(self):
-        """Ensure that get_db returns the expected db.
-        """
+        """Ensure that get_db returns the expected db."""
         db = self.Person._get_db()
         assert self.db == db
 
@@ -48,15 +45,13 @@ class TestClassMethods(unittest.TestCase):
         assert collection_name == self.Person._get_collection_name()
 
     def test_get_collection(self):
-        """Ensure that get_collection returns the expected collection.
-        """
+        """Ensure that get_collection returns the expected collection."""
         collection_name = "person"
         collection = self.Person._get_collection()
         assert self.db[collection_name] == collection
 
     def test_drop_collection(self):
-        """Ensure that the collection may be dropped from the database.
-        """
+        """Ensure that the collection may be dropped from the database."""
         collection_name = "person"
         self.Person(name="Test").save()
         assert collection_name in list_collection_names(self.db)
@@ -78,7 +73,7 @@ class TestClassMethods(unittest.TestCase):
         assert self.Person._meta["delete_rules"] == {(Job, "employee"): NULLIFY}
 
     def test_compare_indexes(self):
-        """ Ensure that the indexes are properly created and that
+        """Ensure that the indexes are properly created and that
         compare_indexes identifies the missing/extra indexes
         """
 
@@ -111,7 +106,7 @@ class TestClassMethods(unittest.TestCase):
         }
 
     def test_compare_indexes_inheritance(self):
-        """ Ensure that the indexes are properly created and that
+        """Ensure that the indexes are properly created and that
         compare_indexes identifies the missing/extra indexes for subclassed
         documents (_cls included)
         """
@@ -151,7 +146,7 @@ class TestClassMethods(unittest.TestCase):
         }
 
     def test_compare_indexes_multiple_subclasses(self):
-        """ Ensure that compare_indexes behaves correctly if called from a
+        """Ensure that compare_indexes behaves correctly if called from a
         class, which base class has multiple subclasses
         """
 
@@ -204,7 +199,7 @@ class TestClassMethods(unittest.TestCase):
         assert actual == expected
 
     def test_list_indexes_inheritance(self):
-        """ ensure that all of the indexes are listed regardless of the super-
+        """ensure that all of the indexes are listed regardless of the super-
         or sub-class that we call it from
         """
 
@@ -236,7 +231,7 @@ class TestClassMethods(unittest.TestCase):
         assert BlogPost.list_indexes() == [
             [("_cls", 1), ("author", 1), ("tags", 1)],
             [("_cls", 1), ("author", 1), ("tags", 1), ("extra_text", 1)],
-            [(u"_id", 1)],
+            [("_id", 1)],
             [("_cls", 1)],
         ]
 
@@ -261,8 +256,7 @@ class TestClassMethods(unittest.TestCase):
         assert Vaccine._meta["delete_rules"][(Cat, "vaccine_made")] == PULL
 
     def test_collection_naming(self):
-        """Ensure that a collection with a specified name may be used.
-        """
+        """Ensure that a collection with a specified name may be used."""
 
         class DefaultNamingTest(Document):
             pass
@@ -294,7 +288,7 @@ class TestClassMethods(unittest.TestCase):
         assert "wibble" == InheritedAbstractNamingTest._get_collection_name()
 
         # Mixin tests
-        class BaseMixin(object):
+        class BaseMixin:
             meta = {"collection": lambda c: c.__name__.lower()}
 
         class OldMixinNamingConvention(Document, BaseMixin):
@@ -305,7 +299,7 @@ class TestClassMethods(unittest.TestCase):
             == OldMixinNamingConvention._get_collection_name()
         )
 
-        class BaseMixin(object):
+        class BaseMixin:
             meta = {"collection": lambda c: c.__name__.lower()}
 
         class BaseDocument(Document, BaseMixin):
@@ -317,8 +311,7 @@ class TestClassMethods(unittest.TestCase):
         assert "basedocument" == MyDocument._get_collection_name()
 
     def test_custom_collection_name_operations(self):
-        """Ensure that a collection with a specified name is used as expected.
-        """
+        """Ensure that a collection with a specified name is used as expected."""
         collection_name = "personCollTest"
 
         class Person(Document):
@@ -338,8 +331,7 @@ class TestClassMethods(unittest.TestCase):
         assert collection_name not in list_collection_names(self.db)
 
     def test_collection_name_and_primary(self):
-        """Ensure that a collection with a specified name may be used.
-        """
+        """Ensure that a collection with a specified name may be used."""
 
         class Person(Document):
             name = StringField(primary_key=True)

tests/document/test_delta.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 
 from bson import SON
@@ -9,7 +8,7 @@ from tests.utils import MongoDBTestCase
 
 class TestDelta(MongoDBTestCase):
     def setUp(self):
-        super(TestDelta, self).setUp()
+        super().setUp()
 
         class Person(Document):
             name = StringField()
@@ -29,7 +28,8 @@ class TestDelta(MongoDBTestCase):
         self.delta(Document)
         self.delta(DynamicDocument)
 
-    def delta(self, DocClass):
+    @staticmethod
+    def delta(DocClass):
         class Doc(DocClass):
             string_field = StringField()
             int_field = IntField()
@@ -428,13 +428,20 @@ class TestDelta(MongoDBTestCase):
         assert doc.dict_field == {"hello": "world"}
         assert doc.list_field == ["1", 2, {"hello": "world"}]
 
-    def test_delta_recursive_db_field(self):
+    def test_delta_recursive_db_field_on_doc_and_embeddeddoc(self):
         self.delta_recursive_db_field(Document, EmbeddedDocument)
+
+    def test_delta_recursive_db_field_on_doc_and_dynamicembeddeddoc(self):
         self.delta_recursive_db_field(Document, DynamicEmbeddedDocument)
+
+    def test_delta_recursive_db_field_on_dynamicdoc_and_embeddeddoc(self):
         self.delta_recursive_db_field(DynamicDocument, EmbeddedDocument)
+
+    def test_delta_recursive_db_field_on_dynamicdoc_and_dynamicembeddeddoc(self):
         self.delta_recursive_db_field(DynamicDocument, DynamicEmbeddedDocument)
 
-    def delta_recursive_db_field(self, DocClass, EmbeddedClass):
+    @staticmethod
+    def delta_recursive_db_field(DocClass, EmbeddedClass):
         class Embedded(EmbeddedClass):
             string_field = StringField(db_field="db_string_field")
             int_field = IntField(db_field="db_int_field")
@@ -487,6 +494,7 @@ class TestDelta(MongoDBTestCase):
         doc = doc.reload(10)
         assert doc.embedded_field.dict_field == {}
 
+        assert doc._get_changed_fields() == []
         doc.embedded_field.list_field = []
         assert doc._get_changed_fields() == ["db_embedded_field.db_list_field"]
         assert doc.embedded_field._delta() == ({}, {"db_list_field": 1})
@@ -537,6 +545,7 @@ class TestDelta(MongoDBTestCase):
             {},
         )
         doc.save()
+        assert doc._get_changed_fields() == []
         doc = doc.reload(10)
 
         assert doc.embedded_field.list_field[0] == "1"
@@ -634,6 +643,10 @@ class TestDelta(MongoDBTestCase):
         doc.save()
         doc = doc.reload(10)
 
+        assert doc._delta() == (
+            {},
+            {},
+        )
         del doc.embedded_field.list_field[2].list_field
         assert doc._delta() == (
             {},
@@ -732,12 +745,12 @@ class TestDelta(MongoDBTestCase):
         assert organization._get_changed_fields() == []
 
         updates, removals = organization._delta()
-        assert {} == removals
-        assert {} == updates
+        assert removals == {}
+        assert updates == {}
 
         organization.employees.append(person)
         updates, removals = organization._delta()
-        assert {} == removals
+        assert removals == {}
         assert "employees" in updates
 
     def test_delta_with_dbref_false(self):
@@ -749,12 +762,12 @@ class TestDelta(MongoDBTestCase):
         assert organization._get_changed_fields() == []
 
         updates, removals = organization._delta()
-        assert {} == removals
-        assert {} == updates
+        assert removals == {}
+        assert updates == {}
 
         organization.employees.append(person)
         updates, removals = organization._delta()
-        assert {} == removals
+        assert removals == {}
         assert "employees" in updates
 
     def test_nested_nested_fields_mark_as_changed(self):
@@ -767,19 +780,46 @@ class TestDelta(MongoDBTestCase):
 
         MyDoc.drop_collection()
 
-        mydoc = MyDoc(
-            name="testcase1", subs={"a": {"b": EmbeddedDoc(name="foo")}}
-        ).save()
+        MyDoc(name="testcase1", subs={"a": {"b": EmbeddedDoc(name="foo")}}).save()
 
         mydoc = MyDoc.objects.first()
         subdoc = mydoc.subs["a"]["b"]
         subdoc.name = "bar"
 
-        assert ["name"] == subdoc._get_changed_fields()
-        assert ["subs.a.b.name"] == mydoc._get_changed_fields()
+        assert subdoc._get_changed_fields() == ["name"]
+        assert mydoc._get_changed_fields() == ["subs.a.b.name"]
 
         mydoc._clear_changed_fields()
-        assert [] == mydoc._get_changed_fields()
+        assert mydoc._get_changed_fields() == []
+
+    def test_nested_nested_fields_db_field_set__gets_mark_as_changed_and_cleaned(self):
+        class EmbeddedDoc(EmbeddedDocument):
+            name = StringField(db_field="db_name")
+
+        class MyDoc(Document):
+            embed = EmbeddedDocumentField(EmbeddedDoc, db_field="db_embed")
+            name = StringField(db_field="db_name")
+
+        MyDoc.drop_collection()
+
+        MyDoc(name="testcase1", embed=EmbeddedDoc(name="foo")).save()
+
+        mydoc = MyDoc.objects.first()
+        mydoc.embed.name = "foo1"
+
+        assert mydoc.embed._get_changed_fields() == ["db_name"]
+        assert mydoc._get_changed_fields() == ["db_embed.db_name"]
+
+        mydoc = MyDoc.objects.first()
+        embed = EmbeddedDoc(name="foo2")
+        embed.name = "bar"
+        mydoc.embed = embed
+
+        assert embed._get_changed_fields() == ["db_name"]
+        assert mydoc._get_changed_fields() == ["db_embed"]
+
+        mydoc._clear_changed_fields()
+        assert mydoc._get_changed_fields() == []
 
     def test_lower_level_mark_as_changed(self):
         class EmbeddedDoc(EmbeddedDocument):
@@ -794,17 +834,17 @@ class TestDelta(MongoDBTestCase):
 
         mydoc = MyDoc.objects.first()
         mydoc.subs["a"] = EmbeddedDoc()
-        assert ["subs.a"] == mydoc._get_changed_fields()
+        assert mydoc._get_changed_fields() == ["subs.a"]
 
         subdoc = mydoc.subs["a"]
         subdoc.name = "bar"
 
-        assert ["name"] == subdoc._get_changed_fields()
-        assert ["subs.a"] == mydoc._get_changed_fields()
+        assert subdoc._get_changed_fields() == ["name"]
+        assert mydoc._get_changed_fields() == ["subs.a"]
         mydoc.save()
 
         mydoc._clear_changed_fields()
-        assert [] == mydoc._get_changed_fields()
+        assert mydoc._get_changed_fields() == []
 
     def test_upper_level_mark_as_changed(self):
         class EmbeddedDoc(EmbeddedDocument):
@@ -821,15 +861,15 @@ class TestDelta(MongoDBTestCase):
         subdoc = mydoc.subs["a"]
         subdoc.name = "bar"
 
-        assert ["name"] == subdoc._get_changed_fields()
-        assert ["subs.a.name"] == mydoc._get_changed_fields()
+        assert subdoc._get_changed_fields() == ["name"]
+        assert mydoc._get_changed_fields() == ["subs.a.name"]
 
         mydoc.subs["a"] = EmbeddedDoc()
-        assert ["subs.a"] == mydoc._get_changed_fields()
+        assert mydoc._get_changed_fields() == ["subs.a"]
         mydoc.save()
 
         mydoc._clear_changed_fields()
-        assert [] == mydoc._get_changed_fields()
+        assert mydoc._get_changed_fields() == []
 
     def test_referenced_object_changed_attributes(self):
         """Ensures that when you save a new reference to a field, the referenced object isn't altered"""

tests/document/test_dynamic.py

@@ -10,7 +10,7 @@ __all__ = ("TestDynamicDocument",)
 
 class TestDynamicDocument(MongoDBTestCase):
     def setUp(self):
-        super(TestDynamicDocument, self).setUp()
+        super().setUp()
 
         class Person(DynamicDocument):
             name = StringField()
@@ -37,6 +37,19 @@ class TestDynamicDocument(MongoDBTestCase):
         # Confirm no changes to self.Person
         assert not hasattr(self.Person, "age")
 
+    def test_dynamic_document_parse_values_in_constructor_like_document_do(self):
+        class ProductDynamicDocument(DynamicDocument):
+            title = StringField()
+            price = FloatField()
+
+        class ProductDocument(Document):
+            title = StringField()
+            price = FloatField()
+
+        product = ProductDocument(title="Blabla", price="12.5")
+        dyn_product = ProductDynamicDocument(title="Blabla", price="12.5")
+        assert product.price == dyn_product.price == 12.5
+
     def test_change_scope_of_variable(self):
         """Test changing the scope of a dynamic field has no adverse effects"""
         p = self.Person()
@@ -105,17 +118,17 @@ class TestDynamicDocument(MongoDBTestCase):
         p.save()
 
         raw_p = Person.objects.as_pymongo().get(id=p.id)
-        assert raw_p == {"_cls": u"Person", "_id": p.id, "name": u"Dean"}
+        assert raw_p == {"_cls": "Person", "_id": p.id, "name": "Dean"}
 
         p.name = "OldDean"
         p.newattr = "garbage"
         p.save()
         raw_p = Person.objects.as_pymongo().get(id=p.id)
         assert raw_p == {
-            "_cls": u"Person",
+            "_cls": "Person",
             "_id": p.id,
             "name": "OldDean",
-            "newattr": u"garbage",
+            "newattr": "garbage",
         }
 
     def test_fields_containing_underscore(self):
@@ -131,14 +144,14 @@ class TestDynamicDocument(MongoDBTestCase):
         p.save()
 
         raw_p = WeirdPerson.objects.as_pymongo().get(id=p.id)
-        assert raw_p == {"_id": p.id, "_name": u"Dean", "name": u"Dean"}
+        assert raw_p == {"_id": p.id, "_name": "Dean", "name": "Dean"}
 
         p.name = "OldDean"
         p._name = "NewDean"
         p._newattr1 = "garbage"  # Unknown fields won't be added
         p.save()
         raw_p = WeirdPerson.objects.as_pymongo().get(id=p.id)
-        assert raw_p == {"_id": p.id, "_name": u"NewDean", "name": u"OldDean"}
+        assert raw_p == {"_id": p.id, "_name": "NewDean", "name": "OldDean"}
 
     def test_dynamic_document_queries(self):
         """Ensure we can query dynamic fields"""

tests/document/test_indexes.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 from datetime import datetime
 
@@ -172,8 +171,7 @@ class TestIndexes(unittest.TestCase):
         assert MyDoc._meta["index_specs"] == [{"fields": [("keywords", 1)]}]
 
     def test_embedded_document_index_meta(self):
-        """Ensure that embedded document indexes are created explicitly
-        """
+        """Ensure that embedded document indexes are created explicitly"""
 
         class Rank(EmbeddedDocument):
             title = StringField(required=True)
@@ -195,8 +193,7 @@ class TestIndexes(unittest.TestCase):
         assert [("rank.title", 1)] in info
 
     def test_explicit_geo2d_index(self):
-        """Ensure that geo2d indexes work when created via meta[indexes]
-        """
+        """Ensure that geo2d indexes work when created via meta[indexes]"""
 
         class Place(Document):
             location = DictField()
@@ -210,8 +207,7 @@ class TestIndexes(unittest.TestCase):
         assert [("location.point", "2d")] in info
 
     def test_explicit_geo2d_index_embedded(self):
-        """Ensure that geo2d indexes work when created via meta[indexes]
-        """
+        """Ensure that geo2d indexes work when created via meta[indexes]"""
 
         class EmbeddedLocation(EmbeddedDocument):
             location = DictField()
@@ -230,8 +226,7 @@ class TestIndexes(unittest.TestCase):
         assert [("current.location.point", "2d")] in info
 
     def test_explicit_geosphere_index(self):
-        """Ensure that geosphere indexes work when created via meta[indexes]
-        """
+        """Ensure that geosphere indexes work when created via meta[indexes]"""
 
         class Place(Document):
             location = DictField()
@@ -247,8 +242,7 @@ class TestIndexes(unittest.TestCase):
         assert [("location.point", "2dsphere")] in info
 
     def test_explicit_geohaystack_index(self):
-        """Ensure that geohaystack indexes work when created via meta[indexes]
-        """
+        """Ensure that geohaystack indexes work when created via meta[indexes]"""
         pytest.skip(
             "GeoHaystack index creation is not supported for now"
             "from meta, as it requires a bucketSize parameter."
@@ -269,8 +263,7 @@ class TestIndexes(unittest.TestCase):
         assert [("location.point", "geoHaystack")] in info
 
     def test_create_geohaystack_index(self):
-        """Ensure that geohaystack indexes can be created
-        """
+        """Ensure that geohaystack indexes can be created"""
 
         class Place(Document):
             location = DictField()
@@ -365,8 +358,7 @@ class TestIndexes(unittest.TestCase):
         assert sorted(info.keys()) == ["_cls_1_user_guid_1", "_id_"]
 
     def test_embedded_document_index(self):
-        """Tests settings an index on an embedded document
-        """
+        """Tests settings an index on an embedded document"""
 
         class Date(EmbeddedDocument):
             year = IntField(db_field="yr")
@@ -383,8 +375,7 @@ class TestIndexes(unittest.TestCase):
         assert sorted(info.keys()) == ["_id_", "date.yr_-1"]
 
     def test_list_embedded_document_index(self):
-        """Ensure list embedded documents can be indexed
-        """
+        """Ensure list embedded documents can be indexed"""
 
         class Tag(EmbeddedDocument):
             name = StringField(db_field="tag")
@@ -420,8 +411,7 @@ class TestIndexes(unittest.TestCase):
         assert sorted(info.keys()) == ["_cls_1", "_id_"]
 
     def test_covered_index(self):
-        """Ensure that covered indexes can be used
-        """
+        """Ensure that covered indexes can be used"""
 
         class Test(Document):
             a = IntField()
@@ -551,15 +541,15 @@ class TestIndexes(unittest.TestCase):
         assert 5 == query_result.count()
 
         incorrect_collation = {"arndom": "wrdo"}
-        with pytest.raises(OperationFailure):
+        with pytest.raises(OperationFailure) as exc_info:
             BlogPost.objects.collation(incorrect_collation).count()
+        assert "Missing expected field" in str(exc_info.value)
 
         query_result = BlogPost.objects.collation({}).order_by("name")
         assert [x.name for x in query_result] == sorted(names)
 
     def test_unique(self):
-        """Ensure that uniqueness constraints are applied to fields.
-        """
+        """Ensure that uniqueness constraints are applied to fields."""
 
         class BlogPost(Document):
             title = StringField()
@@ -607,8 +597,7 @@ class TestIndexes(unittest.TestCase):
         )
 
     def test_unique_with(self):
-        """Ensure that unique_with constraints are applied to fields.
-        """
+        """Ensure that unique_with constraints are applied to fields."""
 
         class Date(EmbeddedDocument):
             year = IntField(db_field="yr")
@@ -633,8 +622,7 @@ class TestIndexes(unittest.TestCase):
             post3.save()
 
     def test_unique_embedded_document(self):
-        """Ensure that uniqueness constraints are applied to fields on embedded documents.
-        """
+        """Ensure that uniqueness constraints are applied to fields on embedded documents."""
 
         class SubDocument(EmbeddedDocument):
             year = IntField(db_field="yr")

tests/document/test_inheritance.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 import warnings
 
@@ -46,8 +45,7 @@ class TestInheritance(MongoDBTestCase):
         test_doc.delete()
 
     def test_superclasses(self):
-        """Ensure that the correct list of superclasses is assembled.
-        """
+        """Ensure that the correct list of superclasses is assembled."""
 
         class Animal(Document):
             meta = {"allow_inheritance": True}
@@ -217,8 +215,7 @@ class TestInheritance(MongoDBTestCase):
         assert Pike._subclasses == ("Animal.Fish.Pike",)
 
     def test_inheritance_meta_data(self):
-        """Ensure that document may inherit fields from a superclass document.
-        """
+        """Ensure that document may inherit fields from a superclass document."""
 
         class Person(Document):
             name = StringField()
@@ -235,8 +232,7 @@ class TestInheritance(MongoDBTestCase):
         assert Employee._get_collection_name() == Person._get_collection_name()
 
     def test_inheritance_to_mongo_keys(self):
-        """Ensure that document may inherit fields from a superclass document.
-        """
+        """Ensure that document may inherit fields from a superclass document."""
 
         class Person(Document):
             name = StringField()
@@ -260,7 +256,7 @@ class TestInheritance(MongoDBTestCase):
         assert Employee._get_collection_name() == Person._get_collection_name()
 
     def test_indexes_and_multiple_inheritance(self):
-        """ Ensure that all of the indexes are created for a document with
+        """Ensure that all of the indexes are created for a document with
         multiple inheritance.
         """
 
@@ -284,14 +280,11 @@ class TestInheritance(MongoDBTestCase):
         C.ensure_indexes()
 
         assert sorted(
-            [idx["key"] for idx in C._get_collection().index_information().values()]
-        ) == sorted(
-            [[(u"_cls", 1), (u"b", 1)], [(u"_id", 1)], [(u"_cls", 1), (u"a", 1)]]
-        )
+            idx["key"] for idx in C._get_collection().index_information().values()
+        ) == sorted([[("_cls", 1), ("b", 1)], [("_id", 1)], [("_cls", 1), ("a", 1)]])
 
     def test_polymorphic_queries(self):
-        """Ensure that the correct subclasses are returned from a query
-        """
+        """Ensure that the correct subclasses are returned from a query"""
 
         class Animal(Document):
             meta = {"allow_inheritance": True}
@@ -348,8 +341,7 @@ class TestInheritance(MongoDBTestCase):
         assert "_cls" not in obj
 
     def test_cant_turn_off_inheritance_on_subclass(self):
-        """Ensure if inheritance is on in a subclass you cant turn it off.
-        """
+        """Ensure if inheritance is on in a subclass you cant turn it off."""
 
         class Animal(Document):
             name = StringField()
@@ -475,7 +467,7 @@ class TestInheritance(MongoDBTestCase):
         assert city.pk is None
         # TODO: expected error? Shouldn't we create a new error type?
         with pytest.raises(KeyError):
-            setattr(city, "pk", 1)
+            city.pk = 1
 
     def test_allow_inheritance_embedded_document(self):
         """Ensure embedded documents respect inheritance."""
@@ -499,8 +491,7 @@ class TestInheritance(MongoDBTestCase):
         assert "_cls" in doc.to_mongo()
 
     def test_document_inheritance(self):
-        """Ensure mutliple inheritance of abstract documents
-        """
+        """Ensure mutliple inheritance of abstract documents"""
 
         class DateCreatedDocument(Document):
             meta = {"allow_inheritance": True, "abstract": True}
@@ -508,13 +499,8 @@ class TestInheritance(MongoDBTestCase):
         class DateUpdatedDocument(Document):
             meta = {"allow_inheritance": True, "abstract": True}
 
-        try:
-
-            class MyDocument(DateCreatedDocument, DateUpdatedDocument):
-                pass
-
-        except Exception:
-            assert False, "Couldn't create MyDocument class"
+        class MyDocument(DateCreatedDocument, DateUpdatedDocument):
+            pass
 
     def test_abstract_documents(self):
         """Ensure that a document superclass can be marked as abstract

tests/document/test_instance.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import os
 import pickle
 import unittest
@@ -161,8 +160,7 @@ class TestDocumentInstance(MongoDBTestCase):
         Log.objects
 
     def test_repr(self):
-        """Ensure that unicode representation works
-        """
+        """Ensure that unicode representation works"""
 
         class Article(Document):
             title = StringField()
@@ -170,7 +168,7 @@ class TestDocumentInstance(MongoDBTestCase):
             def __unicode__(self):
                 return self.title
 
-        doc = Article(title=u"привет мир")
+        doc = Article(title="привет мир")
 
         assert "<Article: привет мир>" == repr(doc)
 
@@ -183,12 +181,12 @@ class TestDocumentInstance(MongoDBTestCase):
             def __str__(self):
                 return None
 
-        doc = Article(title=u"привет мир")
+        doc = Article(title="привет мир")
         assert "<Article: None>" == repr(doc)
 
     def test_queryset_resurrects_dropped_collection(self):
         self.Person.drop_collection()
-        assert [] == list(self.Person.objects())
+        assert list(self.Person.objects()) == []
 
         # Ensure works correctly with inhertited classes
         class Actor(self.Person):
@@ -196,7 +194,7 @@ class TestDocumentInstance(MongoDBTestCase):
 
         Actor.objects()
         self.Person.drop_collection()
-        assert [] == list(Actor.objects())
+        assert list(Actor.objects()) == []
 
     def test_polymorphic_references(self):
         """Ensure that the correct subclasses are returned from a query
@@ -501,7 +499,7 @@ class TestDocumentInstance(MongoDBTestCase):
         doc.reload()
         Animal.drop_collection()
 
-    def test_update_shard_key_routing(self):
+    def test_save_update_shard_key_routing(self):
         """Ensures updating a doc with a specified shard_key includes it in
         the query.
         """
@@ -523,9 +521,32 @@ class TestDocumentInstance(MongoDBTestCase):
             query_op = q.db.system.profile.find({"ns": "mongoenginetest.animal"})[0]
             assert query_op["op"] == "update"
             if mongo_db <= MONGODB_34:
-                assert set(query_op["query"].keys()) == set(["_id", "is_mammal"])
+                assert set(query_op["query"].keys()) == {"_id", "is_mammal"}
             else:
-                assert set(query_op["command"]["q"].keys()) == set(["_id", "is_mammal"])
+                assert set(query_op["command"]["q"].keys()) == {"_id", "is_mammal"}
+
+        Animal.drop_collection()
+
+    def test_save_create_shard_key_routing(self):
+        """Ensures inserting a doc with a specified shard_key includes it in
+        the query.
+        """
+
+        class Animal(Document):
+            _id = UUIDField(binary=False, primary_key=True, default=uuid.uuid4)
+            is_mammal = BooleanField()
+            name = StringField()
+            meta = {"shard_key": ("is_mammal",)}
+
+        Animal.drop_collection()
+        doc = Animal(is_mammal=True, name="Dog")
+
+        with query_counter() as q:
+            doc.save()
+            query_op = q.db.system.profile.find({"ns": "mongoenginetest.animal"})[0]
+            assert query_op["op"] == "command"
+            assert query_op["command"]["findAndModify"] == "animal"
+            assert set(query_op["command"]["query"].keys()) == {"_id", "is_mammal"}
 
         Animal.drop_collection()
 
@@ -578,7 +599,8 @@ class TestDocumentInstance(MongoDBTestCase):
         doc.embedded_field.list_field.append(1)
         doc.embedded_field.dict_field["woot"] = "woot"
 
-        assert doc._get_changed_fields() == [
+        changed = doc._get_changed_fields()
+        assert changed == [
             "list_field",
             "dict_field.woot",
             "embedded_field.list_field",
@@ -1406,11 +1428,11 @@ class TestDocumentInstance(MongoDBTestCase):
         coll = self.Person._get_collection()
         doc = self.Person(name="John").save()
         raw_doc = coll.find_one({"_id": doc.pk})
-        assert set(raw_doc.keys()) == set(["_id", "_cls", "name"])
+        assert set(raw_doc.keys()) == {"_id", "_cls", "name"}
 
         doc.update(rename__name="first_name")
         raw_doc = coll.find_one({"_id": doc.pk})
-        assert set(raw_doc.keys()) == set(["_id", "_cls", "first_name"])
+        assert set(raw_doc.keys()) == {"_id", "_cls", "first_name"}
         assert raw_doc["first_name"] == "John"
 
     def test_inserts_if_you_set_the_pk(self):
@@ -1530,8 +1552,7 @@ class TestDocumentInstance(MongoDBTestCase):
         assert site.page.log_message == "Error: Dummy message"
 
     def test_update_list_field(self):
-        """Test update on `ListField` with $pull + $in.
-        """
+        """Test update on `ListField` with $pull + $in."""
 
         class Doc(Document):
             foo = ListField(StringField())
@@ -2020,7 +2041,7 @@ class TestDocumentInstance(MongoDBTestCase):
         assert promoted_employee.details is None
 
     def test_object_mixins(self):
-        class NameMixin(object):
+        class NameMixin:
             name = StringField()
 
         class Foo(EmbeddedDocument, NameMixin):
@@ -2034,7 +2055,7 @@ class TestDocumentInstance(MongoDBTestCase):
         assert ["id", "name", "widgets"] == sorted(Bar._fields.keys())
 
     def test_mixin_inheritance(self):
-        class BaseMixIn(object):
+        class BaseMixIn:
             count = IntField()
             data = StringField()
 
@@ -2793,15 +2814,13 @@ class TestDocumentInstance(MongoDBTestCase):
         register_connection("testdb-2", "mongoenginetest2")
 
         class A(Document):
-            """Uses default db_alias
-            """
+            """Uses default db_alias"""
 
             name = StringField()
             meta = {"allow_inheritance": True}
 
         class B(A):
-            """Uses testdb-2 db_alias
-            """
+            """Uses testdb-2 db_alias"""
 
             meta = {"db_alias": "testdb-2"}
 
@@ -2881,50 +2900,32 @@ class TestDocumentInstance(MongoDBTestCase):
         # Checks
         assert ",".join([str(b) for b in Book.objects.all()]) == "1,2,3,4,5,6,7,8,9"
         # bob related books
-        assert (
-            ",".join(
-                [
-                    str(b)
-                    for b in Book.objects.filter(
-                        Q(extra__a=bob) | Q(author=bob) | Q(extra__b=bob)
-                    )
-                ]
-            )
-            == "1,2,3,4"
+        bob_books_qs = Book.objects.filter(
+            Q(extra__a=bob) | Q(author=bob) | Q(extra__b=bob)
         )
+        assert [str(b) for b in bob_books_qs] == ["1", "2", "3", "4"]
+        assert bob_books_qs.count() == 4
 
         # Susan & Karl related books
-        assert (
-            ",".join(
-                [
-                    str(b)
-                    for b in Book.objects.filter(
-                        Q(extra__a__all=[karl, susan])
-                        | Q(author__all=[karl, susan])
-                        | Q(extra__b__all=[karl.to_dbref(), susan.to_dbref()])
-                    )
-                ]
-            )
-            == "1"
+        susan_karl_books_qs = Book.objects.filter(
+            Q(extra__a__all=[karl, susan])
+            | Q(author__all=[karl, susan])
+            | Q(extra__b__all=[karl.to_dbref(), susan.to_dbref()])
         )
+        assert [str(b) for b in susan_karl_books_qs] == ["1"]
+        assert susan_karl_books_qs.count() == 1
 
         # $Where
-        assert (
-            u",".join(
-                [
-                    str(b)
-                    for b in Book.objects.filter(
-                        __raw__={
-                            "$where": """
+        custom_qs = Book.objects.filter(
+            __raw__={
+                "$where": """
                                             function(){
                                                 return this.name == '1' ||
                                                        this.name == '2';}"""
-                        }
-                    )
-                ]
-            )
-            == "1,2"
+            }
         )
+        assert [str(b) for b in custom_qs] == ["1", "2"]
+        assert custom_qs.count() == 2
 
     def test_switch_db_instance(self):
         register_connection("testdb-1", "mongoenginetest2")
@@ -3284,7 +3285,7 @@ class TestDocumentInstance(MongoDBTestCase):
                 for node_name, node in self.nodes.items():
                     node.expand()
                     node.save(*args, **kwargs)
-                super(NodesSystem, self).save(*args, **kwargs)
+                super().save(*args, **kwargs)
 
         NodesSystem.drop_collection()
         Node.drop_collection()
@@ -3411,7 +3412,7 @@ class TestDocumentInstance(MongoDBTestCase):
         assert obj3 != dbref2
         assert dbref2 != obj3
 
-    def test_default_values(self):
+    def test_default_values_dont_get_override_upon_save_when_only_is_used(self):
         class Person(Document):
             created_on = DateTimeField(default=lambda: datetime.utcnow())
             name = StringField()
@@ -3589,8 +3590,7 @@ class TestDocumentInstance(MongoDBTestCase):
         assert u_from_db.height is None
 
     def test_not_saved_eq(self):
-        """Ensure we can compare documents not saved.
-        """
+        """Ensure we can compare documents not saved."""
 
         class Person(Document):
             pass
@@ -3734,7 +3734,7 @@ class TestDocumentInstance(MongoDBTestCase):
         _ = list(Jedi.objects)  # Ensure a proper document loads without errors
 
         # Forces a document with a wrong shape (may occur in case of migration)
-        value = u"I_should_be_a_dict"
+        value = "I_should_be_a_dict"
         coll.insert_one({"light_saber": value})
 
         with pytest.raises(InvalidDocumentError) as exc_info:
@@ -3799,5 +3799,95 @@ class ObjectKeyTestCase(MongoDBTestCase):
         assert book._object_key == {"pk": book.pk, "author__name": "Author"}
 
 
+class DBFieldMappingTest(MongoDBTestCase):
+    def setUp(self):
+        class Fields:
+            w1 = BooleanField(db_field="w2")
+
+            x1 = BooleanField(db_field="x2")
+            x2 = BooleanField(db_field="x3")
+
+            y1 = BooleanField(db_field="y0")
+            y2 = BooleanField(db_field="y1")
+
+            z1 = BooleanField(db_field="z2")
+            z2 = BooleanField(db_field="z1")
+
+        class Doc(Fields, Document):
+            pass
+
+        class DynDoc(Fields, DynamicDocument):
+            pass
+
+        self.Doc = Doc
+        self.DynDoc = DynDoc
+
+    def tearDown(self):
+        for collection in list_collection_names(self.db):
+            self.db.drop_collection(collection)
+
+    def test_setting_fields_in_constructor_of_strict_doc_uses_model_names(self):
+        doc = self.Doc(z1=True, z2=False)
+        assert doc.z1 is True
+        assert doc.z2 is False
+
+    def test_setting_fields_in_constructor_of_dyn_doc_uses_model_names(self):
+        doc = self.DynDoc(z1=True, z2=False)
+        assert doc.z1 is True
+        assert doc.z2 is False
+
+    def test_setting_unknown_field_in_constructor_of_dyn_doc_does_not_overwrite_model_fields(
+        self,
+    ):
+        doc = self.DynDoc(w2=True)
+        assert doc.w1 is None
+        assert doc.w2 is True
+
+    def test_unknown_fields_of_strict_doc_do_not_overwrite_dbfields_1(self):
+        doc = self.Doc()
+        doc.w2 = True
+        doc.x3 = True
+        doc.y0 = True
+        doc.save()
+        reloaded = self.Doc.objects.get(id=doc.id)
+        assert reloaded.w1 is None
+        assert reloaded.x1 is None
+        assert reloaded.x2 is None
+        assert reloaded.y1 is None
+        assert reloaded.y2 is None
+
+    def test_dbfields_are_loaded_to_the_right_modelfield_for_strict_doc_2(self):
+        doc = self.Doc()
+        doc.x2 = True
+        doc.y2 = True
+        doc.z2 = True
+        doc.save()
+        reloaded = self.Doc.objects.get(id=doc.id)
+        assert (
+            reloaded.x1,
+            reloaded.x2,
+            reloaded.y1,
+            reloaded.y2,
+            reloaded.z1,
+            reloaded.z2,
+        ) == (doc.x1, doc.x2, doc.y1, doc.y2, doc.z1, doc.z2)
+
+    def test_dbfields_are_loaded_to_the_right_modelfield_for_dyn_doc_2(self):
+        doc = self.DynDoc()
+        doc.x2 = True
+        doc.y2 = True
+        doc.z2 = True
+        doc.save()
+        reloaded = self.DynDoc.objects.get(id=doc.id)
+        assert (
+            reloaded.x1,
+            reloaded.x2,
+            reloaded.y1,
+            reloaded.y2,
+            reloaded.z1,
+            reloaded.z2,
+        ) == (doc.x1, doc.x2, doc.y1, doc.y2, doc.z1, doc.z2)
+
+
 if __name__ == "__main__":
     unittest.main()

tests/document/test_validation.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 from datetime import datetime
 
@@ -10,8 +9,7 @@ from tests.utils import MongoDBTestCase
 
 class TestValidatorError(MongoDBTestCase):
     def test_to_dict(self):
-        """Ensure a ValidationError handles error to_dict correctly.
-        """
+        """Ensure a ValidationError handles error to_dict correctly."""
         error = ValidationError("root")
         assert error.to_dict() == {}
 
@@ -91,8 +89,7 @@ class TestValidatorError(MongoDBTestCase):
             p.validate()
 
     def test_embedded_document_validation(self):
-        """Ensure that embedded documents may be validated.
-        """
+        """Ensure that embedded documents may be validated."""
 
         class Comment(EmbeddedDocument):
             date = DateTimeField()
@@ -213,10 +210,7 @@ class TestValidatorError(MongoDBTestCase):
         child.reference = parent
 
         # Saving the child should not raise a ValidationError
-        try:
-            child.save()
-        except ValidationError as e:
-            self.fail("ValidationError raised: %s" % e.message)
+        child.save()
 
 
 if __name__ == "__main__":

tests/fields/test_binary_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import uuid
 
 from bson import Binary
@@ -14,14 +13,13 @@ BIN_VALUE = "\xa9\xf3\x8d(\xd7\x03\x84\xb4k[\x0f\xe3\xa2\x19\x85p[J\xa3\xd2>\xde
 
 class TestBinaryField(MongoDBTestCase):
     def test_binary_fields(self):
-        """Ensure that binary fields can be stored and retrieved.
-        """
+        """Ensure that binary fields can be stored and retrieved."""
 
         class Attachment(Document):
             content_type = StringField()
             blob = BinaryField()
 
-        BLOB = "\xe6\x00\xc4\xff\x07".encode("latin-1")
+        BLOB = b"\xe6\x00\xc4\xff\x07"
         MIME_TYPE = "application/octet-stream"
 
         Attachment.drop_collection()
@@ -34,8 +32,7 @@ class TestBinaryField(MongoDBTestCase):
         assert BLOB == bytes(attachment_1.blob)
 
     def test_validation_succeeds(self):
-        """Ensure that valid values can be assigned to binary fields.
-        """
+        """Ensure that valid values can be assigned to binary fields."""
 
         class AttachmentRequired(Document):
             blob = BinaryField(required=True)
@@ -46,11 +43,11 @@ class TestBinaryField(MongoDBTestCase):
         attachment_required = AttachmentRequired()
         with pytest.raises(ValidationError):
             attachment_required.validate()
-        attachment_required.blob = Binary("\xe6\x00\xc4\xff\x07".encode("latin-1"))
+        attachment_required.blob = Binary(b"\xe6\x00\xc4\xff\x07")
         attachment_required.validate()
 
-        _5_BYTES = "\xe6\x00\xc4\xff\x07".encode("latin-1")
-        _4_BYTES = "\xe6\x00\xc4\xff".encode("latin-1")
+        _5_BYTES = b"\xe6\x00\xc4\xff\x07"
+        _4_BYTES = b"\xe6\x00\xc4\xff"
         with pytest.raises(ValidationError):
             AttachmentSizeLimit(blob=_5_BYTES).validate()
         AttachmentSizeLimit(blob=_4_BYTES).validate()
@@ -61,7 +58,7 @@ class TestBinaryField(MongoDBTestCase):
         class Attachment(Document):
             blob = BinaryField()
 
-        for invalid_data in (2, u"Im_a_unicode", ["some_str"]):
+        for invalid_data in (2, "Im_a_unicode", ["some_str"]):
             with pytest.raises(ValidationError):
                 Attachment(blob=invalid_data).validate()
 
@@ -132,7 +129,7 @@ class TestBinaryField(MongoDBTestCase):
 
         MyDocument.drop_collection()
 
-        bin_data = "\xe6\x00\xc4\xff\x07".encode("latin-1")
+        bin_data = b"\xe6\x00\xc4\xff\x07"
         doc = MyDocument(bin_field=bin_data).save()
 
         n_updated = MyDocument.objects(bin_field=bin_data).update_one(

tests/fields/test_boolean_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import pytest
 
 from mongoengine import *

tests/fields/test_cached_reference_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from decimal import Decimal
 
 import pytest
@@ -191,9 +190,9 @@ class TestCachedReferenceField(MongoDBTestCase):
 
         assert dict(a2.to_mongo()) == {
             "_id": a2.pk,
-            "name": u"Wilson Junior",
-            "tp": u"pf",
-            "father": {"_id": a1.pk, "tp": u"pj"},
+            "name": "Wilson Junior",
+            "tp": "pf",
+            "father": {"_id": a1.pk, "tp": "pj"},
         }
 
         assert Person.objects(father=a1)._query == {"father._id": a1.pk}
@@ -205,9 +204,9 @@ class TestCachedReferenceField(MongoDBTestCase):
         a2.reload()
         assert dict(a2.to_mongo()) == {
             "_id": a2.pk,
-            "name": u"Wilson Junior",
-            "tp": u"pf",
-            "father": {"_id": a1.pk, "tp": u"pf"},
+            "name": "Wilson Junior",
+            "tp": "pf",
+            "father": {"_id": a1.pk, "tp": "pf"},
         }
 
     def test_cached_reference_fields_on_embedded_documents(self):

tests/fields/test_complex_datetime_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import datetime
 import itertools
 import math
@@ -61,7 +60,7 @@ class ComplexDateTimeFieldTest(MongoDBTestCase):
             assert log == log1
 
         # Test string padding
-        microsecond = map(int, [math.pow(10, x) for x in range(6)])
+        microsecond = map(int, (math.pow(10, x) for x in range(6)))
         mm = dd = hh = ii = ss = [1, 10]
 
         for values in itertools.product([2014], mm, dd, hh, ii, ss, microsecond):

tests/fields/test_date_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import datetime
 
 import pytest

tests/fields/test_datetime_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import datetime as dt
 
 import pytest

tests/fields/test_decimal_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from decimal import Decimal
 
 import pytest
@@ -9,8 +8,7 @@ from tests.utils import MongoDBTestCase
 
 class TestDecimalField(MongoDBTestCase):
     def test_validation(self):
-        """Ensure that invalid values cannot be assigned to decimal fields.
-        """
+        """Ensure that invalid values cannot be assigned to decimal fields."""
 
         class Person(Document):
             height = DecimalField(min_value=Decimal("0.1"), max_value=Decimal("3.5"))

tests/fields/test_dict_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from bson import InvalidDocument
 import pytest
 
@@ -113,7 +112,7 @@ class TestDictField(MongoDBTestCase):
         post.info.setdefault("authors", [])
         post.save()
         post.reload()
-        assert [] == post.info["authors"]
+        assert post.info["authors"] == []
 
     def test_dictfield_dump_document(self):
         """Ensure a DictField can handle another document's dump."""

tests/fields/test_email_field.py

@@ -1,9 +1,6 @@
-# -*- coding: utf-8 -*-
-import sys
-
 import pytest
 
-from mongoengine import *
+from mongoengine import Document, EmailField, ValidationError
 from tests.utils import MongoDBTestCase
 
 
@@ -31,11 +28,11 @@ class TestEmailField(MongoDBTestCase):
             user.validate()
 
         # unicode domain
-        user = User(email=u"user@пример.рф")
+        user = User(email="user@пример.рф")
         user.validate()
 
         # invalid unicode domain
-        user = User(email=u"user@пример")
+        user = User(email="user@пример")
         with pytest.raises(ValidationError):
             user.validate()
 
@@ -49,7 +46,7 @@ class TestEmailField(MongoDBTestCase):
             email = EmailField()
 
         # unicode user shouldn't validate by default...
-        user = User(email=u"Dörte@Sörensen.example.com")
+        user = User(email="Dörte@Sörensen.example.com")
         with pytest.raises(ValidationError):
             user.validate()
 
@@ -57,7 +54,7 @@ class TestEmailField(MongoDBTestCase):
         class User(Document):
             email = EmailField(allow_utf8_user=True)
 
-        user = User(email=u"Dörte@Sörensen.example.com")
+        user = User(email="Dörte@Sörensen.example.com")
         user.validate()
 
     def test_email_field_domain_whitelist(self):

tests/fields/test_embedded_document_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import pytest
 
 from mongoengine import (
@@ -75,7 +74,7 @@ class TestEmbeddedDocumentField(MongoDBTestCase):
         # Test non exiting attribute
         with pytest.raises(InvalidQueryError) as exc_info:
             Person.objects(settings__notexist="bar").first()
-        assert str(exc_info.value) == u'Cannot resolve field "notexist"'
+        assert str(exc_info.value) == 'Cannot resolve field "notexist"'
 
         with pytest.raises(LookUpError):
             Person.objects.only("settings.notexist")
@@ -111,7 +110,7 @@ class TestEmbeddedDocumentField(MongoDBTestCase):
         # Test non exiting attribute
         with pytest.raises(InvalidQueryError) as exc_info:
             assert Person.objects(settings__notexist="bar").first().id == p.id
-        assert str(exc_info.value) == u'Cannot resolve field "notexist"'
+        assert str(exc_info.value) == 'Cannot resolve field "notexist"'
 
         # Test existing attribute
         assert Person.objects(settings__base_foo="basefoo").first().id == p.id
@@ -319,7 +318,7 @@ class TestGenericEmbeddedDocumentField(MongoDBTestCase):
         # Test non exiting attribute
         with pytest.raises(InvalidQueryError) as exc_info:
             Person.objects(settings__notexist="bar").first()
-        assert str(exc_info.value) == u'Cannot resolve field "notexist"'
+        assert str(exc_info.value) == 'Cannot resolve field "notexist"'
 
         with pytest.raises(LookUpError):
             Person.objects.only("settings.notexist")
@@ -347,7 +346,7 @@ class TestGenericEmbeddedDocumentField(MongoDBTestCase):
         # Test non exiting attribute
         with pytest.raises(InvalidQueryError) as exc_info:
             assert Person.objects(settings__notexist="bar").first().id == p.id
-        assert str(exc_info.value) == u'Cannot resolve field "notexist"'
+        assert str(exc_info.value) == 'Cannot resolve field "notexist"'
 
         # Test existing attribute
         assert Person.objects(settings__base_foo="basefoo").first().id == p.id

tests/fields/test_enum_field.py

@@ -0,0 +1,122 @@
+from enum import Enum
+
+from bson import InvalidDocument
+import pytest
+
+from mongoengine import *
+from tests.utils import MongoDBTestCase, get_as_pymongo
+
+
+class Status(Enum):
+    NEW = "new"
+    DONE = "done"
+
+
+class ModelWithEnum(Document):
+    status = EnumField(Status)
+
+
+class TestStringEnumField(MongoDBTestCase):
+    def test_storage(self):
+        model = ModelWithEnum(status=Status.NEW).save()
+        assert get_as_pymongo(model) == {"_id": model.id, "status": "new"}
+
+    def test_set_enum(self):
+        ModelWithEnum.drop_collection()
+        ModelWithEnum(status=Status.NEW).save()
+        assert ModelWithEnum.objects(status=Status.NEW).count() == 1
+        assert ModelWithEnum.objects.first().status == Status.NEW
+
+    def test_set_by_value(self):
+        ModelWithEnum.drop_collection()
+        ModelWithEnum(status="new").save()
+        assert ModelWithEnum.objects.first().status == Status.NEW
+
+    def test_filter(self):
+        ModelWithEnum.drop_collection()
+        ModelWithEnum(status="new").save()
+        assert ModelWithEnum.objects(status="new").count() == 1
+        assert ModelWithEnum.objects(status=Status.NEW).count() == 1
+        assert ModelWithEnum.objects(status=Status.DONE).count() == 0
+
+    def test_change_value(self):
+        m = ModelWithEnum(status="new")
+        m.status = Status.DONE
+        m.save()
+        assert m.status == Status.DONE
+
+    def test_set_default(self):
+        class ModelWithDefault(Document):
+            status = EnumField(Status, default=Status.DONE)
+
+        m = ModelWithDefault().save()
+        assert m.status == Status.DONE
+
+    def test_enum_field_can_be_empty(self):
+        ModelWithEnum.drop_collection()
+        m = ModelWithEnum().save()
+        assert m.status is None
+        assert ModelWithEnum.objects()[0].status is None
+        assert ModelWithEnum.objects(status=None).count() == 1
+
+    def test_set_none_explicitly(self):
+        ModelWithEnum.drop_collection()
+        ModelWithEnum(status=None).save()
+        assert ModelWithEnum.objects.first().status is None
+
+    def test_cannot_create_model_with_wrong_enum_value(self):
+        m = ModelWithEnum(status="wrong_one")
+        with pytest.raises(ValidationError):
+            m.validate()
+
+    def test_user_is_informed_when_tries_to_set_choices(self):
+        with pytest.raises(ValueError, match="'choices' can't be set on EnumField"):
+            EnumField(Status, choices=["my", "custom", "options"])
+
+
+class Color(Enum):
+    RED = 1
+    BLUE = 2
+
+
+class ModelWithColor(Document):
+    color = EnumField(Color, default=Color.RED)
+
+
+class TestIntEnumField(MongoDBTestCase):
+    def test_enum_with_int(self):
+        ModelWithColor.drop_collection()
+        m = ModelWithColor().save()
+        assert m.color == Color.RED
+        assert ModelWithColor.objects(color=Color.RED).count() == 1
+        assert ModelWithColor.objects(color=1).count() == 1
+        assert ModelWithColor.objects(color=2).count() == 0
+
+    def test_create_int_enum_by_value(self):
+        model = ModelWithColor(color=2).save()
+        assert model.color == Color.BLUE
+
+    def test_storage_enum_with_int(self):
+        model = ModelWithColor(color=Color.BLUE).save()
+        assert get_as_pymongo(model) == {"_id": model.id, "color": 2}
+
+    def test_validate_model(self):
+        with pytest.raises(ValidationError, match="Value must be one of"):
+            ModelWithColor(color=3).validate()
+
+        with pytest.raises(ValidationError, match="Value must be one of"):
+            ModelWithColor(color="wrong_type").validate()
+
+
+class TestFunkyEnumField(MongoDBTestCase):
+    def test_enum_incompatible_bson_type_fails_during_save(self):
+        class FunkyColor(Enum):
+            YELLOW = object()
+
+        class ModelWithFunkyColor(Document):
+            color = EnumField(FunkyColor)
+
+        m = ModelWithFunkyColor(color=FunkyColor.YELLOW)
+
+        with pytest.raises(InvalidDocument, match="[cC]annot encode object"):
+            m.save()

tests/fields/test_fields.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import datetime
 import unittest
 
@@ -293,7 +292,7 @@ class TestField(MongoDBTestCase):
         HandleNoneFields.drop_collection()
 
         doc = HandleNoneFields()
-        doc.str_fld = u"spam ham egg"
+        doc.str_fld = "spam ham egg"
         doc.int_fld = 42
         doc.flt_fld = 4.2
         doc.com_dt_fld = datetime.datetime.utcnow()
@@ -329,14 +328,14 @@ class TestField(MongoDBTestCase):
         HandleNoneFields.drop_collection()
 
         doc = HandleNoneFields()
-        doc.str_fld = u"spam ham egg"
+        doc.str_fld = "spam ham egg"
         doc.int_fld = 42
         doc.flt_fld = 4.2
         doc.comp_dt_fld = datetime.datetime.utcnow()
         doc.save()
 
         # Unset all the fields
-        HandleNoneFields._get_collection().update(
+        HandleNoneFields._get_collection().update_one(
             {"_id": doc.id},
             {"$unset": {"str_fld": 1, "int_fld": 1, "flt_fld": 1, "comp_dt_fld": 1}},
         )
@@ -427,9 +426,9 @@ class TestField(MongoDBTestCase):
     def test_list_validation(self):
         """Ensure that a list field only accepts lists with valid elements."""
         access_level_choices = (
-            ("a", u"Administration"),
-            ("b", u"Manager"),
-            ("c", u"Staff"),
+            ("a", "Administration"),
+            ("b", "Manager"),
+            ("c", "Staff"),
         )
 
         class User(Document):
@@ -477,7 +476,7 @@ class TestField(MongoDBTestCase):
         post.access_list = ["a", "b"]
         post.validate()
 
-        assert post.get_access_list_display() == u"Administration, Manager"
+        assert post.get_access_list_display() == "Administration, Manager"
 
         post.comments = ["a"]
         with pytest.raises(ValidationError):
@@ -545,8 +544,7 @@ class TestField(MongoDBTestCase):
         post.validate()
 
     def test_sorted_list_sorting(self):
-        """Ensure that a sorted list field properly sorts values.
-        """
+        """Ensure that a sorted list field properly sorts values."""
 
         class Comment(EmbeddedDocument):
             order = IntField()
@@ -662,8 +660,7 @@ class TestField(MongoDBTestCase):
         )
 
     def test_list_field_manipulative_operators(self):
-        """Ensure that ListField works with standard list operators that manipulate the list.
-        """
+        """Ensure that ListField works with standard list operators that manipulate the list."""
 
         class BlogPost(Document):
             ref = StringField()
@@ -1084,7 +1081,7 @@ class TestField(MongoDBTestCase):
 
         e = Simple().save()
         e.mapping = []
-        assert [] == e._changed_fields
+        assert e._changed_fields == []
 
         class Simple(Document):
             mapping = DictField()
@@ -1093,7 +1090,7 @@ class TestField(MongoDBTestCase):
 
         e = Simple().save()
         e.mapping = {}
-        assert [] == e._changed_fields
+        assert e._changed_fields == []
 
     def test_slice_marks_field_as_changed(self):
         class Simple(Document):
@@ -1360,9 +1357,9 @@ class TestField(MongoDBTestCase):
         foo.delete()
         bar = Bar.objects.get()
         with pytest.raises(DoesNotExist):
-            getattr(bar, "ref")
+            bar.ref
         with pytest.raises(DoesNotExist):
-            getattr(bar, "generic_ref")
+            bar.generic_ref
 
         # When auto_dereference is disabled, there is no trouble returning DBRef
         bar = Bar.objects.get()
@@ -1373,8 +1370,7 @@ class TestField(MongoDBTestCase):
         assert bar.generic_ref == {"_ref": expected, "_cls": "Foo"}
 
     def test_list_item_dereference(self):
-        """Ensure that DBRef items in ListFields are dereferenced.
-        """
+        """Ensure that DBRef items in ListFields are dereferenced."""
 
         class User(Document):
             name = StringField()
@@ -1399,8 +1395,7 @@ class TestField(MongoDBTestCase):
         assert group_obj.members[1].name == user2.name
 
     def test_recursive_reference(self):
-        """Ensure that ReferenceFields can reference their own documents.
-        """
+        """Ensure that ReferenceFields can reference their own documents."""
 
         class Employee(Document):
             name = StringField()
@@ -1427,8 +1422,7 @@ class TestField(MongoDBTestCase):
         assert peter.friends == friends
 
     def test_recursive_embedding(self):
-        """Ensure that EmbeddedDocumentFields can contain their own documents.
-        """
+        """Ensure that EmbeddedDocumentFields can contain their own documents."""
 
         class TreeNode(EmbeddedDocument):
             name = StringField()
@@ -1504,8 +1498,7 @@ class TestField(MongoDBTestCase):
             AbstractDoc.drop_collection()
 
     def test_reference_class_with_abstract_parent(self):
-        """Ensure that a class with an abstract parent can be referenced.
-        """
+        """Ensure that a class with an abstract parent can be referenced."""
 
         class Sibling(Document):
             name = StringField()
@@ -1575,8 +1568,7 @@ class TestField(MongoDBTestCase):
             brother.save()
 
     def test_generic_reference(self):
-        """Ensure that a GenericReferenceField properly dereferences items.
-        """
+        """Ensure that a GenericReferenceField properly dereferences items."""
 
         class Link(Document):
             title = StringField()
@@ -1615,8 +1607,7 @@ class TestField(MongoDBTestCase):
         assert isinstance(bm.bookmark_object, Link)
 
     def test_generic_reference_list(self):
-        """Ensure that a ListField properly dereferences generic references.
-        """
+        """Ensure that a ListField properly dereferences generic references."""
 
         class Link(Document):
             title = StringField()
@@ -1719,8 +1710,7 @@ class TestField(MongoDBTestCase):
         assert bm.bookmark_object == post_1
 
     def test_generic_reference_string_choices(self):
-        """Ensure that a GenericReferenceField can handle choices as strings
-        """
+        """Ensure that a GenericReferenceField can handle choices as strings"""
 
         class Link(Document):
             title = StringField()
@@ -1812,8 +1802,7 @@ class TestField(MongoDBTestCase):
         assert user.bookmarks == [post_1]
 
     def test_generic_reference_list_item_modification(self):
-        """Ensure that modifications of related documents (through generic reference) don't influence on querying
-        """
+        """Ensure that modifications of related documents (through generic reference) don't influence on querying"""
 
         class Post(Document):
             title = StringField()
@@ -1901,8 +1890,7 @@ class TestField(MongoDBTestCase):
         assert doc == doc2
 
     def test_choices_allow_using_sets_as_choices(self):
-        """Ensure that sets can be used when setting choices
-        """
+        """Ensure that sets can be used when setting choices"""
 
         class Shirt(Document):
             size = StringField(choices={"M", "L"})
@@ -1921,8 +1909,7 @@ class TestField(MongoDBTestCase):
         shirt.validate()
 
     def test_choices_validation_accept_possible_value(self):
-        """Ensure that value is in a container of allowed values.
-        """
+        """Ensure that value is in a container of allowed values."""
 
         class Shirt(Document):
             size = StringField(choices=("S", "M"))
@@ -1931,8 +1918,7 @@ class TestField(MongoDBTestCase):
         shirt.validate()
 
     def test_choices_validation_reject_unknown_value(self):
-        """Ensure that unallowed value are rejected upon validation
-        """
+        """Ensure that unallowed value are rejected upon validation"""
 
         class Shirt(Document):
             size = StringField(choices=("S", "M"))
@@ -1990,8 +1976,7 @@ class TestField(MongoDBTestCase):
             shirt1.validate()
 
     def test_simple_choices_validation(self):
-        """Ensure that value is in a container of allowed values.
-        """
+        """Ensure that value is in a container of allowed values."""
 
         class Shirt(Document):
             size = StringField(max_length=3, choices=("S", "M", "L", "XL", "XXL"))
@@ -2040,12 +2025,11 @@ class TestField(MongoDBTestCase):
             shirt.validate()
 
     def test_simple_choices_validation_invalid_value(self):
-        """Ensure that error messages are correct.
-        """
+        """Ensure that error messages are correct."""
         SIZES = ("S", "M", "L", "XL", "XXL")
         COLORS = (("R", "Red"), ("B", "Blue"))
-        SIZE_MESSAGE = u"Value must be one of ('S', 'M', 'L', 'XL', 'XXL')"
-        COLOR_MESSAGE = u"Value must be one of ['R', 'B']"
+        SIZE_MESSAGE = "Value must be one of ('S', 'M', 'L', 'XL', 'XXL')"
+        COLOR_MESSAGE = "Value must be one of ['R', 'B']"
 
         class Shirt(Document):
             size = StringField(max_length=3, choices=SIZES)
@@ -2108,7 +2092,7 @@ class TestField(MongoDBTestCase):
             assert "comments" in error_dict
             assert 1 in error_dict["comments"]
             assert "content" in error_dict["comments"][1]
-            assert error_dict["comments"][1]["content"] == u"Field is required"
+            assert error_dict["comments"][1]["content"] == "Field is required"
 
         post.comments[1].content = "here we go"
         post.validate()
@@ -2120,7 +2104,7 @@ class TestField(MongoDBTestCase):
 
         class EnumField(BaseField):
             def __init__(self, **kwargs):
-                super(EnumField, self).__init__(**kwargs)
+                super().__init__(**kwargs)
 
             def to_mongo(self, value):
                 return value
@@ -2273,6 +2257,13 @@ class TestField(MongoDBTestCase):
         with pytest.raises(FieldDoesNotExist):
             Doc(bar="test")
 
+    def test_undefined_field_works_no_confusion_with_db_field(self):
+        class Doc(Document):
+            foo = StringField(db_field="bar")
+
+        with pytest.raises(FieldDoesNotExist):
+            Doc(bar="test")
+
 
 class TestEmbeddedDocumentListField(MongoDBTestCase):
     def setUp(self):
@@ -2615,11 +2606,11 @@ class TestEmbeddedDocumentListField(MongoDBTestCase):
         """
         post = self.BlogPost(
             comments=[
-                self.Comments(author="user1", message=u"сообщение"),
-                self.Comments(author="user2", message=u"хабарлама"),
+                self.Comments(author="user1", message="сообщение"),
+                self.Comments(author="user2", message="хабарлама"),
             ]
         ).save()
-        assert post.comments.get(message=u"сообщение").author == "user1"
+        assert post.comments.get(message="сообщение").author == "user1"
 
     def test_save(self):
         """

tests/fields/test_file_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import copy
 import os
 import tempfile
@@ -12,7 +11,7 @@ from mongoengine import *
 from mongoengine.connection import get_db
 
 try:
-    from PIL import Image
+    from PIL import Image  # noqa: F401
 
     HAS_PIL = True
 except ImportError:
@@ -49,15 +48,14 @@ class TestFileField(MongoDBTestCase):
         DemoFile.objects.create()
 
     def test_file_fields(self):
-        """Ensure that file fields can be written to and their data retrieved
-        """
+        """Ensure that file fields can be written to and their data retrieved"""
 
         class PutFile(Document):
             the_file = FileField()
 
         PutFile.drop_collection()
 
-        text = "Hello, World!".encode("latin-1")
+        text = b"Hello, World!"
         content_type = "text/plain"
 
         putfile = PutFile()
@@ -92,16 +90,15 @@ class TestFileField(MongoDBTestCase):
         result.the_file.delete()
 
     def test_file_fields_stream(self):
-        """Ensure that file fields can be written to and their data retrieved
-        """
+        """Ensure that file fields can be written to and their data retrieved"""
 
         class StreamFile(Document):
             the_file = FileField()
 
         StreamFile.drop_collection()
 
-        text = "Hello, World!".encode("latin-1")
-        more_text = "Foo Bar".encode("latin-1")
+        text = b"Hello, World!"
+        more_text = b"Foo Bar"
         content_type = "text/plain"
 
         streamfile = StreamFile()
@@ -136,8 +133,8 @@ class TestFileField(MongoDBTestCase):
 
         StreamFile.drop_collection()
 
-        text = "Hello, World!".encode("latin-1")
-        more_text = "Foo Bar".encode("latin-1")
+        text = b"Hello, World!"
+        more_text = b"Foo Bar"
 
         streamfile = StreamFile()
         streamfile.save()
@@ -166,8 +163,8 @@ class TestFileField(MongoDBTestCase):
         class SetFile(Document):
             the_file = FileField()
 
-        text = "Hello, World!".encode("latin-1")
-        more_text = "Foo Bar".encode("latin-1")
+        text = b"Hello, World!"
+        more_text = b"Foo Bar"
 
         SetFile.drop_collection()
 
@@ -195,7 +192,7 @@ class TestFileField(MongoDBTestCase):
         GridDocument.drop_collection()
 
         with tempfile.TemporaryFile() as f:
-            f.write("Hello World!".encode("latin-1"))
+            f.write(b"Hello World!")
             f.flush()
 
             # Test without default
@@ -212,7 +209,7 @@ class TestFileField(MongoDBTestCase):
             assert doc_b.the_file.grid_id == doc_c.the_file.grid_id
 
             # Test with default
-            doc_d = GridDocument(the_file="".encode("latin-1"))
+            doc_d = GridDocument(the_file=b"")
             doc_d.save()
 
             doc_e = GridDocument.objects.with_id(doc_d.id)
@@ -229,8 +226,7 @@ class TestFileField(MongoDBTestCase):
         assert ["doc_b", "doc_e"] == grid_fs.list()
 
     def test_file_uniqueness(self):
-        """Ensure that each instance of a FileField is unique
-        """
+        """Ensure that each instance of a FileField is unique"""
 
         class TestFile(Document):
             name = StringField()
@@ -239,7 +235,7 @@ class TestFileField(MongoDBTestCase):
         # First instance
         test_file = TestFile()
         test_file.name = "Hello, World!"
-        test_file.the_file.put("Hello, World!".encode("latin-1"))
+        test_file.the_file.put(b"Hello, World!")
         test_file.save()
 
         # Second instance
@@ -286,8 +282,7 @@ class TestFileField(MongoDBTestCase):
         assert test_file.the_file.get().length == 4971
 
     def test_file_boolean(self):
-        """Ensure that a boolean test of a FileField indicates its presence
-        """
+        """Ensure that a boolean test of a FileField indicates its presence"""
 
         class TestFile(Document):
             the_file = FileField()
@@ -296,9 +291,7 @@ class TestFileField(MongoDBTestCase):
 
         test_file = TestFile()
         assert not bool(test_file.the_file)
-        test_file.the_file.put(
-            "Hello, World!".encode("latin-1"), content_type="text/plain"
-        )
+        test_file.the_file.put(b"Hello, World!", content_type="text/plain")
         test_file.save()
         assert bool(test_file.the_file)
 
@@ -320,7 +313,7 @@ class TestFileField(MongoDBTestCase):
         class TestFile(Document):
             the_file = FileField()
 
-        text = "Hello, World!".encode("latin-1")
+        text = b"Hello, World!"
         content_type = "text/plain"
 
         testfile = TestFile()
@@ -364,7 +357,7 @@ class TestFileField(MongoDBTestCase):
         testfile.the_file.put(text, content_type=content_type, filename="hello")
         testfile.save()
 
-        text = "Bonjour, World!".encode("latin-1")
+        text = b"Bonjour, World!"
         testfile.the_file.replace(text, content_type=content_type, filename="hello")
         testfile.save()
 
@@ -388,7 +381,7 @@ class TestFileField(MongoDBTestCase):
         TestImage.drop_collection()
 
         with tempfile.TemporaryFile() as f:
-            f.write("Hello World!".encode("latin-1"))
+            f.write(b"Hello World!")
             f.flush()
 
             t = TestImage()
@@ -430,7 +423,7 @@ class TestFileField(MongoDBTestCase):
     @require_pil
     def test_image_field_resize(self):
         class TestImage(Document):
-            image = ImageField(size=(185, 37))
+            image = ImageField(size=(185, 37, True))
 
         TestImage.drop_collection()
 
@@ -472,7 +465,7 @@ class TestFileField(MongoDBTestCase):
     @require_pil
     def test_image_field_thumbnail(self):
         class TestImage(Document):
-            image = ImageField(thumbnail_size=(92, 18))
+            image = ImageField(thumbnail_size=(92, 18, True))
 
         TestImage.drop_collection()
 
@@ -504,21 +497,21 @@ class TestFileField(MongoDBTestCase):
         # First instance
         test_file = TestFile()
         test_file.name = "Hello, World!"
-        test_file.the_file.put("Hello, World!".encode("latin-1"), name="hello.txt")
+        test_file.the_file.put(b"Hello, World!", name="hello.txt")
         test_file.save()
 
         data = get_db("test_files").macumba.files.find_one()
         assert data.get("name") == "hello.txt"
 
         test_file = TestFile.objects.first()
-        assert test_file.the_file.read() == "Hello, World!".encode("latin-1")
+        assert test_file.the_file.read() == b"Hello, World!"
 
         test_file = TestFile.objects.first()
-        test_file.the_file = "Hello, World!".encode("latin-1")
+        test_file.the_file = b"Hello, World!"
         test_file.save()
 
         test_file = TestFile.objects.first()
-        assert test_file.the_file.read() == "Hello, World!".encode("latin-1")
+        assert test_file.the_file.read() == b"Hello, World!"
 
     def test_copyable(self):
         class PutFile(Document):
@@ -526,7 +519,7 @@ class TestFileField(MongoDBTestCase):
 
         PutFile.drop_collection()
 
-        text = "Hello, World!".encode("latin-1")
+        text = b"Hello, World!"
         content_type = "text/plain"
 
         putfile = PutFile()

tests/fields/test_float_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import pytest
 
 from mongoengine import *
@@ -20,8 +19,7 @@ class TestFloatField(MongoDBTestCase):
         assert 1 == TestDocument.objects(float_fld__ne=1).count()
 
     def test_validation(self):
-        """Ensure that invalid values cannot be assigned to float fields.
-        """
+        """Ensure that invalid values cannot be assigned to float fields."""
 
         class Person(Document):
             height = FloatField(min_value=0.1, max_value=3.5)

tests/fields/test_geo_fields.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 
 from mongoengine import *
@@ -9,7 +8,7 @@ class TestGeoField(MongoDBTestCase):
     def _test_for_expected_error(self, Cls, loc, expected):
         try:
             Cls(loc=loc).validate()
-            self.fail("Should not validate the location {0}".format(loc))
+            self.fail(f"Should not validate the location {loc}")
         except ValidationError as e:
             assert expected == e.to_dict()["loc"]
 
@@ -291,8 +290,7 @@ class TestGeoField(MongoDBTestCase):
         Location(loc=[[[[1, 2], [3, 4], [5, 6], [1, 2]]]]).validate()
 
     def test_indexes_geopoint(self):
-        """Ensure that indexes are created automatically for GeoPointFields.
-        """
+        """Ensure that indexes are created automatically for GeoPointFields."""
 
         class Event(Document):
             title = StringField()
@@ -318,8 +316,7 @@ class TestGeoField(MongoDBTestCase):
         assert geo_indicies == [{"fields": [("venue.location", "2d")]}]
 
     def test_indexes_2dsphere(self):
-        """Ensure that indexes are created automatically for GeoPointFields.
-        """
+        """Ensure that indexes are created automatically for GeoPointFields."""
 
         class Event(Document):
             title = StringField()
@@ -333,8 +330,7 @@ class TestGeoField(MongoDBTestCase):
         assert {"fields": [("point", "2dsphere")]} in geo_indicies
 
     def test_indexes_2dsphere_embedded(self):
-        """Ensure that indexes are created automatically for GeoPointFields.
-        """
+        """Ensure that indexes are created automatically for GeoPointFields."""
 
         class Venue(EmbeddedDocument):
             name = StringField()
@@ -381,7 +377,7 @@ class TestGeoField(MongoDBTestCase):
 
             meta = {"indexes": [[("location", "2dsphere"), ("datetime", 1)]]}
 
-        assert [] == Log._geo_indices()
+        assert Log._geo_indices() == []
 
         Log.drop_collection()
         Log.ensure_indexes()
@@ -401,7 +397,7 @@ class TestGeoField(MongoDBTestCase):
                 "indexes": [{"fields": [("location", "2dsphere"), ("datetime", 1)]}]
             }
 
-        assert [] == Log._geo_indices()
+        assert Log._geo_indices() == []
 
         Log.drop_collection()
         Log.ensure_indexes()

tests/fields/test_int_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import pytest
 
 from mongoengine import *
@@ -8,8 +7,7 @@ from tests.utils import MongoDBTestCase
 
 class TestIntField(MongoDBTestCase):
     def test_int_validation(self):
-        """Ensure that invalid values cannot be assigned to int fields.
-        """
+        """Ensure that invalid values cannot be assigned to int fields."""
 
         class Person(Document):
             age = IntField(min_value=0, max_value=110)

tests/fields/test_lazy_reference_field.py

@@ -1,9 +1,9 @@
-# -*- coding: utf-8 -*-
 from bson import DBRef, ObjectId
 import pytest
 
 from mongoengine import *
 from mongoengine.base import LazyReference
+from mongoengine.context_managers import query_counter
 
 from tests.utils import MongoDBTestCase
 
@@ -331,6 +331,50 @@ class TestLazyReferenceField(MongoDBTestCase):
         occ.in_embedded.in_list = [animal1.id, animal2.id]
         check_fields_type(occ)
 
+    def test_lazy_reference_embedded_dereferencing(self):
+        # Test case for #2375
+
+        # -- Test documents
+
+        class Author(Document):
+            name = StringField()
+
+        class AuthorReference(EmbeddedDocument):
+            author = LazyReferenceField(Author)
+
+        class Book(Document):
+            authors = EmbeddedDocumentListField(AuthorReference)
+
+        # -- Cleanup
+
+        Author.drop_collection()
+        Book.drop_collection()
+
+        # -- Create test data
+
+        author_1 = Author(name="A1").save()
+        author_2 = Author(name="A2").save()
+        author_3 = Author(name="A3").save()
+        book = Book(
+            authors=[
+                AuthorReference(author=author_1),
+                AuthorReference(author=author_2),
+                AuthorReference(author=author_3),
+            ]
+        ).save()
+
+        with query_counter() as qc:
+            book = Book.objects.first()
+            # Accessing the list must not trigger dereferencing.
+            book.authors
+            assert qc == 1
+
+        for ref in book.authors:
+            with pytest.raises(AttributeError):
+                ref["author"].name
+            assert isinstance(ref.author, LazyReference)
+            assert isinstance(ref.author.id, ObjectId)
+
 
 class TestGenericLazyReferenceField(MongoDBTestCase):
     def test_generic_lazy_reference_simple(self):

tests/fields/test_long_field.py

@@ -25,8 +25,7 @@ class TestLongField(MongoDBTestCase):
         assert isinstance(doc.some_long, int)
 
     def test_long_validation(self):
-        """Ensure that invalid values cannot be assigned to long fields.
-        """
+        """Ensure that invalid values cannot be assigned to long fields."""
 
         class TestDocument(Document):
             value = LongField(min_value=0, max_value=110)

tests/fields/test_map_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import datetime
 
 import pytest
@@ -136,11 +135,11 @@ class TestMapField(MongoDBTestCase):
 
         BlogPost.drop_collection()
 
-        tree = BlogPost(info_dict={u"éééé": {"description": u"VALUE: éééé"}})
+        tree = BlogPost(info_dict={"éééé": {"description": "VALUE: éééé"}})
 
         tree.save()
 
         assert (
-            BlogPost.objects.get(id=tree.id).info_dict[u"éééé"].description
-            == u"VALUE: éééé"
+            BlogPost.objects.get(id=tree.id).info_dict["éééé"].description
+            == "VALUE: éééé"
         )

tests/fields/test_reference_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from bson import DBRef, SON
 import pytest
 
@@ -88,7 +87,7 @@ class TestReferenceField(MongoDBTestCase):
             parent = ReferenceField("self", dbref=False)
 
         p = Person(name="Steve", parent=DBRef("person", "abcdefghijklmnop"))
-        assert p.to_mongo() == SON([("name", u"Steve"), ("parent", "abcdefghijklmnop")])
+        assert p.to_mongo() == SON([("name", "Steve"), ("parent", "abcdefghijklmnop")])
 
     def test_objectid_reference_fields(self):
         class Person(Document):
@@ -108,8 +107,7 @@ class TestReferenceField(MongoDBTestCase):
         assert p.parent == p1
 
     def test_undefined_reference(self):
-        """Ensure that ReferenceFields may reference undefined Documents.
-        """
+        """Ensure that ReferenceFields may reference undefined Documents."""
 
         class Product(Document):
             name = StringField()

tests/fields/test_sequence_field.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 from mongoengine import *
 
 from tests.utils import MongoDBTestCase
@@ -168,8 +166,8 @@ class TestSequenceField(MongoDBTestCase):
         ids = [i.id for i in Person.objects]
         assert ids == list(range(1, 11))
 
-        id = [i.id for i in Animal.objects]
-        assert id == list(range(1, 11))
+        _id = [i.id for i in Animal.objects]
+        assert _id == list(range(1, 11))
 
         c = self.db["mongoengine.counters"].find_one({"_id": "person.id"})
         assert c["next"] == 10

tests/fields/test_url_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import pytest
 
 from mongoengine import *
@@ -27,7 +26,7 @@ class TestURLField(MongoDBTestCase):
             url = URLField()
 
         link = Link()
-        link.url = u"http://привет.com"
+        link.url = "http://привет.com"
 
         # TODO fix URL validation - this *IS* a valid URL
         # For now we just want to make sure that the error message is correct
@@ -35,12 +34,11 @@ class TestURLField(MongoDBTestCase):
             link.validate()
         assert (
             str(exc_info.value)
-            == u"ValidationError (Link:None) (Invalid URL: http://\u043f\u0440\u0438\u0432\u0435\u0442.com: ['url'])"
+            == "ValidationError (Link:None) (Invalid URL: http://\u043f\u0440\u0438\u0432\u0435\u0442.com: ['url'])"
         )
 
     def test_url_scheme_validation(self):
-        """Ensure that URLFields validate urls with specific schemes properly.
-        """
+        """Ensure that URLFields validate urls with specific schemes properly."""
 
         class Link(Document):
             url = URLField()

tests/fields/test_uuid_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import uuid
 
 import pytest
@@ -18,8 +17,7 @@ class TestUUIDField(MongoDBTestCase):
         assert get_as_pymongo(person) == {"_id": person.id, "api_key": str(uid)}
 
     def test_field_string(self):
-        """Test UUID fields storing as String
-        """
+        """Test UUID fields storing as String"""
         Person.drop_collection()
 
         uu = uuid.uuid4()

tests/fixtures.py

@@ -53,7 +53,7 @@ signals.post_save.connect(PickleSignalsTest.post_save, sender=PickleSignalsTest)
 signals.post_delete.connect(PickleSignalsTest.post_delete, sender=PickleSignalsTest)
 
 
-class Mixin(object):
+class Mixin:
     name = StringField()
 
 

tests/queryset/test_field_list.py

@@ -148,8 +148,7 @@ class TestOnlyExcludeAll(unittest.TestCase):
         assert qs._loaded_fields.as_dict() == {"c": {"$slice": 2}, "a": 1}
 
     def test_only(self):
-        """Ensure that QuerySet.only only returns the requested fields.
-        """
+        """Ensure that QuerySet.only only returns the requested fields."""
         person = self.Person(name="test", age=25)
         person.save()
 
@@ -365,8 +364,7 @@ class TestOnlyExcludeAll(unittest.TestCase):
         Email.drop_collection()
 
     def test_slicing_fields(self):
-        """Ensure that query slicing an array works.
-        """
+        """Ensure that query slicing an array works."""
 
         class Numbers(Document):
             n = ListField(IntField())
@@ -401,8 +399,7 @@ class TestOnlyExcludeAll(unittest.TestCase):
         assert numbers.n == [-5, -4, -3, -2, -1]
 
     def test_slicing_nested_fields(self):
-        """Ensure that query slicing an embedded array works.
-        """
+        """Ensure that query slicing an embedded array works."""
 
         class EmbeddedNumber(EmbeddedDocument):
             n = ListField(IntField())

tests/queryset/test_geo.py

@@ -496,8 +496,8 @@ class TestGeoQueries(MongoDBTestCase):
         p.save()
         qs = Place.objects().only("location")
         assert qs.as_pymongo()[0]["location"] == {
-            u"type": u"Point",
-            u"coordinates": [24.946861267089844, 60.16311983618494],
+            "type": "Point",
+            "coordinates": [24.946861267089844, 60.16311983618494],
         }
 
     def test_2dsphere_point_sets_correctly(self):

tests/queryset/test_pickable.py

@@ -1,8 +1,6 @@
 import pickle
-import unittest
 
 from mongoengine import Document, IntField, StringField
-from mongoengine.connection import connect
 from tests.utils import MongoDBTestCase
 
 
@@ -18,18 +16,15 @@ class TestQuerysetPickable(MongoDBTestCase):
     """
 
     def setUp(self):
-        super(TestQuerysetPickable, self).setUp()
+        super().setUp()
         self.john = Person.objects.create(name="John", age=21)
 
     def test_picke_simple_qs(self):
-
         qs = Person.objects.all()
-
         pickle.dumps(qs)
 
     def _get_loaded(self, qs):
         s = pickle.dumps(qs)
-
         return pickle.loads(s)
 
     def test_unpickle(self):

tests/queryset/test_queryset.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 import datetime
 import unittest
 import uuid
@@ -7,7 +5,6 @@ from decimal import Decimal
 
 from bson import DBRef, ObjectId
 import pymongo
-from pymongo.read_concern import ReadConcern
 from pymongo.read_preferences import ReadPreference
 from pymongo.results import UpdateResult
 import pytest
@@ -62,8 +59,7 @@ class TestQueryset(unittest.TestCase):
         self.mongodb_version = get_mongodb_version()
 
     def test_initialisation(self):
-        """Ensure that a QuerySet is correctly initialised by QuerySetManager.
-        """
+        """Ensure that a QuerySet is correctly initialised by QuerySetManager."""
         assert isinstance(self.Person.objects, QuerySet)
         assert (
             self.Person.objects._collection.name == self.Person._get_collection_name()
@@ -114,6 +110,38 @@ class TestQueryset(unittest.TestCase):
         assert person.name == "User A"
         assert person.age == 20
 
+    def test_slicing_sets_empty_limit_skip(self):
+        self.Person.objects.insert(
+            [self.Person(name=f"User {i}", age=i) for i in range(5)],
+            load_bulk=False,
+        )
+
+        self.Person.objects.create(name="User B", age=30)
+        self.Person.objects.create(name="User C", age=40)
+
+        qs = self.Person.objects()[1:2]
+        assert (qs._empty, qs._skip, qs._limit) == (False, 1, 1)
+        assert len(list(qs)) == 1
+
+        # Test edge case of [1:1] which should return nothing
+        # and require a hack so that it doesn't clash with limit(0)
+        qs = self.Person.objects()[1:1]
+        assert (qs._empty, qs._skip, qs._limit) == (True, 1, 0)
+        assert len(list(qs)) == 0
+
+        qs2 = qs[1:5]  # Make sure that further slicing resets _empty
+        assert (qs2._empty, qs2._skip, qs2._limit) == (False, 1, 4)
+        assert len(list(qs2)) == 4
+
+    def test_limit_0_returns_all_documents(self):
+        self.Person.objects.create(name="User A", age=20)
+        self.Person.objects.create(name="User B", age=30)
+
+        n_docs = self.Person.objects().count()
+
+        persons = list(self.Person.objects().limit(0))
+        assert len(persons) == 2 == n_docs
+
     def test_limit(self):
         """Ensure that QuerySet.limit works as expected."""
         user_a = self.Person.objects.create(name="User A", age=20)
@@ -241,8 +269,7 @@ class TestQueryset(unittest.TestCase):
         )
 
     def test_find_one(self):
-        """Ensure that a query using find_one returns a valid result.
-        """
+        """Ensure that a query using find_one returns a valid result."""
         person1 = self.Person(name="User A", age=20)
         person1.save()
         person2 = self.Person(name="User B", age=30)
@@ -287,8 +314,7 @@ class TestQueryset(unittest.TestCase):
             self.Person.objects.get()
 
     def test_get_multiple_match_raises_multipleobjectsreturned(self):
-        """Ensure that a query using ``get`` returns at most one result.
-        """
+        """Ensure that a query using ``get`` returns at most one result."""
         assert self.Person.objects().count() == 0
 
         person1 = self.Person(name="User A", age=20)
@@ -321,8 +347,7 @@ class TestQueryset(unittest.TestCase):
         assert person == person3
 
     def test_find_array_position(self):
-        """Ensure that query by array position works.
-        """
+        """Ensure that query by array position works."""
 
         class Comment(EmbeddedDocument):
             name = StringField()
@@ -377,6 +402,9 @@ class TestQueryset(unittest.TestCase):
 
         assert list(A.objects.none()) == []
         assert list(A.objects.none().all()) == []
+        assert list(A.objects.none().limit(1)) == []
+        assert list(A.objects.none().skip(1)) == []
+        assert list(A.objects.none()[:5]) == []
 
     def test_chaining(self):
         class A(Document):
@@ -420,7 +448,7 @@ class TestQueryset(unittest.TestCase):
 
         # test iterating over the result set
         cnt = 0
-        for a in A.objects.batch_size(10):
+        for _ in A.objects.batch_size(10):
             cnt += 1
         assert cnt == 100
 
@@ -428,7 +456,7 @@ class TestQueryset(unittest.TestCase):
         qs = A.objects.all()
         qs = qs.limit(10).batch_size(20).skip(91)
         cnt = 0
-        for a in qs:
+        for _ in qs:
             cnt += 1
         assert cnt == 9
 
@@ -1108,8 +1136,7 @@ class TestQueryset(unittest.TestCase):
             assert q == 2
 
     def test_repeated_iteration(self):
-        """Ensure that QuerySet rewinds itself one iteration finishes.
-        """
+        """Ensure that QuerySet rewinds itself one iteration finishes."""
         self.Person(name="Person 1").save()
         self.Person(name="Person 2").save()
 
@@ -1118,7 +1145,7 @@ class TestQueryset(unittest.TestCase):
         people2 = [person for person in queryset]
 
         # Check that it still works even if iteration is interrupted.
-        for person in queryset:
+        for _person in queryset:
             break
         people3 = [person for person in queryset]
 
@@ -1154,12 +1181,11 @@ class TestQueryset(unittest.TestCase):
         assert "[<Doc: 1>, <Doc: 2>, <Doc: 3>]" == "%s" % docs
 
         assert docs.count(with_limit_and_skip=True) == 3
-        for doc in docs:
+        for _ in docs:
             assert ".. queryset mid-iteration .." == repr(docs)
 
     def test_regex_query_shortcuts(self):
-        """Ensure that contains, startswith, endswith, etc work.
-        """
+        """Ensure that contains, startswith, endswith, etc work."""
         person = self.Person(name="Guido van Rossum")
         person.save()
 
@@ -1215,8 +1241,7 @@ class TestQueryset(unittest.TestCase):
         assert obj == person
 
     def test_not(self):
-        """Ensure that the __not operator works as expected.
-        """
+        """Ensure that the __not operator works as expected."""
         alice = self.Person(name="Alice", age=25)
         alice.save()
 
@@ -1227,8 +1252,7 @@ class TestQueryset(unittest.TestCase):
         assert obj is None
 
     def test_filter_chaining(self):
-        """Ensure filters can be chained together.
-        """
+        """Ensure filters can be chained together."""
 
         class Blog(Document):
             id = StringField(primary_key=True)
@@ -1304,8 +1328,7 @@ class TestQueryset(unittest.TestCase):
             assert qs[i] == expected[i]
 
     def test_ordering(self):
-        """Ensure default ordering is applied and can be overridden.
-        """
+        """Ensure default ordering is applied and can be overridden."""
 
         class BlogPost(Document):
             title = StringField()
@@ -1375,8 +1398,7 @@ class TestQueryset(unittest.TestCase):
             assert ORDER_BY_KEY not in q.get_ops()[0][CMD_QUERY_KEY]
 
     def test_no_ordering_for_get(self):
-        """ Ensure that Doc.objects.get doesn't use any ordering.
-        """
+        """Ensure that Doc.objects.get doesn't use any ordering."""
         ORDER_BY_KEY, CMD_QUERY_KEY = get_key_compat(self.mongodb_version)
 
         class BlogPost(Document):
@@ -1452,8 +1474,7 @@ class TestQueryset(unittest.TestCase):
         assert result.author is None
 
     def test_find_dict_item(self):
-        """Ensure that DictField items may be found.
-        """
+        """Ensure that DictField items may be found."""
 
         class BlogPost(Document):
             info = DictField()
@@ -1469,8 +1490,7 @@ class TestQueryset(unittest.TestCase):
         BlogPost.drop_collection()
 
     def test_exec_js_query(self):
-        """Ensure that queries are properly formed for use in exec_js.
-        """
+        """Ensure that queries are properly formed for use in exec_js."""
 
         class BlogPost(Document):
             hits = IntField()
@@ -1507,8 +1527,7 @@ class TestQueryset(unittest.TestCase):
         BlogPost.drop_collection()
 
     def test_exec_js_field_sub(self):
-        """Ensure that field substitutions occur properly in exec_js functions.
-        """
+        """Ensure that field substitutions occur properly in exec_js functions."""
 
         class Comment(EmbeddedDocument):
             content = StringField(db_field="body")
@@ -1550,9 +1569,9 @@ class TestQueryset(unittest.TestCase):
 
         results = BlogPost.objects.exec_js(code)
         expected_results = [
-            {u"comment": u"cool", u"document": u"post1"},
-            {u"comment": u"yay", u"document": u"post1"},
-            {u"comment": u"nice stuff", u"document": u"post2"},
+            {"comment": "cool", "document": "post1"},
+            {"comment": "yay", "document": "post1"},
+            {"comment": "nice stuff", "document": "post2"},
         ]
         assert results == expected_results
 
@@ -1564,8 +1583,7 @@ class TestQueryset(unittest.TestCase):
         BlogPost.drop_collection()
 
     def test_delete(self):
-        """Ensure that documents are properly deleted from the database.
-        """
+        """Ensure that documents are properly deleted from the database."""
         self.Person(name="User A", age=20).save()
         self.Person(name="User B", age=30).save()
         self.Person(name="User C", age=40).save()
@@ -1579,8 +1597,7 @@ class TestQueryset(unittest.TestCase):
         assert self.Person.objects.count() == 0
 
     def test_reverse_delete_rule_cascade(self):
-        """Ensure cascading deletion of referring documents from the database.
-        """
+        """Ensure cascading deletion of referring documents from the database."""
 
         class BlogPost(Document):
             content = StringField()
@@ -1712,8 +1729,7 @@ class TestQueryset(unittest.TestCase):
         assert 0 == Category.objects.count()
 
     def test_reverse_delete_rule_nullify(self):
-        """Ensure nullification of references to deleted documents.
-        """
+        """Ensure nullification of references to deleted documents."""
 
         class Category(Document):
             name = StringField()
@@ -1809,8 +1825,7 @@ class TestQueryset(unittest.TestCase):
             self.Person.objects.delete()
 
     def test_reverse_delete_rule_pull(self):
-        """Ensure pulling of references to deleted documents.
-        """
+        """Ensure pulling of references to deleted documents."""
 
         class BlogPost(Document):
             content = StringField()
@@ -1885,8 +1900,7 @@ class TestQueryset(unittest.TestCase):
         assert 8 == Log.objects.count()
 
     def test_delete_with_limit_handles_delete_rules(self):
-        """Ensure cascading deletion of referring documents from the database.
-        """
+        """Ensure cascading deletion of referring documents from the database."""
 
         class BlogPost(Document):
             content = StringField()
@@ -1918,8 +1932,7 @@ class TestQueryset(unittest.TestCase):
         assert del_result is None
 
     def test_reference_field_find(self):
-        """Ensure cascading deletion of referring documents from the database.
-        """
+        """Ensure cascading deletion of referring documents from the database."""
 
         class BlogPost(Document):
             content = StringField()
@@ -1940,8 +1953,7 @@ class TestQueryset(unittest.TestCase):
         assert 1 == BlogPost.objects(author__in=["%s" % me.pk]).count()
 
     def test_reference_field_find_dbref(self):
-        """Ensure cascading deletion of referring documents from the database.
-        """
+        """Ensure cascading deletion of referring documents from the database."""
 
         class BlogPost(Document):
             content = StringField()
@@ -2025,8 +2037,7 @@ class TestQueryset(unittest.TestCase):
             BlogPost.objects.update_one(inc__review=0.1)  # test with floats
 
     def test_update_listfield_operator(self):
-        """Ensure that atomic updates work properly.
-        """
+        """Ensure that atomic updates work properly."""
 
         class BlogPost(Document):
             tags = ListField(StringField())
@@ -2074,8 +2085,7 @@ class TestQueryset(unittest.TestCase):
         assert "title" not in pymongo_doc
 
     def test_update_push_with_position(self):
-        """Ensure that the 'push' update with position works properly.
-        """
+        """Ensure that the 'push' update with position works properly."""
 
         class BlogPost(Document):
             slug = StringField()
@@ -2100,8 +2110,7 @@ class TestQueryset(unittest.TestCase):
         assert post.tags == ["scala", "mongodb", "python", "java"]
 
     def test_update_push_list_of_list(self):
-        """Ensure that the 'push' update operation works in the list of list
-        """
+        """Ensure that the 'push' update operation works in the list of list"""
 
         class BlogPost(Document):
             slug = StringField()
@@ -2116,8 +2125,7 @@ class TestQueryset(unittest.TestCase):
         assert post.tags == [["value1", 123]]
 
     def test_update_push_and_pull_add_to_set(self):
-        """Ensure that the 'pull' update operation works correctly.
-        """
+        """Ensure that the 'pull' update operation works correctly."""
 
         class BlogPost(Document):
             slug = StringField()
@@ -2226,8 +2234,7 @@ class TestQueryset(unittest.TestCase):
             )
 
     def test_pull_from_nested_embedded_using_in_nin(self):
-        """Ensure that the 'pull' update operation works on embedded documents using 'in' and 'nin' operators.
-        """
+        """Ensure that the 'pull' update operation works on embedded documents using 'in' and 'nin' operators."""
 
         class User(EmbeddedDocument):
             name = StringField()
@@ -2458,8 +2465,7 @@ class TestQueryset(unittest.TestCase):
         assert doc.pk == doc.embedded[0]._instance.pk
 
     def test_order_by(self):
-        """Ensure that QuerySets may be ordered.
-        """
+        """Ensure that QuerySets may be ordered."""
         self.Person(name="User B", age=40).save()
         self.Person(name="User A", age=20).save()
         self.Person(name="User C", age=30).save()
@@ -2527,8 +2533,7 @@ class TestQueryset(unittest.TestCase):
         self.assertSequence(qs, expected)
 
     def test_order_by_chaining(self):
-        """Ensure that an order_by query chains properly and allows .only()
-        """
+        """Ensure that an order_by query chains properly and allows .only()"""
         self.Person(name="User B", age=40).save()
         self.Person(name="User A", age=20).save()
         self.Person(name="User C", age=30).save()
@@ -2602,8 +2607,7 @@ class TestQueryset(unittest.TestCase):
                 assert op[CMD_QUERY_KEY][COMMENT_KEY] == "looking for an adult"
 
     def test_map_reduce(self):
-        """Ensure map/reduce is both mapping and reducing.
-        """
+        """Ensure map/reduce is both mapping and reducing."""
 
         class BlogPost(Document):
             title = StringField()
@@ -2716,10 +2720,10 @@ class TestQueryset(unittest.TestCase):
         f1.save()
 
         # persons of first family
-        Person(id=1, family=f1, name=u"Wilson Jr", age=21).save()
-        Person(id=2, family=f1, name=u"Wilson Father", age=45).save()
-        Person(id=3, family=f1, name=u"Eliana Costa", age=40).save()
-        Person(id=4, family=f1, name=u"Tayza Mariana", age=17).save()
+        Person(id=1, family=f1, name="Wilson Jr", age=21).save()
+        Person(id=2, family=f1, name="Wilson Father", age=45).save()
+        Person(id=3, family=f1, name="Eliana Costa", age=40).save()
+        Person(id=4, family=f1, name="Tayza Mariana", age=17).save()
 
         # creating second family
         f2 = Family(id=2, log="Av prof frasc brunno")
@@ -2797,10 +2801,10 @@ class TestQueryset(unittest.TestCase):
             "_id": 1,
             "value": {
                 "persons": [
-                    {"age": 21, "name": u"Wilson Jr"},
-                    {"age": 45, "name": u"Wilson Father"},
-                    {"age": 40, "name": u"Eliana Costa"},
-                    {"age": 17, "name": u"Tayza Mariana"},
+                    {"age": 21, "name": "Wilson Jr"},
+                    {"age": 45, "name": "Wilson Father"},
+                    {"age": 40, "name": "Eliana Costa"},
+                    {"age": 17, "name": "Tayza Mariana"},
                 ],
                 "totalAge": 123,
             },
@@ -2810,9 +2814,9 @@ class TestQueryset(unittest.TestCase):
             "_id": 2,
             "value": {
                 "persons": [
-                    {"age": 16, "name": u"Isabella Luanna"},
-                    {"age": 36, "name": u"Sandra Mara"},
-                    {"age": 10, "name": u"Igor Gabriel"},
+                    {"age": 16, "name": "Isabella Luanna"},
+                    {"age": 36, "name": "Sandra Mara"},
+                    {"age": 10, "name": "Igor Gabriel"},
                 ],
                 "totalAge": 62,
             },
@@ -2822,8 +2826,8 @@ class TestQueryset(unittest.TestCase):
             "_id": 3,
             "value": {
                 "persons": [
-                    {"age": 30, "name": u"Arthur WA"},
-                    {"age": 25, "name": u"Paula Leonel"},
+                    {"age": 30, "name": "Arthur WA"},
+                    {"age": 25, "name": "Paula Leonel"},
                 ],
                 "totalAge": 55,
             },
@@ -2955,8 +2959,7 @@ class TestQueryset(unittest.TestCase):
         Link.drop_collection()
 
     def test_item_frequencies(self):
-        """Ensure that item frequencies are properly generated from lists.
-        """
+        """Ensure that item frequencies are properly generated from lists."""
 
         class BlogPost(Document):
             hits = IntField()
@@ -2970,7 +2973,7 @@ class TestQueryset(unittest.TestCase):
 
         def test_assertions(f):
             f = {key: int(val) for key, val in f.items()}
-            assert set(["music", "film", "actors", "watch"]) == set(f.keys())
+            assert {"music", "film", "actors", "watch"} == set(f.keys())
             assert f["music"] == 3
             assert f["actors"] == 2
             assert f["watch"] == 2
@@ -2984,7 +2987,7 @@ class TestQueryset(unittest.TestCase):
         # Ensure query is taken into account
         def test_assertions(f):
             f = {key: int(val) for key, val in f.items()}
-            assert set(["music", "actors", "watch"]) == set(f.keys())
+            assert {"music", "actors", "watch"} == set(f.keys())
             assert f["music"] == 2
             assert f["actors"] == 1
             assert f["watch"] == 1
@@ -3012,7 +3015,7 @@ class TestQueryset(unittest.TestCase):
 
         # Check item_frequencies works for non-list fields
         def test_assertions(f):
-            assert set([1, 2]) == set(f.keys())
+            assert {1, 2} == set(f.keys())
             assert f[1] == 1
             assert f[2] == 2
 
@@ -3024,8 +3027,7 @@ class TestQueryset(unittest.TestCase):
         BlogPost.drop_collection()
 
     def test_item_frequencies_on_embedded(self):
-        """Ensure that item frequencies are properly generated from lists.
-        """
+        """Ensure that item frequencies are properly generated from lists."""
 
         class Phone(EmbeddedDocument):
             number = StringField()
@@ -3050,7 +3052,7 @@ class TestQueryset(unittest.TestCase):
 
         def test_assertions(f):
             f = {key: int(val) for key, val in f.items()}
-            assert set(["62-3331-1656", "62-3332-1656"]) == set(f.keys())
+            assert {"62-3331-1656", "62-3332-1656"} == set(f.keys())
             assert f["62-3331-1656"] == 2
             assert f["62-3332-1656"] == 1
 
@@ -3062,7 +3064,7 @@ class TestQueryset(unittest.TestCase):
         # Ensure query is taken into account
         def test_assertions(f):
             f = {key: int(val) for key, val in f.items()}
-            assert set(["62-3331-1656"]) == set(f.keys())
+            assert {"62-3331-1656"} == set(f.keys())
             assert f["62-3331-1656"] == 2
 
         exec_js = Person.objects(phone__number="62-3331-1656").item_frequencies(
@@ -3129,10 +3131,10 @@ class TestQueryset(unittest.TestCase):
         p.save()
 
         ot = Person.objects.item_frequencies("extra.tag", map_reduce=False)
-        assert ot == {None: 1.0, u"friend": 1.0}
+        assert ot == {None: 1.0, "friend": 1.0}
 
         ot = Person.objects.item_frequencies("extra.tag", map_reduce=True)
-        assert ot == {None: 1.0, u"friend": 1.0}
+        assert ot == {None: 1.0, "friend": 1.0}
 
     def test_item_frequencies_with_0_values(self):
         class Test(Document):
@@ -3168,10 +3170,10 @@ class TestQueryset(unittest.TestCase):
 
         Test.drop_collection()
 
-        for i in range(50):
+        for _ in range(50):
             Test(val=1).save()
 
-        for i in range(20):
+        for _ in range(20):
             Test(val=2).save()
 
         freqs = Test.objects.item_frequencies("val", map_reduce=False, normalize=True)
@@ -3181,8 +3183,7 @@ class TestQueryset(unittest.TestCase):
         assert freqs == {1: 50.0 / 70, 2: 20.0 / 70}
 
     def test_average(self):
-        """Ensure that field can be averaged correctly.
-        """
+        """Ensure that field can be averaged correctly."""
         self.Person(name="person", age=0).save()
         assert int(self.Person.objects.average("age")) == 0
 
@@ -3205,7 +3206,7 @@ class TestQueryset(unittest.TestCase):
 
         for i, weight in enumerate(ages):
             self.Person(
-                name="test meta%i", person_meta=self.PersonMeta(weight=weight)
+                name=f"test meta{i}", person_meta=self.PersonMeta(weight=weight)
             ).save()
 
         assert (
@@ -3222,8 +3223,7 @@ class TestQueryset(unittest.TestCase):
         assert self.Person.objects.filter(age__gte=50).average("age") == avg
 
     def test_sum(self):
-        """Ensure that field can be summed over correctly.
-        """
+        """Ensure that field can be summed over correctly."""
         ages = [23, 54, 12, 94, 27]
         for i, age in enumerate(ages):
             self.Person(name="test%s" % i, age=age).save()
@@ -3245,7 +3245,7 @@ class TestQueryset(unittest.TestCase):
 
         # test summing over a filtered queryset
         assert self.Person.objects.filter(age__gte=50).sum("age") == sum(
-            [a for a in ages if a >= 50]
+            a for a in ages if a >= 50
         )
 
     def test_sum_over_db_field(self):
@@ -3373,19 +3373,21 @@ class TestQueryset(unittest.TestCase):
         assert Doc.objects.sum("values") == 1360
 
     def test_distinct(self):
-        """Ensure that the QuerySet.distinct method works.
-        """
+        """Ensure that the QuerySet.distinct method works."""
         self.Person(name="Mr Orange", age=20).save()
         self.Person(name="Mr White", age=20).save()
         self.Person(name="Mr Orange", age=30).save()
         self.Person(name="Mr Pink", age=30).save()
-        assert set(self.Person.objects.distinct("name")) == set(
-            ["Mr Orange", "Mr White", "Mr Pink"]
-        )
-        assert set(self.Person.objects.distinct("age")) == set([20, 30])
-        assert set(self.Person.objects(age=30).distinct("name")) == set(
-            ["Mr Orange", "Mr Pink"]
-        )
+        assert set(self.Person.objects.distinct("name")) == {
+            "Mr Orange",
+            "Mr White",
+            "Mr Pink",
+        }
+        assert set(self.Person.objects.distinct("age")) == {20, 30}
+        assert set(self.Person.objects(age=30).distinct("name")) == {
+            "Mr Orange",
+            "Mr Pink",
+        }
 
     def test_distinct_handles_references(self):
         class Foo(Document):
@@ -3446,8 +3448,8 @@ class TestQueryset(unittest.TestCase):
         assert count == 1
 
         News(
-            title=u"As eleições no Brasil já estão em planejamento",
-            content=u"A candidata dilma roussef já começa o teu planejamento",
+            title="As eleições no Brasil já estão em planejamento",
+            content="A candidata dilma roussef já começa o teu planejamento",
             is_active=False,
         ).save()
 
@@ -3514,8 +3516,7 @@ class TestQueryset(unittest.TestCase):
         assert Foo.objects.distinct("bar") == [bar]
 
     def test_distinct_handles_db_field(self):
-        """Ensure that distinct resolves field name to db_field as expected.
-        """
+        """Ensure that distinct resolves field name to db_field as expected."""
 
         class Product(Document):
             product_id = IntField(db_field="pid")
@@ -3526,8 +3527,8 @@ class TestQueryset(unittest.TestCase):
         Product(product_id=2).save()
         Product(product_id=1).save()
 
-        assert set(Product.objects.distinct("product_id")) == set([1, 2])
-        assert set(Product.objects.distinct("pid")) == set([1, 2])
+        assert set(Product.objects.distinct("product_id")) == {1, 2}
+        assert set(Product.objects.distinct("pid")) == {1, 2}
 
         Product.drop_collection()
 
@@ -3611,8 +3612,7 @@ class TestQueryset(unittest.TestCase):
         assert Foo.objects.distinct("bar_lst") == [bar_1, bar_2]
 
     def test_custom_manager(self):
-        """Ensure that custom QuerySetManager instances work as expected.
-        """
+        """Ensure that custom QuerySetManager instances work as expected."""
 
         class BlogPost(Document):
             tags = ListField(StringField())
@@ -3713,8 +3713,7 @@ class TestQueryset(unittest.TestCase):
         assert 1 == Bar.objects.count()
 
     def test_query_value_conversion(self):
-        """Ensure that query values are properly converted when necessary.
-        """
+        """Ensure that query values are properly converted when necessary."""
 
         class BlogPost(Document):
             author = ReferenceField(self.Person)
@@ -3740,8 +3739,7 @@ class TestQueryset(unittest.TestCase):
         BlogPost.drop_collection()
 
     def test_update_value_conversion(self):
-        """Ensure that values used in updates are converted before use.
-        """
+        """Ensure that values used in updates are converted before use."""
 
         class Group(Document):
             members = ListField(ReferenceField(self.Person))
@@ -3766,8 +3764,7 @@ class TestQueryset(unittest.TestCase):
         Group.drop_collection()
 
     def test_bulk(self):
-        """Ensure bulk querying by object id returns a proper dict.
-        """
+        """Ensure bulk querying by object id returns a proper dict."""
 
         class BlogPost(Document):
             title = StringField()
@@ -3805,8 +3802,7 @@ class TestQueryset(unittest.TestCase):
         self.Person.drop_collection()
 
     def test_custom_querysets(self):
-        """Ensure that custom QuerySet classes may be used.
-        """
+        """Ensure that custom QuerySet classes may be used."""
 
         class CustomQuerySet(QuerySet):
             def not_empty(self):
@@ -3826,8 +3822,7 @@ class TestQueryset(unittest.TestCase):
         Post.drop_collection()
 
     def test_custom_querysets_set_manager_directly(self):
-        """Ensure that custom QuerySet classes may be used.
-        """
+        """Ensure that custom QuerySet classes may be used."""
 
         class CustomQuerySet(QuerySet):
             def not_empty(self):
@@ -3850,8 +3845,7 @@ class TestQueryset(unittest.TestCase):
         Post.drop_collection()
 
     def test_custom_querysets_managers_directly(self):
-        """Ensure that custom QuerySet classes may be used.
-        """
+        """Ensure that custom QuerySet classes may be used."""
 
         class CustomQuerySetManager(QuerySetManager):
             @staticmethod
@@ -3872,8 +3866,7 @@ class TestQueryset(unittest.TestCase):
         Post.drop_collection()
 
     def test_custom_querysets_inherited(self):
-        """Ensure that custom QuerySet classes may be used.
-        """
+        """Ensure that custom QuerySet classes may be used."""
 
         class CustomQuerySet(QuerySet):
             def not_empty(self):
@@ -3895,8 +3888,7 @@ class TestQueryset(unittest.TestCase):
         Post.drop_collection()
 
     def test_custom_querysets_inherited_direct(self):
-        """Ensure that custom QuerySet classes may be used.
-        """
+        """Ensure that custom QuerySet classes may be used."""
 
         class CustomQuerySet(QuerySet):
             def not_empty(self):
@@ -3957,8 +3949,7 @@ class TestQueryset(unittest.TestCase):
         assert A.objects(b=[{"c": "c"}]).count() == 0
 
     def test_call_after_limits_set(self):
-        """Ensure that re-filtering after slicing works
-        """
+        """Ensure that re-filtering after slicing works"""
 
         class Post(Document):
             title = StringField()
@@ -3974,8 +3965,7 @@ class TestQueryset(unittest.TestCase):
         Post.drop_collection()
 
     def test_order_then_filter(self):
-        """Ensure that ordering still works after filtering.
-        """
+        """Ensure that ordering still works after filtering."""
 
         class Number(Document):
             n = IntField()
@@ -3992,8 +3982,7 @@ class TestQueryset(unittest.TestCase):
         Number.drop_collection()
 
     def test_clone(self):
-        """Ensure that cloning clones complex querysets
-        """
+        """Ensure that cloning clones complex querysets"""
 
         class Number(Document):
             n = IntField()
@@ -4021,9 +4010,33 @@ class TestQueryset(unittest.TestCase):
 
         Number.drop_collection()
 
+    def test_clone_retains_settings(self):
+        """Ensure that cloning retains the read_preference and read_concern"""
+
+        class Number(Document):
+            n = IntField()
+
+        Number.drop_collection()
+
+        qs = Number.objects
+        qs_clone = qs.clone()
+        assert qs._read_preference == qs_clone._read_preference
+        assert qs._read_concern == qs_clone._read_concern
+
+        qs = Number.objects.read_preference(ReadPreference.PRIMARY_PREFERRED)
+        qs_clone = qs.clone()
+        assert qs._read_preference == ReadPreference.PRIMARY_PREFERRED
+        assert qs._read_preference == qs_clone._read_preference
+
+        qs = Number.objects.read_concern({"level": "majority"})
+        qs_clone = qs.clone()
+        assert qs._read_concern.document == {"level": "majority"}
+        assert qs._read_concern == qs_clone._read_concern
+
+        Number.drop_collection()
+
     def test_using(self):
-        """Ensure that switching databases for a queryset is possible
-        """
+        """Ensure that switching databases for a queryset is possible"""
 
         class Number2(Document):
             n = IntField()
@@ -4075,8 +4088,7 @@ class TestQueryset(unittest.TestCase):
         Number.drop_collection()
 
     def test_order_works_with_primary(self):
-        """Ensure that order_by and primary work.
-        """
+        """Ensure that order_by and primary work."""
 
         class Number(Document):
             n = IntField(primary_key=True)
@@ -4095,8 +4107,7 @@ class TestQueryset(unittest.TestCase):
         Number.drop_collection()
 
     def test_ensure_index(self):
-        """Ensure that manual creation of indexes works.
-        """
+        """Ensure that manual creation of indexes works."""
 
         class Comment(Document):
             message = StringField()
@@ -4112,8 +4123,7 @@ class TestQueryset(unittest.TestCase):
         assert ([("_cls", 1), ("message", 1)], False, False) in info
 
     def test_where(self):
-        """Ensure that where clauses work.
-        """
+        """Ensure that where clauses work."""
 
         class IntPair(Document):
             fielda = IntField()
@@ -4214,15 +4224,15 @@ class TestQueryset(unittest.TestCase):
         ulist = list(UserDoc.objects.scalar("name", "age"))
 
         assert ulist == [
-            (u"Wilson Jr", 19),
-            (u"Wilson", 43),
-            (u"Eliana", 37),
-            (u"Tayza", 15),
+            ("Wilson Jr", 19),
+            ("Wilson", 43),
+            ("Eliana", 37),
+            ("Tayza", 15),
         ]
 
         ulist = list(UserDoc.objects.scalar("name").order_by("age"))
 
-        assert ulist == [(u"Tayza"), (u"Wilson Jr"), (u"Eliana"), (u"Wilson")]
+        assert ulist == [("Tayza"), ("Wilson Jr"), ("Eliana"), ("Wilson")]
 
     def test_scalar_embedded(self):
         class Profile(EmbeddedDocument):
@@ -4261,7 +4271,7 @@ class TestQueryset(unittest.TestCase):
 
         assert list(
             Person.objects.order_by("profile__age").scalar("profile__name")
-        ) == [u"Wilson Jr", u"Gabriel Falcao", u"Lincoln de souza", u"Walter cruz"]
+        ) == ["Wilson Jr", "Gabriel Falcao", "Lincoln de souza", "Walter cruz"]
 
         ulist = list(
             Person.objects.order_by("locale.city").scalar(
@@ -4269,10 +4279,10 @@ class TestQueryset(unittest.TestCase):
             )
         )
         assert ulist == [
-            (u"Lincoln de souza", 28, u"Belo Horizonte"),
-            (u"Walter cruz", 30, u"Brasilia"),
-            (u"Wilson Jr", 19, u"Corumba-GO"),
-            (u"Gabriel Falcao", 23, u"New York"),
+            ("Lincoln de souza", 28, "Belo Horizonte"),
+            ("Walter cruz", 30, "Brasilia"),
+            ("Wilson Jr", 19, "Corumba-GO"),
+            ("Gabriel Falcao", 23, "New York"),
         ]
 
     def test_scalar_decimal(self):
@@ -4286,7 +4296,7 @@ class TestQueryset(unittest.TestCase):
         Person(name="Wilson Jr", rating=Decimal("1.0")).save()
 
         ulist = list(Person.objects.scalar("name", "rating"))
-        assert ulist == [(u"Wilson Jr", Decimal("1.0"))]
+        assert ulist == [("Wilson Jr", Decimal("1.0"))]
 
     def test_scalar_reference_field(self):
         class State(Document):
@@ -4305,7 +4315,7 @@ class TestQueryset(unittest.TestCase):
         Person(name="Wilson JR", state=s1).save()
 
         plist = list(Person.objects.scalar("name", "state"))
-        assert plist == [(u"Wilson JR", s1)]
+        assert plist == [("Wilson JR", s1)]
 
     def test_scalar_generic_reference_field(self):
         class State(Document):
@@ -4324,7 +4334,7 @@ class TestQueryset(unittest.TestCase):
         Person(name="Wilson JR", state=s1).save()
 
         plist = list(Person.objects.scalar("name", "state"))
-        assert plist == [(u"Wilson JR", s1)]
+        assert plist == [("Wilson JR", s1)]
 
     def test_generic_reference_field_with_only_and_as_pymongo(self):
         class TestPerson(Document):
@@ -4392,8 +4402,7 @@ class TestQueryset(unittest.TestCase):
         assert list(val) == [("test", "test value")]
 
     def test_scalar_cursor_behaviour(self):
-        """Ensure that a query returns a valid set of results.
-        """
+        """Ensure that a query returns a valid set of results."""
         person1 = self.Person(name="User A", age=20)
         person1.save()
         person2 = self.Person(name="User B", age=30)
@@ -4442,7 +4451,9 @@ class TestQueryset(unittest.TestCase):
         assert len(people) == 1
         assert people[0] == "User B"
 
-        people = list(self.Person.objects[1:1].scalar("name"))
+        # people = list(self.Person.objects[1:1].scalar("name"))
+        people = self.Person.objects[1:1]
+        people = people.scalar("name")
         assert len(people) == 0
 
         # Test slice out of range
@@ -4608,8 +4619,7 @@ class TestQueryset(unittest.TestCase):
         assert [b3] == ak
 
     def test_upsert_includes_cls(self):
-        """Upserts should include _cls information for inheritable classes
-        """
+        """Upserts should include _cls information for inheritable classes"""
 
         class Test(Document):
             test = StringField()
@@ -4855,13 +4865,11 @@ class TestQueryset(unittest.TestCase):
         )
 
         results = User.objects.as_pymongo()
-        assert set(results[0].keys()) == set(["_id", "name", "age", "price"])
-        assert set(results[1].keys()) == set(
-            ["_id", "name", "age", "price", "last_login"]
-        )
+        assert set(results[0].keys()) == {"_id", "name", "age", "price"}
+        assert set(results[1].keys()) == {"_id", "name", "age", "price", "last_login"}
 
         results = User.objects.only("id", "name").as_pymongo()
-        assert set(results[0].keys()) == set(["_id", "name"])
+        assert set(results[0].keys()) == {"_id", "name"}
 
         users = User.objects.only("name", "price").as_pymongo()
         results = list(users)
@@ -5305,8 +5313,7 @@ class TestQueryset(unittest.TestCase):
             assert obj.__class__ == C
 
     def test_query_generic_embedded_document(self):
-        """Ensure that querying sub field on generic_embedded_field works
-        """
+        """Ensure that querying sub field on generic_embedded_field works"""
 
         class A(EmbeddedDocument):
             a_name = StringField()
@@ -5494,12 +5501,12 @@ class TestQueryset(unittest.TestCase):
             assert Person.objects._has_data(), "Cursor has data and returned False"
 
     def test_delete_count(self):
-        [self.Person(name="User {0}".format(i), age=i * 10).save() for i in range(1, 4)]
+        [self.Person(name=f"User {i}", age=i * 10).save() for i in range(1, 4)]
         assert (
             self.Person.objects().delete() == 3
         )  # test ordinary QuerySey delete count
 
-        [self.Person(name="User {0}".format(i), age=i * 10).save() for i in range(1, 4)]
+        [self.Person(name=f"User {i}", age=i * 10).save() for i in range(1, 4)]
 
         assert (
             self.Person.objects().skip(1).delete() == 2

tests/queryset/test_queryset_aggregation.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 import unittest
 import warnings
 
@@ -66,7 +64,7 @@ class TestQuerysetAggregate(MongoDBTestCase):
 
         pipeline = [{"$match": {"name": "Isabella Luanna"}}]
         data = Person.objects().aggregate(pipeline)
-        assert list(data) == [{u"_id": p1.pk, u"age": 16, u"name": u"Isabella Luanna"}]
+        assert list(data) == [{"_id": p1.pk, "age": 16, "name": "Isabella Luanna"}]
 
     def test_queryset_aggregation_with_skip(self):
         class Person(Document):

tests/queryset/test_transform.py

@@ -12,8 +12,7 @@ class TestTransform(unittest.TestCase):
         connect(db="mongoenginetest")
 
     def test_transform_query(self):
-        """Ensure that the _transform_query function operates correctly.
-        """
+        """Ensure that the _transform_query function operates correctly."""
         assert transform.query(name="test", age=30) == {"name": "test", "age": 30}
         assert transform.query(age__lt=30) == {"age": {"$lt": 30}}
         assert transform.query(age__gt=20, age__lt=50) == {
@@ -88,8 +87,7 @@ class TestTransform(unittest.TestCase):
         assert update == {"$set": {"tags": ["mongo", "db"]}}
 
     def test_query_field_name(self):
-        """Ensure that the correct field name is used when querying.
-        """
+        """Ensure that the correct field name is used when querying."""
 
         class Comment(EmbeddedDocument):
             content = StringField(db_field="commentContent")
@@ -106,18 +104,17 @@ class TestTransform(unittest.TestCase):
         post = BlogPost(**data)
         post.save()
 
-        assert "postTitle" in BlogPost.objects(title=data["title"])._query
-        assert not ("title" in BlogPost.objects(title=data["title"])._query)
-        assert BlogPost.objects(title=data["title"]).count() == 1
+        qs = BlogPost.objects(title=data["title"])
+        assert qs._query == {"postTitle": data["title"]}
+        assert qs.count() == 1
 
-        assert "_id" in BlogPost.objects(pk=post.id)._query
-        assert BlogPost.objects(pk=post.id).count() == 1
+        qs = BlogPost.objects(pk=post.id)
+        assert qs._query == {"_id": post.id}
+        assert qs.count() == 1
 
-        assert (
-            "postComments.commentContent"
-            in BlogPost.objects(comments__content="test")._query
-        )
-        assert BlogPost.objects(comments__content="test").count() == 1
+        qs = BlogPost.objects(comments__content="test")
+        assert qs._query == {"postComments.commentContent": "test"}
+        assert qs.count() == 1
 
         BlogPost.drop_collection()
 
@@ -330,7 +327,7 @@ class TestTransform(unittest.TestCase):
         word = Word(word="abc", index=1)
         update = transform.update(MainDoc, pull__content__text=word)
         assert update == {
-            "$pull": {"content.text": SON([("word", u"abc"), ("index", 1)])}
+            "$pull": {"content.text": SON([("word", "abc"), ("index", 1)])}
         }
 
         update = transform.update(MainDoc, pull__content__heading="xyz")
@@ -344,6 +341,31 @@ class TestTransform(unittest.TestCase):
         )
         assert update == {"$pull": {"content.text": {"word": {"$nin": ["foo", "bar"]}}}}
 
+    def test_transform_embedded_document_list_fields(self):
+        """
+        Test added to check filtering
+        EmbeddedDocumentListField which is inside a EmbeddedDocumentField
+        """
+
+        class Drink(EmbeddedDocument):
+            id = StringField()
+            meta = {"strict": False}
+
+        class Shop(Document):
+            drinks = EmbeddedDocumentListField(Drink)
+
+        Shop.drop_collection()
+        drinks = [Drink(id="drink_1"), Drink(id="drink_2")]
+        Shop.objects.create(drinks=drinks)
+        q_obj = transform.query(
+            Shop, drinks__all=[{"$elemMatch": {"_id": x.id}} for x in drinks]
+        )
+        assert q_obj == {
+            "drinks": {"$all": [{"$elemMatch": {"_id": x.id}} for x in drinks]}
+        }
+
+        Shop.drop_collection()
+
 
 if __name__ == "__main__":
     unittest.main()

tests/queryset/test_visitor.py

@@ -23,8 +23,7 @@ class TestQ(unittest.TestCase):
         self.Person = Person
 
     def test_empty_q(self):
-        """Ensure that empty Q objects won't hurt.
-        """
+        """Ensure that empty Q objects won't hurt."""
         q1 = Q()
         q2 = Q(age__gte=18)
         q3 = Q()
@@ -58,8 +57,7 @@ class TestQ(unittest.TestCase):
         assert Post.objects.filter(Q(created_user=user)).count() == 1
 
     def test_and_combination(self):
-        """Ensure that Q-objects correctly AND together.
-        """
+        """Ensure that Q-objects correctly AND together."""
 
         class TestDoc(Document):
             x = IntField()
@@ -89,8 +87,7 @@ class TestQ(unittest.TestCase):
         assert query.to_query(TestDoc) == mongo_query
 
     def test_or_combination(self):
-        """Ensure that Q-objects correctly OR together.
-        """
+        """Ensure that Q-objects correctly OR together."""
 
         class TestDoc(Document):
             x = IntField()
@@ -101,8 +98,7 @@ class TestQ(unittest.TestCase):
         assert query == {"$or": [{"x": {"$lt": 3}}, {"x": {"$gt": 7}}]}
 
     def test_and_or_combination(self):
-        """Ensure that Q-objects handle ANDing ORed components.
-        """
+        """Ensure that Q-objects handle ANDing ORed components."""
 
         class TestDoc(Document):
             x = IntField()
@@ -136,8 +132,7 @@ class TestQ(unittest.TestCase):
         assert 2 == TestDoc.objects(q1 & q2).count()
 
     def test_or_and_or_combination(self):
-        """Ensure that Q-objects handle ORing ANDed ORed components. :)
-        """
+        """Ensure that Q-objects handle ORing ANDed ORed components. :)"""
 
         class TestDoc(Document):
             x = IntField()
@@ -208,8 +203,7 @@ class TestQ(unittest.TestCase):
         assert test.count() == 3
 
     def test_q(self):
-        """Ensure that Q objects may be used to query for documents.
-        """
+        """Ensure that Q objects may be used to query for documents."""
 
         class BlogPost(Document):
             title = StringField()
@@ -286,8 +280,7 @@ class TestQ(unittest.TestCase):
             self.Person.objects.filter("user1")
 
     def test_q_regex(self):
-        """Ensure that Q objects can be queried using regexes.
-        """
+        """Ensure that Q objects can be queried using regexes."""
         person = self.Person(name="Guido van Rossum")
         person.save()
 
@@ -320,8 +313,7 @@ class TestQ(unittest.TestCase):
         )
 
     def test_q_lists(self):
-        """Ensure that Q objects query ListFields correctly.
-        """
+        """Ensure that Q objects query ListFields correctly."""
 
         class BlogPost(Document):
             tags = ListField(StringField())

tests/test_common.py

@@ -1,5 +1,3 @@
-import unittest
-
 import pytest
 
 from mongoengine import Document

tests/test_connection.py

@@ -1,17 +1,12 @@
 import datetime
+import unittest
 
 from bson.tz_util import utc
 import pymongo
-
 from pymongo import MongoClient, ReadPreference
 from pymongo.errors import InvalidName, OperationFailure
 import pytest
 
-try:
-    import unittest2 as unittest
-except ImportError:
-    import unittest
-
 import mongoengine.connection
 from mongoengine import (
     DateTimeField,
@@ -34,18 +29,6 @@ def get_tz_awareness(connection):
     return connection.codec_options.tz_aware
 
 
-try:
-    import mongomock
-
-    MONGOMOCK_INSTALLED = True
-except ImportError:
-    MONGOMOCK_INSTALLED = False
-
-require_mongomock = pytest.mark.skipif(
-    not MONGOMOCK_INSTALLED, reason="you need mongomock installed to run this testcase"
-)
-
-
 class ConnectionTest(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
@@ -194,14 +177,12 @@ class ConnectionTest(unittest.TestCase):
         assert len(mongoengine.connection._connections) == 3
 
     def test_connect_with_invalid_db_name(self):
-        """Ensure that connect() method fails fast if db name is invalid
-        """
+        """Ensure that connect() method fails fast if db name is invalid"""
         with pytest.raises(InvalidName):
-            connect("mongomock://localhost")
+            connect("mongodb://localhost")
 
     def test_connect_with_db_name_external(self):
-        """Ensure that connect() works if db name is $external
-        """
+        """Ensure that connect() works if db name is $external"""
         """Ensure that the connect() method works properly."""
         connect("$external")
 
@@ -217,112 +198,11 @@ class ConnectionTest(unittest.TestCase):
         assert isinstance(conn, pymongo.mongo_client.MongoClient)
 
     def test_connect_with_invalid_db_name_type(self):
-        """Ensure that connect() method fails fast if db name has invalid type
-        """
+        """Ensure that connect() method fails fast if db name has invalid type"""
         with pytest.raises(TypeError):
             non_string_db_name = ["e. g. list instead of a string"]
             connect(non_string_db_name)
 
-    @require_mongomock
-    def test_connect_in_mocking(self):
-        """Ensure that the connect() method works properly in mocking.
-        """
-        connect("mongoenginetest", host="mongomock://localhost")
-        conn = get_connection()
-        assert isinstance(conn, mongomock.MongoClient)
-
-        connect("mongoenginetest2", host="mongomock://localhost", alias="testdb2")
-        conn = get_connection("testdb2")
-        assert isinstance(conn, mongomock.MongoClient)
-
-        connect(
-            "mongoenginetest3",
-            host="mongodb://localhost",
-            is_mock=True,
-            alias="testdb3",
-        )
-        conn = get_connection("testdb3")
-        assert isinstance(conn, mongomock.MongoClient)
-
-        connect("mongoenginetest4", is_mock=True, alias="testdb4")
-        conn = get_connection("testdb4")
-        assert isinstance(conn, mongomock.MongoClient)
-
-        connect(
-            host="mongodb://localhost:27017/mongoenginetest5",
-            is_mock=True,
-            alias="testdb5",
-        )
-        conn = get_connection("testdb5")
-        assert isinstance(conn, mongomock.MongoClient)
-
-        connect(host="mongomock://localhost:27017/mongoenginetest6", alias="testdb6")
-        conn = get_connection("testdb6")
-        assert isinstance(conn, mongomock.MongoClient)
-
-        connect(
-            host="mongomock://localhost:27017/mongoenginetest7",
-            is_mock=True,
-            alias="testdb7",
-        )
-        conn = get_connection("testdb7")
-        assert isinstance(conn, mongomock.MongoClient)
-
-    @require_mongomock
-    def test_default_database_with_mocking(self):
-        """Ensure that the default database is correctly set when using mongomock.
-        """
-        disconnect_all()
-
-        class SomeDocument(Document):
-            pass
-
-        conn = connect(host="mongomock://localhost:27017/mongoenginetest")
-        some_document = SomeDocument()
-        # database won't exist until we save a document
-        some_document.save()
-        assert conn.get_default_database().name == "mongoenginetest"
-        assert conn.database_names()[0] == "mongoenginetest"
-
-    @require_mongomock
-    def test_connect_with_host_list(self):
-        """Ensure that the connect() method works when host is a list
-
-        Uses mongomock to test w/o needing multiple mongod/mongos processes
-        """
-        connect(host=["mongomock://localhost"])
-        conn = get_connection()
-        assert isinstance(conn, mongomock.MongoClient)
-
-        connect(host=["mongodb://localhost"], is_mock=True, alias="testdb2")
-        conn = get_connection("testdb2")
-        assert isinstance(conn, mongomock.MongoClient)
-
-        connect(host=["localhost"], is_mock=True, alias="testdb3")
-        conn = get_connection("testdb3")
-        assert isinstance(conn, mongomock.MongoClient)
-
-        connect(
-            host=["mongomock://localhost:27017", "mongomock://localhost:27018"],
-            alias="testdb4",
-        )
-        conn = get_connection("testdb4")
-        assert isinstance(conn, mongomock.MongoClient)
-
-        connect(
-            host=["mongodb://localhost:27017", "mongodb://localhost:27018"],
-            is_mock=True,
-            alias="testdb5",
-        )
-        conn = get_connection("testdb5")
-        assert isinstance(conn, mongomock.MongoClient)
-
-        connect(
-            host=["localhost:27017", "localhost:27018"], is_mock=True, alias="testdb6"
-        )
-        conn = get_connection("testdb6")
-        assert isinstance(conn, mongomock.MongoClient)
-
     def test_disconnect_cleans_globals(self):
         """Ensure that the disconnect() method cleans the globals objects"""
         connections = mongoengine.connection._connections
@@ -452,8 +332,7 @@ class ConnectionTest(unittest.TestCase):
         disconnect_all()
 
     def test_sharing_connections(self):
-        """Ensure that connections are shared when the connection settings are exactly the same
-        """
+        """Ensure that connections are shared when the connection settings are exactly the same"""
         connect("mongoenginetests", alias="testdb1")
         expected_connection = get_connection("testdb1")
 
@@ -564,8 +443,7 @@ class ConnectionTest(unittest.TestCase):
         authd_conn.admin.system.users.delete_many({})
 
     def test_register_connection(self):
-        """Ensure that connections with different aliases may be registered.
-        """
+        """Ensure that connections with different aliases may be registered."""
         register_connection("testdb", "mongoenginetest2")
 
         with pytest.raises(ConnectionFailure):
@@ -578,8 +456,7 @@ class ConnectionTest(unittest.TestCase):
         assert db.name == "mongoenginetest2"
 
     def test_register_connection_defaults(self):
-        """Ensure that defaults are used when the host and port are None.
-        """
+        """Ensure that defaults are used when the host and port are None."""
         register_connection("testdb", "mongoenginetest", host=None, port=None)
 
         conn = get_connection("testdb")

tests/test_connection_mongomock.py

@@ -0,0 +1,173 @@
+import unittest
+
+import pytest
+
+import mongoengine.connection
+from mongoengine import (
+    Document,
+    StringField,
+    connect,
+    disconnect_all,
+)
+from mongoengine.connection import get_connection
+
+
+try:
+    import mongomock
+
+    MONGOMOCK_INSTALLED = True
+except ImportError:
+    MONGOMOCK_INSTALLED = False
+
+require_mongomock = pytest.mark.skipif(
+    not MONGOMOCK_INSTALLED, reason="you need mongomock installed to run this testcase"
+)
+
+
+class MongoMockConnectionTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        disconnect_all()
+
+    @classmethod
+    def tearDownClass(cls):
+        disconnect_all()
+
+    def tearDown(self):
+        mongoengine.connection._connection_settings = {}
+        mongoengine.connection._connections = {}
+        mongoengine.connection._dbs = {}
+
+    @require_mongomock
+    def test_connect_in_mocking(self):
+        """Ensure that the connect() method works properly in mocking."""
+        connect("mongoenginetest", host="mongomock://localhost")
+        conn = get_connection()
+        assert isinstance(conn, mongomock.MongoClient)
+
+        connect("mongoenginetest2", host="mongomock://localhost", alias="testdb2")
+        conn = get_connection("testdb2")
+        assert isinstance(conn, mongomock.MongoClient)
+
+        connect(
+            "mongoenginetest3",
+            host="mongodb://localhost",
+            is_mock=True,
+            alias="testdb3",
+        )
+        conn = get_connection("testdb3")
+        assert isinstance(conn, mongomock.MongoClient)
+
+        connect("mongoenginetest4", is_mock=True, alias="testdb4")
+        conn = get_connection("testdb4")
+        assert isinstance(conn, mongomock.MongoClient)
+
+        connect(
+            host="mongodb://localhost:27017/mongoenginetest5",
+            is_mock=True,
+            alias="testdb5",
+        )
+        conn = get_connection("testdb5")
+        assert isinstance(conn, mongomock.MongoClient)
+
+        connect(host="mongomock://localhost:27017/mongoenginetest6", alias="testdb6")
+        conn = get_connection("testdb6")
+        assert isinstance(conn, mongomock.MongoClient)
+
+        connect(
+            host="mongomock://localhost:27017/mongoenginetest7",
+            is_mock=True,
+            alias="testdb7",
+        )
+        conn = get_connection("testdb7")
+        assert isinstance(conn, mongomock.MongoClient)
+
+    @require_mongomock
+    def test_default_database_with_mocking(self):
+        """Ensure that the default database is correctly set when using mongomock."""
+        disconnect_all()
+
+        class SomeDocument(Document):
+            pass
+
+        conn = connect(host="mongomock://localhost:27017/mongoenginetest")
+        some_document = SomeDocument()
+        # database won't exist until we save a document
+        some_document.save()
+        assert SomeDocument.objects.count() == 1
+        assert conn.get_default_database().name == "mongoenginetest"
+        assert conn.list_database_names()[0] == "mongoenginetest"
+
+    @require_mongomock
+    def test_basic_queries_against_mongomock(self):
+        disconnect_all()
+
+        connect(host="mongomock://localhost:27017/mongoenginetest")
+
+        class Person(Document):
+            name = StringField()
+
+        Person.drop_collection()
+        assert Person.objects.count() == 0
+
+        bob = Person(name="Bob").save()
+        john = Person(name="John").save()
+        assert Person.objects.count() == 2
+
+        qs = Person.objects(name="Bob")
+        assert qs.count() == 1
+        assert qs.first() == bob
+        assert list(qs.as_pymongo()) == [{"_id": bob.id, "name": "Bob"}]
+
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}]
+        data = Person.objects.order_by("name").aggregate(pipeline)
+        assert list(data) == [
+            {"_id": bob.id, "name": "BOB"},
+            {"_id": john.id, "name": "JOHN"},
+        ]
+
+        Person.drop_collection()
+        assert Person.objects.count() == 0
+
+    @require_mongomock
+    def test_connect_with_host_list(self):
+        """Ensure that the connect() method works when host is a list
+
+        Uses mongomock to test w/o needing multiple mongod/mongos processes
+        """
+        connect(host=["mongomock://localhost"])
+        conn = get_connection()
+        assert isinstance(conn, mongomock.MongoClient)
+
+        connect(host=["mongodb://localhost"], is_mock=True, alias="testdb2")
+        conn = get_connection("testdb2")
+        assert isinstance(conn, mongomock.MongoClient)
+
+        connect(host=["localhost"], is_mock=True, alias="testdb3")
+        conn = get_connection("testdb3")
+        assert isinstance(conn, mongomock.MongoClient)
+
+        connect(
+            host=["mongomock://localhost:27017", "mongomock://localhost:27018"],
+            alias="testdb4",
+        )
+        conn = get_connection("testdb4")
+        assert isinstance(conn, mongomock.MongoClient)
+
+        connect(
+            host=["mongodb://localhost:27017", "mongodb://localhost:27018"],
+            is_mock=True,
+            alias="testdb5",
+        )
+        conn = get_connection("testdb5")
+        assert isinstance(conn, mongomock.MongoClient)
+
+        connect(
+            host=["localhost:27017", "localhost:27018"], is_mock=True, alias="testdb6"
+        )
+        conn = get_connection("testdb6")
+        assert isinstance(conn, mongomock.MongoClient)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_context_managers.py

@@ -117,8 +117,7 @@ class TestContextManagers:
         assert 1 == Group.objects.count()
 
     def test_no_dereference_context_manager_object_id(self):
-        """Ensure that DBRef items in ListFields aren't dereferenced.
-        """
+        """Ensure that DBRef items in ListFields aren't dereferenced."""
         connect("mongoenginetest")
 
         class User(Document):
@@ -155,8 +154,7 @@ class TestContextManagers:
         assert isinstance(group.generic, User)
 
     def test_no_dereference_context_manager_dbref(self):
-        """Ensure that DBRef items in ListFields aren't dereferenced.
-        """
+        """Ensure that DBRef items in ListFields aren't dereferenced."""
         connect("mongoenginetest")
 
         class User(Document):
@@ -182,11 +180,11 @@ class TestContextManagers:
 
         with no_dereference(Group) as Group:
             group = Group.objects.first()
-            assert all([not isinstance(m, User) for m in group.members])
+            assert all(not isinstance(m, User) for m in group.members)
             assert not isinstance(group.ref, User)
             assert not isinstance(group.generic, User)
 
-        assert all([isinstance(m, User) for m in group.members])
+        assert all(isinstance(m, User) for m in group.members)
         assert isinstance(group.ref, User)
         assert isinstance(group.generic, User)
 

tests/test_datastructures.py

@@ -6,13 +6,17 @@ from mongoengine import Document
 from mongoengine.base.datastructures import BaseDict, BaseList, StrictDict
 
 
-class DocumentStub(object):
+class DocumentStub:
     def __init__(self):
         self._changed_fields = []
+        self._unset_fields = []
 
     def _mark_as_changed(self, key):
         self._changed_fields.append(key)
 
+    def _mark_as_unset(self, key):
+        self._unset_fields.append(key)
+
 
 class TestBaseDict:
     @staticmethod
@@ -314,7 +318,7 @@ class TestBaseList:
     def test___setitem___item_0_calls_mark_as_changed(self):
         base_list = self._get_baselist([True])
         base_list[0] = False
-        assert base_list._instance._changed_fields == ["my_name"]
+        assert base_list._instance._changed_fields == ["my_name.0"]
         assert base_list == [False]
 
     def test___setitem___item_1_calls_mark_as_changed(self):
@@ -417,7 +421,7 @@ class TestStrictDict(unittest.TestCase):
         d.a = 1
         assert d.a == 1
         with pytest.raises(AttributeError):
-            getattr(d, "b")
+            d.b
 
     def test_setattr_raises_on_nonexisting_attr(self):
         d = self.dtype()

tests/test_dereference.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 
 from bson import DBRef, ObjectId
@@ -17,8 +16,7 @@ class FieldTest(unittest.TestCase):
         cls.db.drop_database("mongoenginetest")
 
     def test_list_item_dereference(self):
-        """Ensure that DBRef items in ListFields are dereferenced.
-        """
+        """Ensure that DBRef items in ListFields are dereferenced."""
 
         class User(Document):
             name = StringField()
@@ -51,7 +49,7 @@ class FieldTest(unittest.TestCase):
             len(group_obj.members)
             assert q == 2
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
 
         # Document select_related
@@ -60,7 +58,7 @@ class FieldTest(unittest.TestCase):
 
             group_obj = Group.objects.first().select_related()
             assert q == 2
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
 
         # Queryset select_related
@@ -69,15 +67,14 @@ class FieldTest(unittest.TestCase):
             group_objs = Group.objects.select_related()
             assert q == 2
             for group_obj in group_objs:
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 2
 
         User.drop_collection()
         Group.drop_collection()
 
     def test_list_item_dereference_dref_false(self):
-        """Ensure that DBRef items in ListFields are dereferenced.
-        """
+        """Ensure that DBRef items in ListFields are dereferenced."""
 
         class User(Document):
             name = StringField()
@@ -102,14 +99,14 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first()
             assert q == 1
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
             assert group_obj._data["members"]._dereferenced
 
             # verifies that no additional queries gets executed
             # if we re-iterate over the ListField once it is
             # dereferenced
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
             assert group_obj._data["members"]._dereferenced
 
@@ -120,7 +117,7 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first().select_related()
 
             assert q == 2
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
 
         # Queryset select_related
@@ -129,12 +126,11 @@ class FieldTest(unittest.TestCase):
             group_objs = Group.objects.select_related()
             assert q == 2
             for group_obj in group_objs:
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 2
 
     def test_list_item_dereference_orphan_dbref(self):
-        """Ensure that orphan DBRef items in ListFields are dereferenced.
-        """
+        """Ensure that orphan DBRef items in ListFields are dereferenced."""
 
         class User(Document):
             name = StringField()
@@ -162,14 +158,14 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first()
             assert q == 1
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
             assert group_obj._data["members"]._dereferenced
 
             # verifies that no additional queries gets executed
             # if we re-iterate over the ListField once it is
             # dereferenced
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
             assert group_obj._data["members"]._dereferenced
 
@@ -177,8 +173,7 @@ class FieldTest(unittest.TestCase):
         Group.drop_collection()
 
     def test_list_item_dereference_dref_false_stores_as_type(self):
-        """Ensure that DBRef items are stored as their type
-        """
+        """Ensure that DBRef items are stored as their type"""
 
         class User(Document):
             my_id = IntField(primary_key=True)
@@ -199,8 +194,7 @@ class FieldTest(unittest.TestCase):
         assert group.members == [user]
 
     def test_handle_old_style_references(self):
-        """Ensure that DBRef items in ListFields are dereferenced.
-        """
+        """Ensure that DBRef items in ListFields are dereferenced."""
 
         class User(Document):
             name = StringField()
@@ -233,8 +227,7 @@ class FieldTest(unittest.TestCase):
         assert group.members[-1].name == "String!"
 
     def test_migrate_references(self):
-        """Example of migrating ReferenceField storage
-        """
+        """Example of migrating ReferenceField storage"""
 
         # Create some sample data
         class User(Document):
@@ -279,8 +272,7 @@ class FieldTest(unittest.TestCase):
         assert isinstance(raw_data["members"][0], ObjectId)
 
     def test_recursive_reference(self):
-        """Ensure that ReferenceFields can reference their own documents.
-        """
+        """Ensure that ReferenceFields can reference their own documents."""
 
         class Employee(Document):
             name = StringField()
@@ -370,8 +362,7 @@ class FieldTest(unittest.TestCase):
         assert Post.objects.all()[0].user_lists == [[u1, u2], [u3]]
 
     def test_circular_reference(self):
-        """Ensure you can handle circular references
-        """
+        """Ensure you can handle circular references"""
 
         class Relation(EmbeddedDocument):
             name = StringField()
@@ -404,8 +395,7 @@ class FieldTest(unittest.TestCase):
         assert "[<Person: Mother>, <Person: Daughter>]" == "%s" % Person.objects()
 
     def test_circular_reference_on_self(self):
-        """Ensure you can handle circular references
-        """
+        """Ensure you can handle circular references"""
 
         class Person(Document):
             name = StringField()
@@ -426,13 +416,13 @@ class FieldTest(unittest.TestCase):
 
         daughter.relations.append(mother)
         daughter.relations.append(daughter)
+        assert daughter._get_changed_fields() == ["relations"]
         daughter.save()
 
         assert "[<Person: Mother>, <Person: Daughter>]" == "%s" % Person.objects()
 
     def test_circular_tree_reference(self):
-        """Ensure you can handle circular references with more than one level
-        """
+        """Ensure you can handle circular references with more than one level"""
 
         class Other(EmbeddedDocument):
             name = StringField()
@@ -515,10 +505,10 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first()
             assert q == 1
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
             for m in group_obj.members:
@@ -531,10 +521,10 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first().select_related()
             assert q == 4
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
             for m in group_obj.members:
@@ -548,18 +538,17 @@ class FieldTest(unittest.TestCase):
             assert q == 4
 
             for group_obj in group_objs:
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 4
 
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 4
 
                 for m in group_obj.members:
                     assert "User" in m.__class__.__name__
 
     def test_generic_reference_orphan_dbref(self):
-        """Ensure that generic orphan DBRef items in ListFields are dereferenced.
-        """
+        """Ensure that generic orphan DBRef items in ListFields are dereferenced."""
 
         class UserA(Document):
             name = StringField()
@@ -603,11 +592,11 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first()
             assert q == 1
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
             assert group_obj._data["members"]._dereferenced
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
             assert group_obj._data["members"]._dereferenced
 
@@ -659,10 +648,10 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first()
             assert q == 1
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
             for m in group_obj.members:
@@ -675,10 +664,10 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first().select_related()
             assert q == 4
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
             for m in group_obj.members:
@@ -692,10 +681,10 @@ class FieldTest(unittest.TestCase):
             assert q == 4
 
             for group_obj in group_objs:
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 4
 
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 4
 
                 for m in group_obj.members:
@@ -734,10 +723,10 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first()
             assert q == 1
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
 
-            for k, m in group_obj.members.items():
+            for _, m in group_obj.members.items():
                 assert isinstance(m, User)
 
         # Document select_related
@@ -747,7 +736,7 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first().select_related()
             assert q == 2
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
 
             for k, m in group_obj.members.items():
@@ -761,7 +750,7 @@ class FieldTest(unittest.TestCase):
             assert q == 2
 
             for group_obj in group_objs:
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 2
 
                 for k, m in group_obj.members.items():
@@ -812,10 +801,10 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first()
             assert q == 1
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
             for k, m in group_obj.members.items():
@@ -828,10 +817,10 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first().select_related()
             assert q == 4
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
             for k, m in group_obj.members.items():
@@ -845,10 +834,10 @@ class FieldTest(unittest.TestCase):
             assert q == 4
 
             for group_obj in group_objs:
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 4
 
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 4
 
                 for k, m in group_obj.members.items():
@@ -863,7 +852,7 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first()
             assert q == 1
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 1
             assert group_obj.members == {}
 
@@ -902,10 +891,10 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first()
             assert q == 1
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
 
             for k, m in group_obj.members.items():
@@ -918,10 +907,10 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first().select_related()
             assert q == 2
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
 
             for k, m in group_obj.members.items():
@@ -935,13 +924,13 @@ class FieldTest(unittest.TestCase):
             assert q == 2
 
             for group_obj in group_objs:
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 2
 
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 2
 
-                for k, m in group_obj.members.items():
+                for _, m in group_obj.members.items():
                     assert isinstance(m, UserA)
 
         UserA.drop_collection()
@@ -989,13 +978,13 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first()
             assert q == 1
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
-            for k, m in group_obj.members.items():
+            for _, m in group_obj.members.items():
                 assert "User" in m.__class__.__name__
 
         # Document select_related
@@ -1005,13 +994,13 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first().select_related()
             assert q == 4
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
-            for k, m in group_obj.members.items():
+            for _, m in group_obj.members.items():
                 assert "User" in m.__class__.__name__
 
         # Queryset select_related
@@ -1022,13 +1011,13 @@ class FieldTest(unittest.TestCase):
             assert q == 4
 
             for group_obj in group_objs:
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 4
 
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 4
 
-                for k, m in group_obj.members.items():
+                for _, m in group_obj.members.items():
                     assert "User" in m.__class__.__name__
 
         Group.objects.delete()
@@ -1040,7 +1029,7 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first()
             assert q == 1
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 1
 
         UserA.drop_collection()
@@ -1169,8 +1158,7 @@ class FieldTest(unittest.TestCase):
         assert msg.author.name == "new-name"
 
     def test_list_lookup_not_checked_in_map(self):
-        """Ensure we dereference list data correctly
-        """
+        """Ensure we dereference list data correctly"""
 
         class Comment(Document):
             id = IntField(primary_key=True)
@@ -1192,8 +1180,7 @@ class FieldTest(unittest.TestCase):
         assert 1 == msg.comments[1].id
 
     def test_list_item_dereference_dref_false_save_doesnt_cause_extra_queries(self):
-        """Ensure that DBRef items in ListFields are dereferenced.
-        """
+        """Ensure that DBRef items in ListFields are dereferenced."""
 
         class User(Document):
             name = StringField()
@@ -1222,8 +1209,7 @@ class FieldTest(unittest.TestCase):
             assert q == 2
 
     def test_list_item_dereference_dref_true_save_doesnt_cause_extra_queries(self):
-        """Ensure that DBRef items in ListFields are dereferenced.
-        """
+        """Ensure that DBRef items in ListFields are dereferenced."""
 
         class User(Document):
             name = StringField()
@@ -1335,7 +1321,7 @@ class FieldTest(unittest.TestCase):
         BrandGroup.drop_collection()
 
         brand1 = Brand(title="Moschino").save()
-        brand2 = Brand(title=u"Денис Симачёв").save()
+        brand2 = Brand(title="Денис Симачёв").save()
 
         BrandGroup(title="top_brands", brands=[brand1, brand2]).save()
         brand_groups = BrandGroup.objects().all()

tests/test_replicaset_connection.py

@@ -22,8 +22,7 @@ class ConnectionTest(unittest.TestCase):
         mongoengine.connection._dbs = {}
 
     def test_replicaset_uri_passes_read_preference(self):
-        """Requires a replica set called "rs" on port 27017
-        """
+        """Requires a replica set called "rs" on port 27017"""
         try:
             conn = mongoengine.connect(
                 db="mongoenginetest",

tests/test_signals.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 
 from mongoengine import *

tests/test_utils.py

@@ -1,5 +1,4 @@
 import re
-import unittest
 
 import pytest
 
@@ -10,7 +9,7 @@ signal_output = []
 
 class TestLazyRegexCompiler:
     def test_lazy_regex_compiler_verify_laziness_of_descriptor(self):
-        class UserEmail(object):
+        class UserEmail:
             EMAIL_REGEX = LazyRegexCompiler("@", flags=32)
 
         descriptor = UserEmail.__dict__["EMAIL_REGEX"]
@@ -24,7 +23,7 @@ class TestLazyRegexCompiler:
         assert user_email.EMAIL_REGEX is UserEmail.EMAIL_REGEX
 
     def test_lazy_regex_compiler_verify_cannot_set_descriptor_on_instance(self):
-        class UserEmail(object):
+        class UserEmail:
             EMAIL_REGEX = LazyRegexCompiler("@")
 
         user_email = UserEmail()
@@ -32,7 +31,7 @@ class TestLazyRegexCompiler:
             user_email.EMAIL_REGEX = re.compile("@")
 
     def test_lazy_regex_compiler_verify_can_override_class_attr(self):
-        class UserEmail(object):
+        class UserEmail:
             EMAIL_REGEX = LazyRegexCompiler("@")
 
         UserEmail.EMAIL_REGEX = re.compile("cookies")

tests/utils.py

@@ -50,7 +50,7 @@ def _decorated_with_ver_requirement(func, mongo_version_req, oper):
     ran against MongoDB < v3.6.
 
     :param mongo_version_req: The mongodb version requirement (tuple(int, int))
-    :param oper: The operator to apply (e.g: operator.ge)
+    :param oper: The operator to apply (e.g. operator.ge)
     """
 
     def _inner(*args, **kwargs):
@@ -59,7 +59,7 @@ def _decorated_with_ver_requirement(func, mongo_version_req, oper):
             return func(*args, **kwargs)
 
         pretty_version = ".".join(str(n) for n in mongo_version_req)
-        pytest.skip("Needs MongoDB v{}+".format(pretty_version))
+        pytest.skip(f"Needs MongoDB v{pretty_version}+")
 
     _inner.__name__ = func.__name__
     _inner.__doc__ = func.__doc__

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = {py35,pypy3}-{mg34,mg36,mg39,mg310}
+envlist = {py35,pypy3}-{mg34,mg36,mg39,mg311}
 
 [testenv]
 commands =
@@ -8,6 +8,6 @@ deps =
     mg34: pymongo>=3.4,<3.5
     mg36: pymongo>=3.6,<3.7
     mg39: pymongo>=3.9,<3.10
-    mg310: pymongo>=3.10,<3.11
+    mg311: pymongo>=3.11,<3.12
 setenv =
     PYTHON_EGG_CACHE = {envdir}/python-eggs